개념 설명 전체 · v6.6 / block/blk-mq.c

    1 // SPDX-License-Identifier: GPL-2.0
    2 /*
    3  * Block multiqueue core code
    4  *
    5  * Copyright (C) 2013-2014 Jens Axboe
    6  * Copyright (C) 2013-2014 Christoph Hellwig
    7  */
    8 #include <linux/kernel.h>
    9 #include <linux/module.h>
   10 #include <linux/backing-dev.h>
   11 #include <linux/bio.h>
   12 #include <linux/blkdev.h>
   13 #include <linux/blk-integrity.h>
   14 #include <linux/kmemleak.h>
   15 #include <linux/mm.h>
   16 #include <linux/init.h>
   17 #include <linux/slab.h>
   18 #include <linux/workqueue.h>
   19 #include <linux/smp.h>
   20 #include <linux/interrupt.h>
   21 #include <linux/llist.h>
   22 #include <linux/cpu.h>
   23 #include <linux/cache.h>
   24 #include <linux/sched/sysctl.h>
   25 #include <linux/sched/topology.h>
   26 #include <linux/sched/signal.h>
   27 #include <linux/delay.h>
   28 #include <linux/crash_dump.h>
   29 #include <linux/prefetch.h>
   30 #include <linux/blk-crypto.h>
   31 #include <linux/part_stat.h>
   32 
   33 #include <trace/events/block.h>
   34 
   35 #include <linux/t10-pi.h>
   36 #include "blk.h"
   37 #include "blk-mq.h"
   38 #include "blk-mq-debugfs.h"
   39 #include "blk-pm.h"
   40 #include "blk-stat.h"
   41 #include "blk-mq-sched.h"
   42 #include "blk-rq-qos.h"
   43 #include "blk-ioprio.h"
   44 
   45 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
   46 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
   47 
   48 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags);
   49 static void blk_mq_request_bypass_insert(struct request *rq,
   50 		blk_insert_t flags);
   51 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
   52 		struct list_head *list);
   53 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
   54 			 struct io_comp_batch *iob, unsigned int flags);
   55 
   56 /*
   57  * Check if any of the ctx, dispatch list or elevator
   58  * have pending work in this hardware queue.
   59  */
   60 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
   61 {
   62 	return !list_empty_careful(&hctx->dispatch) ||
   63 		sbitmap_any_bit_set(&hctx->ctx_map) ||
   64 			blk_mq_sched_has_work(hctx);
   65 }
   66 
   67 /*
   68  * Mark this ctx as having pending work in this hardware queue
   69  */
   70 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
   71 				     struct blk_mq_ctx *ctx)
   72 {
   73 	const int bit = ctx->index_hw[hctx->type];
   74 
   75 	if (!sbitmap_test_bit(&hctx->ctx_map, bit))
   76 		sbitmap_set_bit(&hctx->ctx_map, bit);
   77 }
   78 
   79 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
   80 				      struct blk_mq_ctx *ctx)
   81 {
   82 	const int bit = ctx->index_hw[hctx->type];
   83 
   84 	sbitmap_clear_bit(&hctx->ctx_map, bit);
   85 }
   86 
   87 struct mq_inflight {
   88 	struct block_device *part;
   89 	unsigned int inflight[2];
   90 };
   91 
   92 static bool blk_mq_check_inflight(struct request *rq, void *priv)
   93 {
   94 	struct mq_inflight *mi = priv;
   95 
   96 	if (rq->part && blk_do_io_stat(rq) &&
   97 	    (!mi->part->bd_partno || rq->part == mi->part) &&
   98 	    blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
   99 		mi->inflight[rq_data_dir(rq)]++;
  100 
  101 	return true;
  102 }
  103 
  104 unsigned int blk_mq_in_flight(struct request_queue *q,
  105 		struct block_device *part)
  106 {
  107 	struct mq_inflight mi = { .part = part };
  108 
  109 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
  110 
  111 	return mi.inflight[0] + mi.inflight[1];
  112 }
  113 
  114 void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
  115 		unsigned int inflight[2])
  116 {
  117 	struct mq_inflight mi = { .part = part };
  118 
  119 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
  120 	inflight[0] = mi.inflight[0];
  121 	inflight[1] = mi.inflight[1];
  122 }
  123 
  124 void blk_freeze_queue_start(struct request_queue *q)
  125 {
  126 	mutex_lock(&q->mq_freeze_lock);
  127 	if (++q->mq_freeze_depth == 1) {
  128 		percpu_ref_kill(&q->q_usage_counter);
  129 		mutex_unlock(&q->mq_freeze_lock);
  130 		if (queue_is_mq(q))
  131 			blk_mq_run_hw_queues(q, false);
  132 	} else {
  133 		mutex_unlock(&q->mq_freeze_lock);
  134 	}
  135 }
  136 EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
  137 
  138 void blk_mq_freeze_queue_wait(struct request_queue *q)
  139 {
  140 	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
  141 }
  142 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
  143 
  144 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
  145 				     unsigned long timeout)
  146 {
  147 	return wait_event_timeout(q->mq_freeze_wq,
  148 					percpu_ref_is_zero(&q->q_usage_counter),
  149 					timeout);
  150 }
  151 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
  152 
  153 /*
  154  * Guarantee no request is in use, so we can change any data structure of
  155  * the queue afterward.
  156  */
  157 void blk_freeze_queue(struct request_queue *q)
  158 {
  159 	/*
  160 	 * In the !blk_mq case we are only calling this to kill the
  161 	 * q_usage_counter, otherwise this increases the freeze depth
  162 	 * and waits for it to return to zero.  For this reason there is
  163 	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
  164 	 * exported to drivers as the only user for unfreeze is blk_mq.
  165 	 */
  166 	blk_freeze_queue_start(q);
  167 	blk_mq_freeze_queue_wait(q);
  168 }
  169 
  170 void blk_mq_freeze_queue(struct request_queue *q)
  171 {
  172 	/*
  173 	 * ...just an alias to keep freeze and unfreeze actions balanced
  174 	 * in the blk_mq_* namespace
  175 	 */
  176 	blk_freeze_queue(q);
  177 }
  178 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
  179 
  180 void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
  181 {
  182 	mutex_lock(&q->mq_freeze_lock);
  183 	if (force_atomic)
  184 		q->q_usage_counter.data->force_atomic = true;
  185 	q->mq_freeze_depth--;
  186 	WARN_ON_ONCE(q->mq_freeze_depth < 0);
  187 	if (!q->mq_freeze_depth) {
  188 		percpu_ref_resurrect(&q->q_usage_counter);
  189 		wake_up_all(&q->mq_freeze_wq);
  190 	}
  191 	mutex_unlock(&q->mq_freeze_lock);
  192 }
  193 
  194 void blk_mq_unfreeze_queue(struct request_queue *q)
  195 {
  196 	__blk_mq_unfreeze_queue(q, false);
  197 }
  198 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
  199 
  200 /*
  201  * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
  202  * mpt3sas driver such that this function can be removed.
  203  */
  204 void blk_mq_quiesce_queue_nowait(struct request_queue *q)
  205 {
  206 	unsigned long flags;
  207 
  208 	spin_lock_irqsave(&q->queue_lock, flags);
  209 	if (!q->quiesce_depth++)
  210 		blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
  211 	spin_unlock_irqrestore(&q->queue_lock, flags);
  212 }
  213 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
  214 
  215 /**
  216  * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done
  217  * @set: tag_set to wait on
  218  *
  219  * Note: it is driver's responsibility for making sure that quiesce has
  220  * been started on or more of the request_queues of the tag_set.  This
  221  * function only waits for the quiesce on those request_queues that had
  222  * the quiesce flag set using blk_mq_quiesce_queue_nowait.
  223  */
  224 void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set)
  225 {
  226 	if (set->flags & BLK_MQ_F_BLOCKING)
  227 		synchronize_srcu(set->srcu);
  228 	else
  229 		synchronize_rcu();
  230 }
  231 EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done);
  232 
  233 /**
  234  * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
  235  * @q: request queue.
  236  *
  237  * Note: this function does not prevent that the struct request end_io()
  238  * callback function is invoked. Once this function is returned, we make
  239  * sure no dispatch can happen until the queue is unquiesced via
  240  * blk_mq_unquiesce_queue().
  241  */
  242 void blk_mq_quiesce_queue(struct request_queue *q)
  243 {
  244 	blk_mq_quiesce_queue_nowait(q);
  245 	/* nothing to wait for non-mq queues */
  246 	if (queue_is_mq(q))
  247 		blk_mq_wait_quiesce_done(q->tag_set);
  248 }
  249 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
  250 
  251 /*
  252  * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
  253  * @q: request queue.
  254  *
  255  * This function recovers queue into the state before quiescing
  256  * which is done by blk_mq_quiesce_queue.
  257  */
  258 void blk_mq_unquiesce_queue(struct request_queue *q)
  259 {
  260 	unsigned long flags;
  261 	bool run_queue = false;
  262 
  263 	spin_lock_irqsave(&q->queue_lock, flags);
  264 	if (WARN_ON_ONCE(q->quiesce_depth <= 0)) {
  265 		;
  266 	} else if (!--q->quiesce_depth) {
  267 		blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
  268 		run_queue = true;
  269 	}
  270 	spin_unlock_irqrestore(&q->queue_lock, flags);
  271 
  272 	/* dispatch requests which are inserted during quiescing */
  273 	if (run_queue)
  274 		blk_mq_run_hw_queues(q, true);
  275 }
  276 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
  277 
  278 void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
  279 {
  280 	struct request_queue *q;
  281 
  282 	mutex_lock(&set->tag_list_lock);
  283 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
  284 		if (!blk_queue_skip_tagset_quiesce(q))
  285 			blk_mq_quiesce_queue_nowait(q);
  286 	}
  287 	blk_mq_wait_quiesce_done(set);
  288 	mutex_unlock(&set->tag_list_lock);
  289 }
  290 EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
  291 
  292 void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
  293 {
  294 	struct request_queue *q;
  295 
  296 	mutex_lock(&set->tag_list_lock);
  297 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
  298 		if (!blk_queue_skip_tagset_quiesce(q))
  299 			blk_mq_unquiesce_queue(q);
  300 	}
  301 	mutex_unlock(&set->tag_list_lock);
  302 }
  303 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
  304 
  305 void blk_mq_wake_waiters(struct request_queue *q)
  306 {
  307 	struct blk_mq_hw_ctx *hctx;
  308 	unsigned long i;
  309 
  310 	queue_for_each_hw_ctx(q, hctx, i)
  311 		if (blk_mq_hw_queue_mapped(hctx))
  312 			blk_mq_tag_wakeup_all(hctx->tags, true);
  313 }
  314 
  315 void blk_rq_init(struct request_queue *q, struct request *rq)
  316 {
  317 	memset(rq, 0, sizeof(*rq));
  318 
  319 	INIT_LIST_HEAD(&rq->queuelist);
  320 	rq->q = q;
  321 	rq->__sector = (sector_t) -1;
  322 	INIT_HLIST_NODE(&rq->hash);
  323 	RB_CLEAR_NODE(&rq->rb_node);
  324 	rq->tag = BLK_MQ_NO_TAG;
  325 	rq->internal_tag = BLK_MQ_NO_TAG;
  326 	rq->start_time_ns = ktime_get_ns();
  327 	rq->part = NULL;
  328 	blk_crypto_rq_set_defaults(rq);
  329 }
  330 EXPORT_SYMBOL(blk_rq_init);
  331 
  332 /* Set start and alloc time when the allocated request is actually used */
  333 static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
  334 {
  335 	if (blk_mq_need_time_stamp(rq))
  336 		rq->start_time_ns = ktime_get_ns();
  337 	else
  338 		rq->start_time_ns = 0;
  339 
  340 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
  341 	if (blk_queue_rq_alloc_time(rq->q))
  342 		rq->alloc_time_ns = alloc_time_ns ?: rq->start_time_ns;
  343 	else
  344 		rq->alloc_time_ns = 0;
  345 #endif
  346 }
  347 
  348 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
  349 		struct blk_mq_tags *tags, unsigned int tag)
  350 {
  351 	struct blk_mq_ctx *ctx = data->ctx;
  352 	struct blk_mq_hw_ctx *hctx = data->hctx;
  353 	struct request_queue *q = data->q;
  354 	struct request *rq = tags->static_rqs[tag];
  355 
  356 	rq->q = q;
  357 	rq->mq_ctx = ctx;
  358 	rq->mq_hctx = hctx;
  359 	rq->cmd_flags = data->cmd_flags;
  360 
  361 	if (data->flags & BLK_MQ_REQ_PM)
  362 		data->rq_flags |= RQF_PM;
  363 	if (blk_queue_io_stat(q))
  364 		data->rq_flags |= RQF_IO_STAT;
  365 	rq->rq_flags = data->rq_flags;
  366 
  367 	if (data->rq_flags & RQF_SCHED_TAGS) {
  368 		rq->tag = BLK_MQ_NO_TAG;
  369 		rq->internal_tag = tag;
  370 	} else {
  371 		rq->tag = tag;
  372 		rq->internal_tag = BLK_MQ_NO_TAG;
  373 	}
  374 	rq->timeout = 0;
  375 
  376 	rq->part = NULL;
  377 	rq->io_start_time_ns = 0;
  378 	rq->stats_sectors = 0;
  379 	rq->nr_phys_segments = 0;
  380 #if defined(CONFIG_BLK_DEV_INTEGRITY)
  381 	rq->nr_integrity_segments = 0;
  382 #endif
  383 	rq->end_io = NULL;
  384 	rq->end_io_data = NULL;
  385 
  386 	blk_crypto_rq_set_defaults(rq);
  387 	INIT_LIST_HEAD(&rq->queuelist);
  388 	/* tag was already set */
  389 	WRITE_ONCE(rq->deadline, 0);
  390 	req_ref_set(rq, 1);
  391 
  392 	if (rq->rq_flags & RQF_USE_SCHED) {
  393 		struct elevator_queue *e = data->q->elevator;
  394 
  395 		INIT_HLIST_NODE(&rq->hash);
  396 		RB_CLEAR_NODE(&rq->rb_node);
  397 
  398 		if (e->type->ops.prepare_request)
  399 			e->type->ops.prepare_request(rq);
  400 	}
  401 
  402 	return rq;
  403 }
  404 
  405 static inline struct request *
  406 __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
  407 {
  408 	unsigned int tag, tag_offset;
  409 	struct blk_mq_tags *tags;
  410 	struct request *rq;
  411 	unsigned long tag_mask;
  412 	int i, nr = 0;
  413 
  414 	tag_mask = blk_mq_get_tags(data, data->nr_tags, &tag_offset);
  415 	if (unlikely(!tag_mask))
  416 		return NULL;
  417 
  418 	tags = blk_mq_tags_from_data(data);
  419 	for (i = 0; tag_mask; i++) {
  420 		if (!(tag_mask & (1UL << i)))
  421 			continue;
  422 		tag = tag_offset + i;
  423 		prefetch(tags->static_rqs[tag]);
  424 		tag_mask &= ~(1UL << i);
  425 		rq = blk_mq_rq_ctx_init(data, tags, tag);
  426 		rq_list_add(data->cached_rq, rq);
  427 		nr++;
  428 	}
  429 	/* caller already holds a reference, add for remainder */
  430 	percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
  431 	data->nr_tags -= nr;
  432 
  433 	return rq_list_pop(data->cached_rq);
  434 }
  435 
  436 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
  437 {
  438 	struct request_queue *q = data->q;
  439 	u64 alloc_time_ns = 0;
  440 	struct request *rq;
  441 	unsigned int tag;
  442 
  443 	/* alloc_time includes depth and tag waits */
  444 	if (blk_queue_rq_alloc_time(q))
  445 		alloc_time_ns = ktime_get_ns();
  446 
  447 	if (data->cmd_flags & REQ_NOWAIT)
  448 		data->flags |= BLK_MQ_REQ_NOWAIT;
  449 
  450 	if (q->elevator) {
  451 		/*
  452 		 * All requests use scheduler tags when an I/O scheduler is
  453 		 * enabled for the queue.
  454 		 */
  455 		data->rq_flags |= RQF_SCHED_TAGS;
  456 
  457 		/*
  458 		 * Flush/passthrough requests are special and go directly to the
  459 		 * dispatch list.
  460 		 */
  461 		if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH &&
  462 		    !blk_op_is_passthrough(data->cmd_flags)) {
  463 			struct elevator_mq_ops *ops = &q->elevator->type->ops;
  464 
  465 			WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
  466 
  467 			data->rq_flags |= RQF_USE_SCHED;
  468 			if (ops->limit_depth)
  469 				ops->limit_depth(data->cmd_flags, data);
  470 		}
  471 	}
  472 
  473 retry:
  474 	data->ctx = blk_mq_get_ctx(q);
  475 	data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
  476 	if (!(data->rq_flags & RQF_SCHED_TAGS))
  477 		blk_mq_tag_busy(data->hctx);
  478 
  479 	if (data->flags & BLK_MQ_REQ_RESERVED)
  480 		data->rq_flags |= RQF_RESV;
  481 
  482 	/*
  483 	 * Try batched alloc if we want more than 1 tag.
  484 	 */
  485 	if (data->nr_tags > 1) {
  486 		rq = __blk_mq_alloc_requests_batch(data);
  487 		if (rq) {
  488 			blk_mq_rq_time_init(rq, alloc_time_ns);
  489 			return rq;
  490 		}
  491 		data->nr_tags = 1;
  492 	}
  493 
  494 	/*
  495 	 * Waiting allocations only fail because of an inactive hctx.  In that
  496 	 * case just retry the hctx assignment and tag allocation as CPU hotplug
  497 	 * should have migrated us to an online CPU by now.
  498 	 */
  499 	tag = blk_mq_get_tag(data);
  500 	if (tag == BLK_MQ_NO_TAG) {
  501 		if (data->flags & BLK_MQ_REQ_NOWAIT)
  502 			return NULL;
  503 		/*
  504 		 * Give up the CPU and sleep for a random short time to
  505 		 * ensure that thread using a realtime scheduling class
  506 		 * are migrated off the CPU, and thus off the hctx that
  507 		 * is going away.
  508 		 */
  509 		msleep(3);
  510 		goto retry;
  511 	}
  512 
  513 	rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
  514 	blk_mq_rq_time_init(rq, alloc_time_ns);
  515 	return rq;
  516 }
  517 
  518 static struct request *blk_mq_rq_cache_fill(struct request_queue *q,
  519 					    struct blk_plug *plug,
  520 					    blk_opf_t opf,
  521 					    blk_mq_req_flags_t flags)
  522 {
  523 	struct blk_mq_alloc_data data = {
  524 		.q		= q,
  525 		.flags		= flags,
  526 		.cmd_flags	= opf,
  527 		.nr_tags	= plug->nr_ios,
  528 		.cached_rq	= &plug->cached_rq,
  529 	};
  530 	struct request *rq;
  531 
  532 	if (blk_queue_enter(q, flags))
  533 		return NULL;
  534 
  535 	plug->nr_ios = 1;
  536 
  537 	rq = __blk_mq_alloc_requests(&data);
  538 	if (unlikely(!rq))
  539 		blk_queue_exit(q);
  540 	return rq;
  541 }
  542 
  543 static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
  544 						   blk_opf_t opf,
  545 						   blk_mq_req_flags_t flags)
  546 {
  547 	struct blk_plug *plug = current->plug;
  548 	struct request *rq;
  549 
  550 	if (!plug)
  551 		return NULL;
  552 
  553 	if (rq_list_empty(plug->cached_rq)) {
  554 		if (plug->nr_ios == 1)
  555 			return NULL;
  556 		rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
  557 		if (!rq)
  558 			return NULL;
  559 	} else {
  560 		rq = rq_list_peek(&plug->cached_rq);
  561 		if (!rq || rq->q != q)
  562 			return NULL;
  563 
  564 		if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
  565 			return NULL;
  566 		if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
  567 			return NULL;
  568 
  569 		plug->cached_rq = rq_list_next(rq);
  570 		blk_mq_rq_time_init(rq, 0);
  571 	}
  572 
  573 	rq->cmd_flags = opf;
  574 	INIT_LIST_HEAD(&rq->queuelist);
  575 	return rq;
  576 }
  577 
  578 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
  579 		blk_mq_req_flags_t flags)
  580 {
  581 	struct request *rq;
  582 
  583 	rq = blk_mq_alloc_cached_request(q, opf, flags);
  584 	if (!rq) {
  585 		struct blk_mq_alloc_data data = {
  586 			.q		= q,
  587 			.flags		= flags,
  588 			.cmd_flags	= opf,
  589 			.nr_tags	= 1,
  590 		};
  591 		int ret;
  592 
  593 		ret = blk_queue_enter(q, flags);
  594 		if (ret)
  595 			return ERR_PTR(ret);
  596 
  597 		rq = __blk_mq_alloc_requests(&data);
  598 		if (!rq)
  599 			goto out_queue_exit;
  600 	}
  601 	rq->__data_len = 0;
  602 	rq->__sector = (sector_t) -1;
  603 	rq->bio = rq->biotail = NULL;
  604 	return rq;
  605 out_queue_exit:
  606 	blk_queue_exit(q);
  607 	return ERR_PTR(-EWOULDBLOCK);
  608 }
  609 EXPORT_SYMBOL(blk_mq_alloc_request);
  610 
  611 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
  612 	blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
  613 {
  614 	struct blk_mq_alloc_data data = {
  615 		.q		= q,
  616 		.flags		= flags,
  617 		.cmd_flags	= opf,
  618 		.nr_tags	= 1,
  619 	};
  620 	u64 alloc_time_ns = 0;
  621 	struct request *rq;
  622 	unsigned int cpu;
  623 	unsigned int tag;
  624 	int ret;
  625 
  626 	/* alloc_time includes depth and tag waits */
  627 	if (blk_queue_rq_alloc_time(q))
  628 		alloc_time_ns = ktime_get_ns();
  629 
  630 	/*
  631 	 * If the tag allocator sleeps we could get an allocation for a
  632 	 * different hardware context.  No need to complicate the low level
  633 	 * allocator for this for the rare use case of a command tied to
  634 	 * a specific queue.
  635 	 */
  636 	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) ||
  637 	    WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED)))
  638 		return ERR_PTR(-EINVAL);
  639 
  640 	if (hctx_idx >= q->nr_hw_queues)
  641 		return ERR_PTR(-EIO);
  642 
  643 	ret = blk_queue_enter(q, flags);
  644 	if (ret)
  645 		return ERR_PTR(ret);
  646 
  647 	/*
  648 	 * Check if the hardware context is actually mapped to anything.
  649 	 * If not tell the caller that it should skip this queue.
  650 	 */
  651 	ret = -EXDEV;
  652 	data.hctx = xa_load(&q->hctx_table, hctx_idx);
  653 	if (!blk_mq_hw_queue_mapped(data.hctx))
  654 		goto out_queue_exit;
  655 	cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
  656 	if (cpu >= nr_cpu_ids)
  657 		goto out_queue_exit;
  658 	data.ctx = __blk_mq_get_ctx(q, cpu);
  659 
  660 	if (q->elevator)
  661 		data.rq_flags |= RQF_SCHED_TAGS;
  662 	else
  663 		blk_mq_tag_busy(data.hctx);
  664 
  665 	if (flags & BLK_MQ_REQ_RESERVED)
  666 		data.rq_flags |= RQF_RESV;
  667 
  668 	ret = -EWOULDBLOCK;
  669 	tag = blk_mq_get_tag(&data);
  670 	if (tag == BLK_MQ_NO_TAG)
  671 		goto out_queue_exit;
  672 	rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
  673 	blk_mq_rq_time_init(rq, alloc_time_ns);
  674 	rq->__data_len = 0;
  675 	rq->__sector = (sector_t) -1;
  676 	rq->bio = rq->biotail = NULL;
  677 	return rq;
  678 
  679 out_queue_exit:
  680 	blk_queue_exit(q);
  681 	return ERR_PTR(ret);
  682 }
  683 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
  684 
  685 static void blk_mq_finish_request(struct request *rq)
  686 {
  687 	struct request_queue *q = rq->q;
  688 
  689 	if (rq->rq_flags & RQF_USE_SCHED) {
  690 		q->elevator->type->ops.finish_request(rq);
  691 		/*
  692 		 * For postflush request that may need to be
  693 		 * completed twice, we should clear this flag
  694 		 * to avoid double finish_request() on the rq.
  695 		 */
  696 		rq->rq_flags &= ~RQF_USE_SCHED;
  697 	}
  698 }
  699 
  700 static void __blk_mq_free_request(struct request *rq)
  701 {
  702 	struct request_queue *q = rq->q;
  703 	struct blk_mq_ctx *ctx = rq->mq_ctx;
  704 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
  705 	const int sched_tag = rq->internal_tag;
  706 
  707 	blk_crypto_free_request(rq);
  708 	blk_pm_mark_last_busy(rq);
  709 	rq->mq_hctx = NULL;
  710 
  711 	if (rq->rq_flags & RQF_MQ_INFLIGHT)
  712 		__blk_mq_dec_active_requests(hctx);
  713 
  714 	if (rq->tag != BLK_MQ_NO_TAG)
  715 		blk_mq_put_tag(hctx->tags, ctx, rq->tag);
  716 	if (sched_tag != BLK_MQ_NO_TAG)
  717 		blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
  718 	blk_mq_sched_restart(hctx);
  719 	blk_queue_exit(q);
  720 }
  721 
  722 void blk_mq_free_request(struct request *rq)
  723 {
  724 	struct request_queue *q = rq->q;
  725 
  726 	blk_mq_finish_request(rq);
  727 
  728 	if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
  729 		laptop_io_completion(q->disk->bdi);
  730 
  731 	rq_qos_done(q, rq);
  732 
  733 	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
  734 	if (req_ref_put_and_test(rq))
  735 		__blk_mq_free_request(rq);
  736 }
  737 EXPORT_SYMBOL_GPL(blk_mq_free_request);
  738 
  739 void blk_mq_free_plug_rqs(struct blk_plug *plug)
  740 {
  741 	struct request *rq;
  742 
  743 	while ((rq = rq_list_pop(&plug->cached_rq)) != NULL)
  744 		blk_mq_free_request(rq);
  745 }
  746 
  747 void blk_dump_rq_flags(struct request *rq, char *msg)
  748 {
  749 	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
  750 		rq->q->disk ? rq->q->disk->disk_name : "?",
  751 		(__force unsigned long long) rq->cmd_flags);
  752 
  753 	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
  754 	       (unsigned long long)blk_rq_pos(rq),
  755 	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
  756 	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
  757 	       rq->bio, rq->biotail, blk_rq_bytes(rq));
  758 }
  759 EXPORT_SYMBOL(blk_dump_rq_flags);
  760 
  761 static void req_bio_endio(struct request *rq, struct bio *bio,
  762 			  unsigned int nbytes, blk_status_t error)
  763 {
  764 	if (unlikely(error)) {
  765 		bio->bi_status = error;
  766 	} else if (req_op(rq) == REQ_OP_ZONE_APPEND) {
  767 		/*
  768 		 * Partial zone append completions cannot be supported as the
  769 		 * BIO fragments may end up not being written sequentially.
  770 		 */
  771 		if (bio->bi_iter.bi_size != nbytes)
  772 			bio->bi_status = BLK_STS_IOERR;
  773 		else
  774 			bio->bi_iter.bi_sector = rq->__sector;
  775 	}
  776 
  777 	bio_advance(bio, nbytes);
  778 
  779 	if (unlikely(rq->rq_flags & RQF_QUIET))
  780 		bio_set_flag(bio, BIO_QUIET);
  781 	/* don't actually finish bio if it's part of flush sequence */
  782 	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
  783 		bio_endio(bio);
  784 }
  785 
  786 static void blk_account_io_completion(struct request *req, unsigned int bytes)
  787 {
  788 	if (req->part && blk_do_io_stat(req)) {
  789 		const int sgrp = op_stat_group(req_op(req));
  790 
  791 		part_stat_lock();
  792 		part_stat_add(req->part, sectors[sgrp], bytes >> 9);
  793 		part_stat_unlock();
  794 	}
  795 }
  796 
  797 static void blk_print_req_error(struct request *req, blk_status_t status)
  798 {
  799 	printk_ratelimited(KERN_ERR
  800 		"%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
  801 		"phys_seg %u prio class %u\n",
  802 		blk_status_to_str(status),
  803 		req->q->disk ? req->q->disk->disk_name : "?",
  804 		blk_rq_pos(req), (__force u32)req_op(req),
  805 		blk_op_str(req_op(req)),
  806 		(__force u32)(req->cmd_flags & ~REQ_OP_MASK),
  807 		req->nr_phys_segments,
  808 		IOPRIO_PRIO_CLASS(req->ioprio));
  809 }
  810 
  811 /*
  812  * Fully end IO on a request. Does not support partial completions, or
  813  * errors.
  814  */
  815 static void blk_complete_request(struct request *req)
  816 {
  817 	const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0;
  818 	int total_bytes = blk_rq_bytes(req);
  819 	struct bio *bio = req->bio;
  820 
  821 	trace_block_rq_complete(req, BLK_STS_OK, total_bytes);
  822 
  823 	if (!bio)
  824 		return;
  825 
  826 #ifdef CONFIG_BLK_DEV_INTEGRITY
  827 	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ)
  828 		req->q->integrity.profile->complete_fn(req, total_bytes);
  829 #endif
  830 
  831 	/*
  832 	 * Upper layers may call blk_crypto_evict_key() anytime after the last
  833 	 * bio_endio().  Therefore, the keyslot must be released before that.
  834 	 */
  835 	blk_crypto_rq_put_keyslot(req);
  836 
  837 	blk_account_io_completion(req, total_bytes);
  838 
  839 	do {
  840 		struct bio *next = bio->bi_next;
  841 
  842 		/* Completion has already been traced */
  843 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
  844 
  845 		if (req_op(req) == REQ_OP_ZONE_APPEND)
  846 			bio->bi_iter.bi_sector = req->__sector;
  847 
  848 		if (!is_flush)
  849 			bio_endio(bio);
  850 		bio = next;
  851 	} while (bio);
  852 
  853 	/*
  854 	 * Reset counters so that the request stacking driver
  855 	 * can find how many bytes remain in the request
  856 	 * later.
  857 	 */
  858 	if (!req->end_io) {
  859 		req->bio = NULL;
  860 		req->__data_len = 0;
  861 	}
  862 }
  863 
  864 /**
  865  * blk_update_request - Complete multiple bytes without completing the request
  866  * @req:      the request being processed
  867  * @error:    block status code
  868  * @nr_bytes: number of bytes to complete for @req
  869  *
  870  * Description:
  871  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
  872  *     the request structure even if @req doesn't have leftover.
  873  *     If @req has leftover, sets it up for the next range of segments.
  874  *
  875  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
  876  *     %false return from this function.
  877  *
  878  * Note:
  879  *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
  880  *      except in the consistency check at the end of this function.
  881  *
  882  * Return:
  883  *     %false - this request doesn't have any more data
  884  *     %true  - this request has more data
  885  **/
  886 bool blk_update_request(struct request *req, blk_status_t error,
  887 		unsigned int nr_bytes)
  888 {
  889 	int total_bytes;
  890 
  891 	trace_block_rq_complete(req, error, nr_bytes);
  892 
  893 	if (!req->bio)
  894 		return false;
  895 
  896 #ifdef CONFIG_BLK_DEV_INTEGRITY
  897 	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
  898 	    error == BLK_STS_OK)
  899 		req->q->integrity.profile->complete_fn(req, nr_bytes);
  900 #endif
  901 
  902 	/*
  903 	 * Upper layers may call blk_crypto_evict_key() anytime after the last
  904 	 * bio_endio().  Therefore, the keyslot must be released before that.
  905 	 */
  906 	if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))
  907 		__blk_crypto_rq_put_keyslot(req);
  908 
  909 	if (unlikely(error && !blk_rq_is_passthrough(req) &&
  910 		     !(req->rq_flags & RQF_QUIET)) &&
  911 		     !test_bit(GD_DEAD, &req->q->disk->state)) {
  912 		blk_print_req_error(req, error);
  913 		trace_block_rq_error(req, error, nr_bytes);
  914 	}
  915 
  916 	blk_account_io_completion(req, nr_bytes);
  917 
  918 	total_bytes = 0;
  919 	while (req->bio) {
  920 		struct bio *bio = req->bio;
  921 		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
  922 
  923 		if (bio_bytes == bio->bi_iter.bi_size)
  924 			req->bio = bio->bi_next;
  925 
  926 		/* Completion has already been traced */
  927 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
  928 		req_bio_endio(req, bio, bio_bytes, error);
  929 
  930 		total_bytes += bio_bytes;
  931 		nr_bytes -= bio_bytes;
  932 
  933 		if (!nr_bytes)
  934 			break;
  935 	}
  936 
  937 	/*
  938 	 * completely done
  939 	 */
  940 	if (!req->bio) {
  941 		/*
  942 		 * Reset counters so that the request stacking driver
  943 		 * can find how many bytes remain in the request
  944 		 * later.
  945 		 */
  946 		req->__data_len = 0;
  947 		return false;
  948 	}
  949 
  950 	req->__data_len -= total_bytes;
  951 
  952 	/* update sector only for requests with clear definition of sector */
  953 	if (!blk_rq_is_passthrough(req))
  954 		req->__sector += total_bytes >> 9;
  955 
  956 	/* mixed attributes always follow the first bio */
  957 	if (req->rq_flags & RQF_MIXED_MERGE) {
  958 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
  959 		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
  960 	}
  961 
  962 	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
  963 		/*
  964 		 * If total number of sectors is less than the first segment
  965 		 * size, something has gone terribly wrong.
  966 		 */
  967 		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
  968 			blk_dump_rq_flags(req, "request botched");
  969 			req->__data_len = blk_rq_cur_bytes(req);
  970 		}
  971 
  972 		/* recalculate the number of segments */
  973 		req->nr_phys_segments = blk_recalc_rq_segments(req);
  974 	}
  975 
  976 	return true;
  977 }
  978 EXPORT_SYMBOL_GPL(blk_update_request);
  979 
  980 static inline void blk_account_io_done(struct request *req, u64 now)
  981 {
  982 	trace_block_io_done(req);
  983 
  984 	/*
  985 	 * Account IO completion.  flush_rq isn't accounted as a
  986 	 * normal IO on queueing nor completion.  Accounting the
  987 	 * containing request is enough.
  988 	 */
  989 	if (blk_do_io_stat(req) && req->part &&
  990 	    !(req->rq_flags & RQF_FLUSH_SEQ)) {
  991 		const int sgrp = op_stat_group(req_op(req));
  992 
  993 		part_stat_lock();
  994 		update_io_ticks(req->part, jiffies, true);
  995 		part_stat_inc(req->part, ios[sgrp]);
  996 		part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
  997 		part_stat_unlock();
  998 	}
  999 }
 1000 
 1001 static inline void blk_account_io_start(struct request *req)
 1002 {
 1003 	trace_block_io_start(req);
 1004 
 1005 	if (blk_do_io_stat(req)) {
 1006 		/*
 1007 		 * All non-passthrough requests are created from a bio with one
 1008 		 * exception: when a flush command that is part of a flush sequence
 1009 		 * generated by the state machine in blk-flush.c is cloned onto the
 1010 		 * lower device by dm-multipath we can get here without a bio.
 1011 		 */
 1012 		if (req->bio)
 1013 			req->part = req->bio->bi_bdev;
 1014 		else
 1015 			req->part = req->q->disk->part0;
 1016 
 1017 		part_stat_lock();
 1018 		update_io_ticks(req->part, jiffies, false);
 1019 		part_stat_unlock();
 1020 	}
 1021 }
 1022 
 1023 static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
 1024 {
 1025 	if (rq->rq_flags & RQF_STATS)
 1026 		blk_stat_add(rq, now);
 1027 
 1028 	blk_mq_sched_completed_request(rq, now);
 1029 	blk_account_io_done(rq, now);
 1030 }
 1031 
 1032 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
 1033 {
 1034 	if (blk_mq_need_time_stamp(rq))
 1035 		__blk_mq_end_request_acct(rq, ktime_get_ns());
 1036 
 1037 	blk_mq_finish_request(rq);
 1038 
 1039 	if (rq->end_io) {
 1040 		rq_qos_done(rq->q, rq);
 1041 		if (rq->end_io(rq, error) == RQ_END_IO_FREE)
 1042 			blk_mq_free_request(rq);
 1043 	} else {
 1044 		blk_mq_free_request(rq);
 1045 	}
 1046 }
 1047 EXPORT_SYMBOL(__blk_mq_end_request);
 1048 
 1049 void blk_mq_end_request(struct request *rq, blk_status_t error)
 1050 {
 1051 	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 1052 		BUG();
 1053 	__blk_mq_end_request(rq, error);
 1054 }
 1055 EXPORT_SYMBOL(blk_mq_end_request);
 1056 
 1057 #define TAG_COMP_BATCH		32
 1058 
 1059 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
 1060 					  int *tag_array, int nr_tags)
 1061 {
 1062 	struct request_queue *q = hctx->queue;
 1063 
 1064 	/*
 1065 	 * All requests should have been marked as RQF_MQ_INFLIGHT, so
 1066 	 * update hctx->nr_active in batch
 1067 	 */
 1068 	if (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
 1069 		__blk_mq_sub_active_requests(hctx, nr_tags);
 1070 
 1071 	blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
 1072 	percpu_ref_put_many(&q->q_usage_counter, nr_tags);
 1073 }
 1074 
 1075 void blk_mq_end_request_batch(struct io_comp_batch *iob)
 1076 {
 1077 	int tags[TAG_COMP_BATCH], nr_tags = 0;
 1078 	struct blk_mq_hw_ctx *cur_hctx = NULL;
 1079 	struct request *rq;
 1080 	u64 now = 0;
 1081 
 1082 	if (iob->need_ts)
 1083 		now = ktime_get_ns();
 1084 
 1085 	while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
 1086 		prefetch(rq->bio);
 1087 		prefetch(rq->rq_next);
 1088 
 1089 		blk_complete_request(rq);
 1090 		if (iob->need_ts)
 1091 			__blk_mq_end_request_acct(rq, now);
 1092 
 1093 		blk_mq_finish_request(rq);
 1094 
 1095 		rq_qos_done(rq->q, rq);
 1096 
 1097 		/*
 1098 		 * If end_io handler returns NONE, then it still has
 1099 		 * ownership of the request.
 1100 		 */
 1101 		if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE)
 1102 			continue;
 1103 
 1104 		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
 1105 		if (!req_ref_put_and_test(rq))
 1106 			continue;
 1107 
 1108 		blk_crypto_free_request(rq);
 1109 		blk_pm_mark_last_busy(rq);
 1110 
 1111 		if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) {
 1112 			if (cur_hctx)
 1113 				blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
 1114 			nr_tags = 0;
 1115 			cur_hctx = rq->mq_hctx;
 1116 		}
 1117 		tags[nr_tags++] = rq->tag;
 1118 	}
 1119 
 1120 	if (nr_tags)
 1121 		blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
 1122 }
 1123 EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
 1124 
 1125 static void blk_complete_reqs(struct llist_head *list)
 1126 {
 1127 	struct llist_node *entry = llist_reverse_order(llist_del_all(list));
 1128 	struct request *rq, *next;
 1129 
 1130 	llist_for_each_entry_safe(rq, next, entry, ipi_list)
 1131 		rq->q->mq_ops->complete(rq);
 1132 }
 1133 
 1134 static __latent_entropy void blk_done_softirq(struct softirq_action *h)
 1135 {
 1136 	blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
 1137 }
 1138 
 1139 static int blk_softirq_cpu_dead(unsigned int cpu)
 1140 {
 1141 	blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
 1142 	return 0;
 1143 }
 1144 
 1145 static void __blk_mq_complete_request_remote(void *data)
 1146 {
 1147 	__raise_softirq_irqoff(BLOCK_SOFTIRQ);
 1148 }
 1149 
 1150 static inline bool blk_mq_complete_need_ipi(struct request *rq)
 1151 {
 1152 	int cpu = raw_smp_processor_id();
 1153 
 1154 	if (!IS_ENABLED(CONFIG_SMP) ||
 1155 	    !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
 1156 		return false;
 1157 	/*
 1158 	 * With force threaded interrupts enabled, raising softirq from an SMP
 1159 	 * function call will always result in waking the ksoftirqd thread.
 1160 	 * This is probably worse than completing the request on a different
 1161 	 * cache domain.
 1162 	 */
 1163 	if (force_irqthreads())
 1164 		return false;
 1165 
 1166 	/* same CPU or cache domain?  Complete locally */
 1167 	if (cpu == rq->mq_ctx->cpu ||
 1168 	    (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
 1169 	     cpus_share_cache(cpu, rq->mq_ctx->cpu)))
 1170 		return false;
 1171 
 1172 	/* don't try to IPI to an offline CPU */
 1173 	return cpu_online(rq->mq_ctx->cpu);
 1174 }
 1175 
 1176 static void blk_mq_complete_send_ipi(struct request *rq)
 1177 {
 1178 	unsigned int cpu;
 1179 
 1180 	cpu = rq->mq_ctx->cpu;
 1181 	if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu)))
 1182 		smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu));
 1183 }
 1184 
 1185 static void blk_mq_raise_softirq(struct request *rq)
 1186 {
 1187 	struct llist_head *list;
 1188 
 1189 	preempt_disable();
 1190 	list = this_cpu_ptr(&blk_cpu_done);
 1191 	if (llist_add(&rq->ipi_list, list))
 1192 		raise_softirq(BLOCK_SOFTIRQ);
 1193 	preempt_enable();
 1194 }
 1195 
 1196 bool blk_mq_complete_request_remote(struct request *rq)
 1197 {
 1198 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
 1199 
 1200 	/*
 1201 	 * For request which hctx has only one ctx mapping,
 1202 	 * or a polled request, always complete locally,
 1203 	 * it's pointless to redirect the completion.
 1204 	 */
 1205 	if ((rq->mq_hctx->nr_ctx == 1 &&
 1206 	     rq->mq_ctx->cpu == raw_smp_processor_id()) ||
 1207 	     rq->cmd_flags & REQ_POLLED)
 1208 		return false;
 1209 
 1210 	if (blk_mq_complete_need_ipi(rq)) {
 1211 		blk_mq_complete_send_ipi(rq);
 1212 		return true;
 1213 	}
 1214 
 1215 	if (rq->q->nr_hw_queues == 1) {
 1216 		blk_mq_raise_softirq(rq);
 1217 		return true;
 1218 	}
 1219 	return false;
 1220 }
 1221 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
 1222 
 1223 /**
 1224  * blk_mq_complete_request - end I/O on a request
 1225  * @rq:		the request being processed
 1226  *
 1227  * Description:
 1228  *	Complete a request by scheduling the ->complete_rq operation.
 1229  **/
 1230 void blk_mq_complete_request(struct request *rq)
 1231 {
 1232 	if (!blk_mq_complete_request_remote(rq))
 1233 		rq->q->mq_ops->complete(rq);
 1234 }
 1235 EXPORT_SYMBOL(blk_mq_complete_request);
 1236 
 1237 /**
 1238  * blk_mq_start_request - Start processing a request
 1239  * @rq: Pointer to request to be started
 1240  *
 1241  * Function used by device drivers to notify the block layer that a request
 1242  * is going to be processed now, so blk layer can do proper initializations
 1243  * such as starting the timeout timer.
 1244  */
 1245 void blk_mq_start_request(struct request *rq)
 1246 {
 1247 	struct request_queue *q = rq->q;
 1248 
 1249 	trace_block_rq_issue(rq);
 1250 
 1251 	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
 1252 		rq->io_start_time_ns = ktime_get_ns();
 1253 		rq->stats_sectors = blk_rq_sectors(rq);
 1254 		rq->rq_flags |= RQF_STATS;
 1255 		rq_qos_issue(q, rq);
 1256 	}
 1257 
 1258 	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
 1259 
 1260 	blk_add_timer(rq);
 1261 	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
 1262 
 1263 #ifdef CONFIG_BLK_DEV_INTEGRITY
 1264 	if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
 1265 		q->integrity.profile->prepare_fn(rq);
 1266 #endif
 1267 	if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
 1268 	        WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num);
 1269 }
 1270 EXPORT_SYMBOL(blk_mq_start_request);
 1271 
 1272 /*
 1273  * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
 1274  * queues. This is important for md arrays to benefit from merging
 1275  * requests.
 1276  */
 1277 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
 1278 {
 1279 	if (plug->multiple_queues)
 1280 		return BLK_MAX_REQUEST_COUNT * 2;
 1281 	return BLK_MAX_REQUEST_COUNT;
 1282 }
 1283 
 1284 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
 1285 {
 1286 	struct request *last = rq_list_peek(&plug->mq_list);
 1287 
 1288 	if (!plug->rq_count) {
 1289 		trace_block_plug(rq->q);
 1290 	} else if (plug->rq_count >= blk_plug_max_rq_count(plug) ||
 1291 		   (!blk_queue_nomerges(rq->q) &&
 1292 		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
 1293 		blk_mq_flush_plug_list(plug, false);
 1294 		last = NULL;
 1295 		trace_block_plug(rq->q);
 1296 	}
 1297 
 1298 	if (!plug->multiple_queues && last && last->q != rq->q)
 1299 		plug->multiple_queues = true;
 1300 	/*
 1301 	 * Any request allocated from sched tags can't be issued to
 1302 	 * ->queue_rqs() directly
 1303 	 */
 1304 	if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
 1305 		plug->has_elevator = true;
 1306 	rq->rq_next = NULL;
 1307 	rq_list_add(&plug->mq_list, rq);
 1308 	plug->rq_count++;
 1309 }
 1310 
 1311 /**
 1312  * blk_execute_rq_nowait - insert a request to I/O scheduler for execution
 1313  * @rq:		request to insert
 1314  * @at_head:    insert request at head or tail of queue
 1315  *
 1316  * Description:
 1317  *    Insert a fully prepared request at the back of the I/O scheduler queue
 1318  *    for execution.  Don't wait for completion.
 1319  *
 1320  * Note:
 1321  *    This function will invoke @done directly if the queue is dead.
 1322  */
 1323 void blk_execute_rq_nowait(struct request *rq, bool at_head)
 1324 {
 1325 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 1326 
 1327 	WARN_ON(irqs_disabled());
 1328 	WARN_ON(!blk_rq_is_passthrough(rq));
 1329 
 1330 	blk_account_io_start(rq);
 1331 
 1332 	/*
 1333 	 * As plugging can be enabled for passthrough requests on a zoned
 1334 	 * device, directly accessing the plug instead of using blk_mq_plug()
 1335 	 * should not have any consequences.
 1336 	 */
 1337 	if (current->plug && !at_head) {
 1338 		blk_add_rq_to_plug(current->plug, rq);
 1339 		return;
 1340 	}
 1341 
 1342 	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
 1343 	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
 1344 }
 1345 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
 1346 
 1347 struct blk_rq_wait {
 1348 	struct completion done;
 1349 	blk_status_t ret;
 1350 };
 1351 
 1352 static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret)
 1353 {
 1354 	struct blk_rq_wait *wait = rq->end_io_data;
 1355 
 1356 	wait->ret = ret;
 1357 	complete(&wait->done);
 1358 	return RQ_END_IO_NONE;
 1359 }
 1360 
 1361 bool blk_rq_is_poll(struct request *rq)
 1362 {
 1363 	if (!rq->mq_hctx)
 1364 		return false;
 1365 	if (rq->mq_hctx->type != HCTX_TYPE_POLL)
 1366 		return false;
 1367 	return true;
 1368 }
 1369 EXPORT_SYMBOL_GPL(blk_rq_is_poll);
 1370 
 1371 static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
 1372 {
 1373 	do {
 1374 		blk_hctx_poll(rq->q, rq->mq_hctx, NULL, 0);
 1375 		cond_resched();
 1376 	} while (!completion_done(wait));
 1377 }
 1378 
 1379 /**
 1380  * blk_execute_rq - insert a request into queue for execution
 1381  * @rq:		request to insert
 1382  * @at_head:    insert request at head or tail of queue
 1383  *
 1384  * Description:
 1385  *    Insert a fully prepared request at the back of the I/O scheduler queue
 1386  *    for execution and wait for completion.
 1387  * Return: The blk_status_t result provided to blk_mq_end_request().
 1388  */
 1389 blk_status_t blk_execute_rq(struct request *rq, bool at_head)
 1390 {
 1391 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 1392 	struct blk_rq_wait wait = {
 1393 		.done = COMPLETION_INITIALIZER_ONSTACK(wait.done),
 1394 	};
 1395 
 1396 	WARN_ON(irqs_disabled());
 1397 	WARN_ON(!blk_rq_is_passthrough(rq));
 1398 
 1399 	rq->end_io_data = &wait;
 1400 	rq->end_io = blk_end_sync_rq;
 1401 
 1402 	blk_account_io_start(rq);
 1403 	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
 1404 	blk_mq_run_hw_queue(hctx, false);
 1405 
 1406 	if (blk_rq_is_poll(rq)) {
 1407 		blk_rq_poll_completion(rq, &wait.done);
 1408 	} else {
 1409 		/*
 1410 		 * Prevent hang_check timer from firing at us during very long
 1411 		 * I/O
 1412 		 */
 1413 		unsigned long hang_check = sysctl_hung_task_timeout_secs;
 1414 
 1415 		if (hang_check)
 1416 			while (!wait_for_completion_io_timeout(&wait.done,
 1417 					hang_check * (HZ/2)))
 1418 				;
 1419 		else
 1420 			wait_for_completion_io(&wait.done);
 1421 	}
 1422 
 1423 	return wait.ret;
 1424 }
 1425 EXPORT_SYMBOL(blk_execute_rq);
 1426 
 1427 static void __blk_mq_requeue_request(struct request *rq)
 1428 {
 1429 	struct request_queue *q = rq->q;
 1430 
 1431 	blk_mq_put_driver_tag(rq);
 1432 
 1433 	trace_block_rq_requeue(rq);
 1434 	rq_qos_requeue(q, rq);
 1435 
 1436 	if (blk_mq_request_started(rq)) {
 1437 		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
 1438 		rq->rq_flags &= ~RQF_TIMED_OUT;
 1439 	}
 1440 }
 1441 
 1442 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
 1443 {
 1444 	struct request_queue *q = rq->q;
 1445 	unsigned long flags;
 1446 
 1447 	__blk_mq_requeue_request(rq);
 1448 
 1449 	/* this request will be re-inserted to io scheduler queue */
 1450 	blk_mq_sched_requeue_request(rq);
 1451 
 1452 	spin_lock_irqsave(&q->requeue_lock, flags);
 1453 	list_add_tail(&rq->queuelist, &q->requeue_list);
 1454 	spin_unlock_irqrestore(&q->requeue_lock, flags);
 1455 
 1456 	if (kick_requeue_list)
 1457 		blk_mq_kick_requeue_list(q);
 1458 }
 1459 EXPORT_SYMBOL(blk_mq_requeue_request);
 1460 
 1461 static void blk_mq_requeue_work(struct work_struct *work)
 1462 {
 1463 	struct request_queue *q =
 1464 		container_of(work, struct request_queue, requeue_work.work);
 1465 	LIST_HEAD(rq_list);
 1466 	LIST_HEAD(flush_list);
 1467 	struct request *rq;
 1468 
 1469 	spin_lock_irq(&q->requeue_lock);
 1470 	list_splice_init(&q->requeue_list, &rq_list);
 1471 	list_splice_init(&q->flush_list, &flush_list);
 1472 	spin_unlock_irq(&q->requeue_lock);
 1473 
 1474 	while (!list_empty(&rq_list)) {
 1475 		rq = list_entry(rq_list.next, struct request, queuelist);
 1476 		/*
 1477 		 * If RQF_DONTPREP ist set, the request has been started by the
 1478 		 * driver already and might have driver-specific data allocated
 1479 		 * already.  Insert it into the hctx dispatch list to avoid
 1480 		 * block layer merges for the request.
 1481 		 */
 1482 		if (rq->rq_flags & RQF_DONTPREP) {
 1483 			list_del_init(&rq->queuelist);
 1484 			blk_mq_request_bypass_insert(rq, 0);
 1485 		} else {
 1486 			list_del_init(&rq->queuelist);
 1487 			blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD);
 1488 		}
 1489 	}
 1490 
 1491 	while (!list_empty(&flush_list)) {
 1492 		rq = list_entry(flush_list.next, struct request, queuelist);
 1493 		list_del_init(&rq->queuelist);
 1494 		blk_mq_insert_request(rq, 0);
 1495 	}
 1496 
 1497 	blk_mq_run_hw_queues(q, false);
 1498 }
 1499 
 1500 void blk_mq_kick_requeue_list(struct request_queue *q)
 1501 {
 1502 	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
 1503 }
 1504 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 1505 
 1506 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
 1507 				    unsigned long msecs)
 1508 {
 1509 	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
 1510 				    msecs_to_jiffies(msecs));
 1511 }
 1512 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
 1513 
 1514 static bool blk_mq_rq_inflight(struct request *rq, void *priv)
 1515 {
 1516 	/*
 1517 	 * If we find a request that isn't idle we know the queue is busy
 1518 	 * as it's checked in the iter.
 1519 	 * Return false to stop the iteration.
 1520 	 */
 1521 	if (blk_mq_request_started(rq)) {
 1522 		bool *busy = priv;
 1523 
 1524 		*busy = true;
 1525 		return false;
 1526 	}
 1527 
 1528 	return true;
 1529 }
 1530 
 1531 bool blk_mq_queue_inflight(struct request_queue *q)
 1532 {
 1533 	bool busy = false;
 1534 
 1535 	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
 1536 	return busy;
 1537 }
 1538 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
 1539 
 1540 static void blk_mq_rq_timed_out(struct request *req)
 1541 {
 1542 	req->rq_flags |= RQF_TIMED_OUT;
 1543 	if (req->q->mq_ops->timeout) {
 1544 		enum blk_eh_timer_return ret;
 1545 
 1546 		ret = req->q->mq_ops->timeout(req);
 1547 		if (ret == BLK_EH_DONE)
 1548 			return;
 1549 		WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
 1550 	}
 1551 
 1552 	blk_add_timer(req);
 1553 }
 1554 
 1555 struct blk_expired_data {
 1556 	bool has_timedout_rq;
 1557 	unsigned long next;
 1558 	unsigned long timeout_start;
 1559 };
 1560 
 1561 static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired)
 1562 {
 1563 	unsigned long deadline;
 1564 
 1565 	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
 1566 		return false;
 1567 	if (rq->rq_flags & RQF_TIMED_OUT)
 1568 		return false;
 1569 
 1570 	deadline = READ_ONCE(rq->deadline);
 1571 	if (time_after_eq(expired->timeout_start, deadline))
 1572 		return true;
 1573 
 1574 	if (expired->next == 0)
 1575 		expired->next = deadline;
 1576 	else if (time_after(expired->next, deadline))
 1577 		expired->next = deadline;
 1578 	return false;
 1579 }
 1580 
 1581 void blk_mq_put_rq_ref(struct request *rq)
 1582 {
 1583 	if (is_flush_rq(rq)) {
 1584 		if (rq->end_io(rq, 0) == RQ_END_IO_FREE)
 1585 			blk_mq_free_request(rq);
 1586 	} else if (req_ref_put_and_test(rq)) {
 1587 		__blk_mq_free_request(rq);
 1588 	}
 1589 }
 1590 
 1591 static bool blk_mq_check_expired(struct request *rq, void *priv)
 1592 {
 1593 	struct blk_expired_data *expired = priv;
 1594 
 1595 	/*
 1596 	 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
 1597 	 * be reallocated underneath the timeout handler's processing, then
 1598 	 * the expire check is reliable. If the request is not expired, then
 1599 	 * it was completed and reallocated as a new request after returning
 1600 	 * from blk_mq_check_expired().
 1601 	 */
 1602 	if (blk_mq_req_expired(rq, expired)) {
 1603 		expired->has_timedout_rq = true;
 1604 		return false;
 1605 	}
 1606 	return true;
 1607 }
 1608 
 1609 static bool blk_mq_handle_expired(struct request *rq, void *priv)
 1610 {
 1611 	struct blk_expired_data *expired = priv;
 1612 
 1613 	if (blk_mq_req_expired(rq, expired))
 1614 		blk_mq_rq_timed_out(rq);
 1615 	return true;
 1616 }
 1617 
 1618 static void blk_mq_timeout_work(struct work_struct *work)
 1619 {
 1620 	struct request_queue *q =
 1621 		container_of(work, struct request_queue, timeout_work);
 1622 	struct blk_expired_data expired = {
 1623 		.timeout_start = jiffies,
 1624 	};
 1625 	struct blk_mq_hw_ctx *hctx;
 1626 	unsigned long i;
 1627 
 1628 	/* A deadlock might occur if a request is stuck requiring a
 1629 	 * timeout at the same time a queue freeze is waiting
 1630 	 * completion, since the timeout code would not be able to
 1631 	 * acquire the queue reference here.
 1632 	 *
 1633 	 * That's why we don't use blk_queue_enter here; instead, we use
 1634 	 * percpu_ref_tryget directly, because we need to be able to
 1635 	 * obtain a reference even in the short window between the queue
 1636 	 * starting to freeze, by dropping the first reference in
 1637 	 * blk_freeze_queue_start, and the moment the last request is
 1638 	 * consumed, marked by the instant q_usage_counter reaches
 1639 	 * zero.
 1640 	 */
 1641 	if (!percpu_ref_tryget(&q->q_usage_counter))
 1642 		return;
 1643 
 1644 	/* check if there is any timed-out request */
 1645 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
 1646 	if (expired.has_timedout_rq) {
 1647 		/*
 1648 		 * Before walking tags, we must ensure any submit started
 1649 		 * before the current time has finished. Since the submit
 1650 		 * uses srcu or rcu, wait for a synchronization point to
 1651 		 * ensure all running submits have finished
 1652 		 */
 1653 		blk_mq_wait_quiesce_done(q->tag_set);
 1654 
 1655 		expired.next = 0;
 1656 		blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired);
 1657 	}
 1658 
 1659 	if (expired.next != 0) {
 1660 		mod_timer(&q->timeout, expired.next);
 1661 	} else {
 1662 		/*
 1663 		 * Request timeouts are handled as a forward rolling timer. If
 1664 		 * we end up here it means that no requests are pending and
 1665 		 * also that no request has been pending for a while. Mark
 1666 		 * each hctx as idle.
 1667 		 */
 1668 		queue_for_each_hw_ctx(q, hctx, i) {
 1669 			/* the hctx may be unmapped, so check it here */
 1670 			if (blk_mq_hw_queue_mapped(hctx))
 1671 				blk_mq_tag_idle(hctx);
 1672 		}
 1673 	}
 1674 	blk_queue_exit(q);
 1675 }
 1676 
 1677 struct flush_busy_ctx_data {
 1678 	struct blk_mq_hw_ctx *hctx;
 1679 	struct list_head *list;
 1680 };
 1681 
 1682 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
 1683 {
 1684 	struct flush_busy_ctx_data *flush_data = data;
 1685 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
 1686 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
 1687 	enum hctx_type type = hctx->type;
 1688 
 1689 	spin_lock(&ctx->lock);
 1690 	list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
 1691 	sbitmap_clear_bit(sb, bitnr);
 1692 	spin_unlock(&ctx->lock);
 1693 	return true;
 1694 }
 1695 
 1696 /*
 1697  * Process software queues that have been marked busy, splicing them
 1698  * to the for-dispatch
 1699  */
 1700 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 1701 {
 1702 	struct flush_busy_ctx_data data = {
 1703 		.hctx = hctx,
 1704 		.list = list,
 1705 	};
 1706 
 1707 	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
 1708 }
 1709 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
 1710 
 1711 struct dispatch_rq_data {
 1712 	struct blk_mq_hw_ctx *hctx;
 1713 	struct request *rq;
 1714 };
 1715 
 1716 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
 1717 		void *data)
 1718 {
 1719 	struct dispatch_rq_data *dispatch_data = data;
 1720 	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
 1721 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
 1722 	enum hctx_type type = hctx->type;
 1723 
 1724 	spin_lock(&ctx->lock);
 1725 	if (!list_empty(&ctx->rq_lists[type])) {
 1726 		dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
 1727 		list_del_init(&dispatch_data->rq->queuelist);
 1728 		if (list_empty(&ctx->rq_lists[type]))
 1729 			sbitmap_clear_bit(sb, bitnr);
 1730 	}
 1731 	spin_unlock(&ctx->lock);
 1732 
 1733 	return !dispatch_data->rq;
 1734 }
 1735 
 1736 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
 1737 					struct blk_mq_ctx *start)
 1738 {
 1739 	unsigned off = start ? start->index_hw[hctx->type] : 0;
 1740 	struct dispatch_rq_data data = {
 1741 		.hctx = hctx,
 1742 		.rq   = NULL,
 1743 	};
 1744 
 1745 	__sbitmap_for_each_set(&hctx->ctx_map, off,
 1746 			       dispatch_rq_from_ctx, &data);
 1747 
 1748 	return data.rq;
 1749 }
 1750 
 1751 static bool __blk_mq_alloc_driver_tag(struct request *rq)
 1752 {
 1753 	struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
 1754 	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
 1755 	int tag;
 1756 
 1757 	blk_mq_tag_busy(rq->mq_hctx);
 1758 
 1759 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
 1760 		bt = &rq->mq_hctx->tags->breserved_tags;
 1761 		tag_offset = 0;
 1762 	} else {
 1763 		if (!hctx_may_queue(rq->mq_hctx, bt))
 1764 			return false;
 1765 	}
 1766 
 1767 	tag = __sbitmap_queue_get(bt);
 1768 	if (tag == BLK_MQ_NO_TAG)
 1769 		return false;
 1770 
 1771 	rq->tag = tag + tag_offset;
 1772 	return true;
 1773 }
 1774 
 1775 bool __blk_mq_get_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq)
 1776 {
 1777 	if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_alloc_driver_tag(rq))
 1778 		return false;
 1779 
 1780 	if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
 1781 			!(rq->rq_flags & RQF_MQ_INFLIGHT)) {
 1782 		rq->rq_flags |= RQF_MQ_INFLIGHT;
 1783 		__blk_mq_inc_active_requests(hctx);
 1784 	}
 1785 	hctx->tags->rqs[rq->tag] = rq;
 1786 	return true;
 1787 }
 1788 
 1789 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
 1790 				int flags, void *key)
 1791 {
 1792 	struct blk_mq_hw_ctx *hctx;
 1793 
 1794 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
 1795 
 1796 	spin_lock(&hctx->dispatch_wait_lock);
 1797 	if (!list_empty(&wait->entry)) {
 1798 		struct sbitmap_queue *sbq;
 1799 
 1800 		list_del_init(&wait->entry);
 1801 		sbq = &hctx->tags->bitmap_tags;
 1802 		atomic_dec(&sbq->ws_active);
 1803 	}
 1804 	spin_unlock(&hctx->dispatch_wait_lock);
 1805 
 1806 	blk_mq_run_hw_queue(hctx, true);
 1807 	return 1;
 1808 }
 1809 
 1810 /*
 1811  * Mark us waiting for a tag. For shared tags, this involves hooking us into
 1812  * the tag wakeups. For non-shared tags, we can simply mark us needing a
 1813  * restart. For both cases, take care to check the condition again after
 1814  * marking us as waiting.
 1815  */
 1816 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
 1817 				 struct request *rq)
 1818 {
 1819 	struct sbitmap_queue *sbq;
 1820 	struct wait_queue_head *wq;
 1821 	wait_queue_entry_t *wait;
 1822 	bool ret;
 1823 
 1824 	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
 1825 	    !(blk_mq_is_shared_tags(hctx->flags))) {
 1826 		blk_mq_sched_mark_restart_hctx(hctx);
 1827 
 1828 		/*
 1829 		 * It's possible that a tag was freed in the window between the
 1830 		 * allocation failure and adding the hardware queue to the wait
 1831 		 * queue.
 1832 		 *
 1833 		 * Don't clear RESTART here, someone else could have set it.
 1834 		 * At most this will cost an extra queue run.
 1835 		 */
 1836 		return blk_mq_get_driver_tag(rq);
 1837 	}
 1838 
 1839 	wait = &hctx->dispatch_wait;
 1840 	if (!list_empty_careful(&wait->entry))
 1841 		return false;
 1842 
 1843 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag))
 1844 		sbq = &hctx->tags->breserved_tags;
 1845 	else
 1846 		sbq = &hctx->tags->bitmap_tags;
 1847 	wq = &bt_wait_ptr(sbq, hctx)->wait;
 1848 
 1849 	spin_lock_irq(&wq->lock);
 1850 	spin_lock(&hctx->dispatch_wait_lock);
 1851 	if (!list_empty(&wait->entry)) {
 1852 		spin_unlock(&hctx->dispatch_wait_lock);
 1853 		spin_unlock_irq(&wq->lock);
 1854 		return false;
 1855 	}
 1856 
 1857 	atomic_inc(&sbq->ws_active);
 1858 	wait->flags &= ~WQ_FLAG_EXCLUSIVE;
 1859 	__add_wait_queue(wq, wait);
 1860 
 1861 	/*
 1862 	 * It's possible that a tag was freed in the window between the
 1863 	 * allocation failure and adding the hardware queue to the wait
 1864 	 * queue.
 1865 	 */
 1866 	ret = blk_mq_get_driver_tag(rq);
 1867 	if (!ret) {
 1868 		spin_unlock(&hctx->dispatch_wait_lock);
 1869 		spin_unlock_irq(&wq->lock);
 1870 		return false;
 1871 	}
 1872 
 1873 	/*
 1874 	 * We got a tag, remove ourselves from the wait queue to ensure
 1875 	 * someone else gets the wakeup.
 1876 	 */
 1877 	list_del_init(&wait->entry);
 1878 	atomic_dec(&sbq->ws_active);
 1879 	spin_unlock(&hctx->dispatch_wait_lock);
 1880 	spin_unlock_irq(&wq->lock);
 1881 
 1882 	return true;
 1883 }
 1884 
 1885 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT  8
 1886 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR  4
 1887 /*
 1888  * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
 1889  * - EWMA is one simple way to compute running average value
 1890  * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
 1891  * - take 4 as factor for avoiding to get too small(0) result, and this
 1892  *   factor doesn't matter because EWMA decreases exponentially
 1893  */
 1894 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
 1895 {
 1896 	unsigned int ewma;
 1897 
 1898 	ewma = hctx->dispatch_busy;
 1899 
 1900 	if (!ewma && !busy)
 1901 		return;
 1902 
 1903 	ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
 1904 	if (busy)
 1905 		ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
 1906 	ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
 1907 
 1908 	hctx->dispatch_busy = ewma;
 1909 }
 1910 
 1911 #define BLK_MQ_RESOURCE_DELAY	3		/* ms units */
 1912 
 1913 static void blk_mq_handle_dev_resource(struct request *rq,
 1914 				       struct list_head *list)
 1915 {
 1916 	list_add(&rq->queuelist, list);
 1917 	__blk_mq_requeue_request(rq);
 1918 }
 1919 
 1920 static void blk_mq_handle_zone_resource(struct request *rq,
 1921 					struct list_head *zone_list)
 1922 {
 1923 	/*
 1924 	 * If we end up here it is because we cannot dispatch a request to a
 1925 	 * specific zone due to LLD level zone-write locking or other zone
 1926 	 * related resource not being available. In this case, set the request
 1927 	 * aside in zone_list for retrying it later.
 1928 	 */
 1929 	list_add(&rq->queuelist, zone_list);
 1930 	__blk_mq_requeue_request(rq);
 1931 }
 1932 
 1933 enum prep_dispatch {
 1934 	PREP_DISPATCH_OK,
 1935 	PREP_DISPATCH_NO_TAG,
 1936 	PREP_DISPATCH_NO_BUDGET,
 1937 };
 1938 
 1939 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
 1940 						  bool need_budget)
 1941 {
 1942 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 1943 	int budget_token = -1;
 1944 
 1945 	if (need_budget) {
 1946 		budget_token = blk_mq_get_dispatch_budget(rq->q);
 1947 		if (budget_token < 0) {
 1948 			blk_mq_put_driver_tag(rq);
 1949 			return PREP_DISPATCH_NO_BUDGET;
 1950 		}
 1951 		blk_mq_set_rq_budget_token(rq, budget_token);
 1952 	}
 1953 
 1954 	if (!blk_mq_get_driver_tag(rq)) {
 1955 		/*
 1956 		 * The initial allocation attempt failed, so we need to
 1957 		 * rerun the hardware queue when a tag is freed. The
 1958 		 * waitqueue takes care of that. If the queue is run
 1959 		 * before we add this entry back on the dispatch list,
 1960 		 * we'll re-run it below.
 1961 		 */
 1962 		if (!blk_mq_mark_tag_wait(hctx, rq)) {
 1963 			/*
 1964 			 * All budgets not got from this function will be put
 1965 			 * together during handling partial dispatch
 1966 			 */
 1967 			if (need_budget)
 1968 				blk_mq_put_dispatch_budget(rq->q, budget_token);
 1969 			return PREP_DISPATCH_NO_TAG;
 1970 		}
 1971 	}
 1972 
 1973 	return PREP_DISPATCH_OK;
 1974 }
 1975 
 1976 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
 1977 static void blk_mq_release_budgets(struct request_queue *q,
 1978 		struct list_head *list)
 1979 {
 1980 	struct request *rq;
 1981 
 1982 	list_for_each_entry(rq, list, queuelist) {
 1983 		int budget_token = blk_mq_get_rq_budget_token(rq);
 1984 
 1985 		if (budget_token >= 0)
 1986 			blk_mq_put_dispatch_budget(q, budget_token);
 1987 	}
 1988 }
 1989 
 1990 /*
 1991  * blk_mq_commit_rqs will notify driver using bd->last that there is no
 1992  * more requests. (See comment in struct blk_mq_ops for commit_rqs for
 1993  * details)
 1994  * Attention, we should explicitly call this in unusual cases:
 1995  *  1) did not queue everything initially scheduled to queue
 1996  *  2) the last attempt to queue a request failed
 1997  */
 1998 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,
 1999 			      bool from_schedule)
 2000 {
 2001 	if (hctx->queue->mq_ops->commit_rqs && queued) {
 2002 		trace_block_unplug(hctx->queue, queued, !from_schedule);
 2003 		hctx->queue->mq_ops->commit_rqs(hctx);
 2004 	}
 2005 }
 2006 
 2007 /*
 2008  * Returns true if we did some work AND can potentially do more.
 2009  */
 2010 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
 2011 			     unsigned int nr_budgets)
 2012 {
 2013 	enum prep_dispatch prep;
 2014 	struct request_queue *q = hctx->queue;
 2015 	struct request *rq;
 2016 	int queued;
 2017 	blk_status_t ret = BLK_STS_OK;
 2018 	LIST_HEAD(zone_list);
 2019 	bool needs_resource = false;
 2020 
 2021 	if (list_empty(list))
 2022 		return false;
 2023 
 2024 	/*
 2025 	 * Now process all the entries, sending them to the driver.
 2026 	 */
 2027 	queued = 0;
 2028 	do {
 2029 		struct blk_mq_queue_data bd;
 2030 
 2031 		rq = list_first_entry(list, struct request, queuelist);
 2032 
 2033 		WARN_ON_ONCE(hctx != rq->mq_hctx);
 2034 		prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
 2035 		if (prep != PREP_DISPATCH_OK)
 2036 			break;
 2037 
 2038 		list_del_init(&rq->queuelist);
 2039 
 2040 		bd.rq = rq;
 2041 		bd.last = list_empty(list);
 2042 
 2043 		/*
 2044 		 * once the request is queued to lld, no need to cover the
 2045 		 * budget any more
 2046 		 */
 2047 		if (nr_budgets)
 2048 			nr_budgets--;
 2049 		ret = q->mq_ops->queue_rq(hctx, &bd);
 2050 		switch (ret) {
 2051 		case BLK_STS_OK:
 2052 			queued++;
 2053 			break;
 2054 		case BLK_STS_RESOURCE:
 2055 			needs_resource = true;
 2056 			fallthrough;
 2057 		case BLK_STS_DEV_RESOURCE:
 2058 			blk_mq_handle_dev_resource(rq, list);
 2059 			goto out;
 2060 		case BLK_STS_ZONE_RESOURCE:
 2061 			/*
 2062 			 * Move the request to zone_list and keep going through
 2063 			 * the dispatch list to find more requests the drive can
 2064 			 * accept.
 2065 			 */
 2066 			blk_mq_handle_zone_resource(rq, &zone_list);
 2067 			needs_resource = true;
 2068 			break;
 2069 		default:
 2070 			blk_mq_end_request(rq, ret);
 2071 		}
 2072 	} while (!list_empty(list));
 2073 out:
 2074 	if (!list_empty(&zone_list))
 2075 		list_splice_tail_init(&zone_list, list);
 2076 
 2077 	/* If we didn't flush the entire list, we could have told the driver
 2078 	 * there was more coming, but that turned out to be a lie.
 2079 	 */
 2080 	if (!list_empty(list) || ret != BLK_STS_OK)
 2081 		blk_mq_commit_rqs(hctx, queued, false);
 2082 
 2083 	/*
 2084 	 * Any items that need requeuing? Stuff them into hctx->dispatch,
 2085 	 * that is where we will continue on next queue run.
 2086 	 */
 2087 	if (!list_empty(list)) {
 2088 		bool needs_restart;
 2089 		/* For non-shared tags, the RESTART check will suffice */
 2090 		bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
 2091 			((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) ||
 2092 			blk_mq_is_shared_tags(hctx->flags));
 2093 
 2094 		if (nr_budgets)
 2095 			blk_mq_release_budgets(q, list);
 2096 
 2097 		spin_lock(&hctx->lock);
 2098 		list_splice_tail_init(list, &hctx->dispatch);
 2099 		spin_unlock(&hctx->lock);
 2100 
 2101 		/*
 2102 		 * Order adding requests to hctx->dispatch and checking
 2103 		 * SCHED_RESTART flag. The pair of this smp_mb() is the one
 2104 		 * in blk_mq_sched_restart(). Avoid restart code path to
 2105 		 * miss the new added requests to hctx->dispatch, meantime
 2106 		 * SCHED_RESTART is observed here.
 2107 		 */
 2108 		smp_mb();
 2109 
 2110 		/*
 2111 		 * If SCHED_RESTART was set by the caller of this function and
 2112 		 * it is no longer set that means that it was cleared by another
 2113 		 * thread and hence that a queue rerun is needed.
 2114 		 *
 2115 		 * If 'no_tag' is set, that means that we failed getting
 2116 		 * a driver tag with an I/O scheduler attached. If our dispatch
 2117 		 * waitqueue is no longer active, ensure that we run the queue
 2118 		 * AFTER adding our entries back to the list.
 2119 		 *
 2120 		 * If no I/O scheduler has been configured it is possible that
 2121 		 * the hardware queue got stopped and restarted before requests
 2122 		 * were pushed back onto the dispatch list. Rerun the queue to
 2123 		 * avoid starvation. Notes:
 2124 		 * - blk_mq_run_hw_queue() checks whether or not a queue has
 2125 		 *   been stopped before rerunning a queue.
 2126 		 * - Some but not all block drivers stop a queue before
 2127 		 *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
 2128 		 *   and dm-rq.
 2129 		 *
 2130 		 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
 2131 		 * bit is set, run queue after a delay to avoid IO stalls
 2132 		 * that could otherwise occur if the queue is idle.  We'll do
 2133 		 * similar if we couldn't get budget or couldn't lock a zone
 2134 		 * and SCHED_RESTART is set.
 2135 		 */
 2136 		needs_restart = blk_mq_sched_needs_restart(hctx);
 2137 		if (prep == PREP_DISPATCH_NO_BUDGET)
 2138 			needs_resource = true;
 2139 		if (!needs_restart ||
 2140 		    (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
 2141 			blk_mq_run_hw_queue(hctx, true);
 2142 		else if (needs_resource)
 2143 			blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
 2144 
 2145 		blk_mq_update_dispatch_busy(hctx, true);
 2146 		return false;
 2147 	}
 2148 
 2149 	blk_mq_update_dispatch_busy(hctx, false);
 2150 	return true;
 2151 }
 2152 
 2153 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
 2154 {
 2155 	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
 2156 
 2157 	if (cpu >= nr_cpu_ids)
 2158 		cpu = cpumask_first(hctx->cpumask);
 2159 	return cpu;
 2160 }
 2161 
 2162 /*
 2163  * It'd be great if the workqueue API had a way to pass
 2164  * in a mask and had some smarts for more clever placement.
 2165  * For now we just round-robin here, switching for every
 2166  * BLK_MQ_CPU_WORK_BATCH queued items.
 2167  */
 2168 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 2169 {
 2170 	bool tried = false;
 2171 	int next_cpu = hctx->next_cpu;
 2172 
 2173 	if (hctx->queue->nr_hw_queues == 1)
 2174 		return WORK_CPU_UNBOUND;
 2175 
 2176 	if (--hctx->next_cpu_batch <= 0) {
 2177 select_cpu:
 2178 		next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
 2179 				cpu_online_mask);
 2180 		if (next_cpu >= nr_cpu_ids)
 2181 			next_cpu = blk_mq_first_mapped_cpu(hctx);
 2182 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 2183 	}
 2184 
 2185 	/*
 2186 	 * Do unbound schedule if we can't find a online CPU for this hctx,
 2187 	 * and it should only happen in the path of handling CPU DEAD.
 2188 	 */
 2189 	if (!cpu_online(next_cpu)) {
 2190 		if (!tried) {
 2191 			tried = true;
 2192 			goto select_cpu;
 2193 		}
 2194 
 2195 		/*
 2196 		 * Make sure to re-select CPU next time once after CPUs
 2197 		 * in hctx->cpumask become online again.
 2198 		 */
 2199 		hctx->next_cpu = next_cpu;
 2200 		hctx->next_cpu_batch = 1;
 2201 		return WORK_CPU_UNBOUND;
 2202 	}
 2203 
 2204 	hctx->next_cpu = next_cpu;
 2205 	return next_cpu;
 2206 }
 2207 
 2208 /**
 2209  * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
 2210  * @hctx: Pointer to the hardware queue to run.
 2211  * @msecs: Milliseconds of delay to wait before running the queue.
 2212  *
 2213  * Run a hardware queue asynchronously with a delay of @msecs.
 2214  */
 2215 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 2216 {
 2217 	if (unlikely(blk_mq_hctx_stopped(hctx)))
 2218 		return;
 2219 	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
 2220 				    msecs_to_jiffies(msecs));
 2221 }
 2222 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
 2223 
 2224 /**
 2225  * blk_mq_run_hw_queue - Start to run a hardware queue.
 2226  * @hctx: Pointer to the hardware queue to run.
 2227  * @async: If we want to run the queue asynchronously.
 2228  *
 2229  * Check if the request queue is not in a quiesced state and if there are
 2230  * pending requests to be sent. If this is true, run the queue to send requests
 2231  * to hardware.
 2232  */
 2233 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 2234 {
 2235 	bool need_run;
 2236 
 2237 	/*
 2238 	 * We can't run the queue inline with interrupts disabled.
 2239 	 */
 2240 	WARN_ON_ONCE(!async && in_interrupt());
 2241 
 2242 	might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING);
 2243 
 2244 	/*
 2245 	 * When queue is quiesced, we may be switching io scheduler, or
 2246 	 * updating nr_hw_queues, or other things, and we can't run queue
 2247 	 * any more, even __blk_mq_hctx_has_pending() can't be called safely.
 2248 	 *
 2249 	 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
 2250 	 * quiesced.
 2251 	 */
 2252 	__blk_mq_run_dispatch_ops(hctx->queue, false,
 2253 		need_run = !blk_queue_quiesced(hctx->queue) &&
 2254 		blk_mq_hctx_has_pending(hctx));
 2255 
 2256 	if (!need_run)
 2257 		return;
 2258 
 2259 	if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
 2260 		blk_mq_delay_run_hw_queue(hctx, 0);
 2261 		return;
 2262 	}
 2263 
 2264 	blk_mq_run_dispatch_ops(hctx->queue,
 2265 				blk_mq_sched_dispatch_requests(hctx));
 2266 }
 2267 EXPORT_SYMBOL(blk_mq_run_hw_queue);
 2268 
 2269 /*
 2270  * Return prefered queue to dispatch from (if any) for non-mq aware IO
 2271  * scheduler.
 2272  */
 2273 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
 2274 {
 2275 	struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
 2276 	/*
 2277 	 * If the IO scheduler does not respect hardware queues when
 2278 	 * dispatching, we just don't bother with multiple HW queues and
 2279 	 * dispatch from hctx for the current CPU since running multiple queues
 2280 	 * just causes lock contention inside the scheduler and pointless cache
 2281 	 * bouncing.
 2282 	 */
 2283 	struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
 2284 
 2285 	if (!blk_mq_hctx_stopped(hctx))
 2286 		return hctx;
 2287 	return NULL;
 2288 }
 2289 
 2290 /**
 2291  * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
 2292  * @q: Pointer to the request queue to run.
 2293  * @async: If we want to run the queue asynchronously.
 2294  */
 2295 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
 2296 {
 2297 	struct blk_mq_hw_ctx *hctx, *sq_hctx;
 2298 	unsigned long i;
 2299 
 2300 	sq_hctx = NULL;
 2301 	if (blk_queue_sq_sched(q))
 2302 		sq_hctx = blk_mq_get_sq_hctx(q);
 2303 	queue_for_each_hw_ctx(q, hctx, i) {
 2304 		if (blk_mq_hctx_stopped(hctx))
 2305 			continue;
 2306 		/*
 2307 		 * Dispatch from this hctx either if there's no hctx preferred
 2308 		 * by IO scheduler or if it has requests that bypass the
 2309 		 * scheduler.
 2310 		 */
 2311 		if (!sq_hctx || sq_hctx == hctx ||
 2312 		    !list_empty_careful(&hctx->dispatch))
 2313 			blk_mq_run_hw_queue(hctx, async);
 2314 	}
 2315 }
 2316 EXPORT_SYMBOL(blk_mq_run_hw_queues);
 2317 
 2318 /**
 2319  * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
 2320  * @q: Pointer to the request queue to run.
 2321  * @msecs: Milliseconds of delay to wait before running the queues.
 2322  */
 2323 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
 2324 {
 2325 	struct blk_mq_hw_ctx *hctx, *sq_hctx;
 2326 	unsigned long i;
 2327 
 2328 	sq_hctx = NULL;
 2329 	if (blk_queue_sq_sched(q))
 2330 		sq_hctx = blk_mq_get_sq_hctx(q);
 2331 	queue_for_each_hw_ctx(q, hctx, i) {
 2332 		if (blk_mq_hctx_stopped(hctx))
 2333 			continue;
 2334 		/*
 2335 		 * If there is already a run_work pending, leave the
 2336 		 * pending delay untouched. Otherwise, a hctx can stall
 2337 		 * if another hctx is re-delaying the other's work
 2338 		 * before the work executes.
 2339 		 */
 2340 		if (delayed_work_pending(&hctx->run_work))
 2341 			continue;
 2342 		/*
 2343 		 * Dispatch from this hctx either if there's no hctx preferred
 2344 		 * by IO scheduler or if it has requests that bypass the
 2345 		 * scheduler.
 2346 		 */
 2347 		if (!sq_hctx || sq_hctx == hctx ||
 2348 		    !list_empty_careful(&hctx->dispatch))
 2349 			blk_mq_delay_run_hw_queue(hctx, msecs);
 2350 	}
 2351 }
 2352 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
 2353 
 2354 /*
 2355  * This function is often used for pausing .queue_rq() by driver when
 2356  * there isn't enough resource or some conditions aren't satisfied, and
 2357  * BLK_STS_RESOURCE is usually returned.
 2358  *
 2359  * We do not guarantee that dispatch can be drained or blocked
 2360  * after blk_mq_stop_hw_queue() returns. Please use
 2361  * blk_mq_quiesce_queue() for that requirement.
 2362  */
 2363 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 2364 {
 2365 	cancel_delayed_work(&hctx->run_work);
 2366 
 2367 	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 2368 }
 2369 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 2370 
 2371 /*
 2372  * This function is often used for pausing .queue_rq() by driver when
 2373  * there isn't enough resource or some conditions aren't satisfied, and
 2374  * BLK_STS_RESOURCE is usually returned.
 2375  *
 2376  * We do not guarantee that dispatch can be drained or blocked
 2377  * after blk_mq_stop_hw_queues() returns. Please use
 2378  * blk_mq_quiesce_queue() for that requirement.
 2379  */
 2380 void blk_mq_stop_hw_queues(struct request_queue *q)
 2381 {
 2382 	struct blk_mq_hw_ctx *hctx;
 2383 	unsigned long i;
 2384 
 2385 	queue_for_each_hw_ctx(q, hctx, i)
 2386 		blk_mq_stop_hw_queue(hctx);
 2387 }
 2388 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 2389 
 2390 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 2391 {
 2392 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 2393 
 2394 	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
 2395 }
 2396 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 2397 
 2398 void blk_mq_start_hw_queues(struct request_queue *q)
 2399 {
 2400 	struct blk_mq_hw_ctx *hctx;
 2401 	unsigned long i;
 2402 
 2403 	queue_for_each_hw_ctx(q, hctx, i)
 2404 		blk_mq_start_hw_queue(hctx);
 2405 }
 2406 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 2407 
 2408 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 2409 {
 2410 	if (!blk_mq_hctx_stopped(hctx))
 2411 		return;
 2412 
 2413 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 2414 	blk_mq_run_hw_queue(hctx, async);
 2415 }
 2416 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
 2417 
 2418 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 2419 {
 2420 	struct blk_mq_hw_ctx *hctx;
 2421 	unsigned long i;
 2422 
 2423 	queue_for_each_hw_ctx(q, hctx, i)
 2424 		blk_mq_start_stopped_hw_queue(hctx, async ||
 2425 					(hctx->flags & BLK_MQ_F_BLOCKING));
 2426 }
 2427 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 2428 
 2429 static void blk_mq_run_work_fn(struct work_struct *work)
 2430 {
 2431 	struct blk_mq_hw_ctx *hctx =
 2432 		container_of(work, struct blk_mq_hw_ctx, run_work.work);
 2433 
 2434 	blk_mq_run_dispatch_ops(hctx->queue,
 2435 				blk_mq_sched_dispatch_requests(hctx));
 2436 }
 2437 
 2438 /**
 2439  * blk_mq_request_bypass_insert - Insert a request at dispatch list.
 2440  * @rq: Pointer to request to be inserted.
 2441  * @flags: BLK_MQ_INSERT_*
 2442  *
 2443  * Should only be used carefully, when the caller knows we want to
 2444  * bypass a potential IO scheduler on the target device.
 2445  */
 2446 static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags)
 2447 {
 2448 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 2449 
 2450 	spin_lock(&hctx->lock);
 2451 	if (flags & BLK_MQ_INSERT_AT_HEAD)
 2452 		list_add(&rq->queuelist, &hctx->dispatch);
 2453 	else
 2454 		list_add_tail(&rq->queuelist, &hctx->dispatch);
 2455 	spin_unlock(&hctx->lock);
 2456 }
 2457 
 2458 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
 2459 		struct blk_mq_ctx *ctx, struct list_head *list,
 2460 		bool run_queue_async)
 2461 {
 2462 	struct request *rq;
 2463 	enum hctx_type type = hctx->type;
 2464 
 2465 	/*
 2466 	 * Try to issue requests directly if the hw queue isn't busy to save an
 2467 	 * extra enqueue & dequeue to the sw queue.
 2468 	 */
 2469 	if (!hctx->dispatch_busy && !run_queue_async) {
 2470 		blk_mq_run_dispatch_ops(hctx->queue,
 2471 			blk_mq_try_issue_list_directly(hctx, list));
 2472 		if (list_empty(list))
 2473 			goto out;
 2474 	}
 2475 
 2476 	/*
 2477 	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
 2478 	 * offline now
 2479 	 */
 2480 	list_for_each_entry(rq, list, queuelist) {
 2481 		BUG_ON(rq->mq_ctx != ctx);
 2482 		trace_block_rq_insert(rq);
 2483 		if (rq->cmd_flags & REQ_NOWAIT)
 2484 			run_queue_async = true;
 2485 	}
 2486 
 2487 	spin_lock(&ctx->lock);
 2488 	list_splice_tail_init(list, &ctx->rq_lists[type]);
 2489 	blk_mq_hctx_mark_pending(hctx, ctx);
 2490 	spin_unlock(&ctx->lock);
 2491 out:
 2492 	blk_mq_run_hw_queue(hctx, run_queue_async);
 2493 }
 2494 
 2495 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)
 2496 {
 2497 	struct request_queue *q = rq->q;
 2498 	struct blk_mq_ctx *ctx = rq->mq_ctx;
 2499 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 2500 
 2501 	if (blk_rq_is_passthrough(rq)) {
 2502 		/*
 2503 		 * Passthrough request have to be added to hctx->dispatch
 2504 		 * directly.  The device may be in a situation where it can't
 2505 		 * handle FS request, and always returns BLK_STS_RESOURCE for
 2506 		 * them, which gets them added to hctx->dispatch.
 2507 		 *
 2508 		 * If a passthrough request is required to unblock the queues,
 2509 		 * and it is added to the scheduler queue, there is no chance to
 2510 		 * dispatch it given we prioritize requests in hctx->dispatch.
 2511 		 */
 2512 		blk_mq_request_bypass_insert(rq, flags);
 2513 	} else if (req_op(rq) == REQ_OP_FLUSH) {
 2514 		/*
 2515 		 * Firstly normal IO request is inserted to scheduler queue or
 2516 		 * sw queue, meantime we add flush request to dispatch queue(
 2517 		 * hctx->dispatch) directly and there is at most one in-flight
 2518 		 * flush request for each hw queue, so it doesn't matter to add
 2519 		 * flush request to tail or front of the dispatch queue.
 2520 		 *
 2521 		 * Secondly in case of NCQ, flush request belongs to non-NCQ
 2522 		 * command, and queueing it will fail when there is any
 2523 		 * in-flight normal IO request(NCQ command). When adding flush
 2524 		 * rq to the front of hctx->dispatch, it is easier to introduce
 2525 		 * extra time to flush rq's latency because of S_SCHED_RESTART
 2526 		 * compared with adding to the tail of dispatch queue, then
 2527 		 * chance of flush merge is increased, and less flush requests
 2528 		 * will be issued to controller. It is observed that ~10% time
 2529 		 * is saved in blktests block/004 on disk attached to AHCI/NCQ
 2530 		 * drive when adding flush rq to the front of hctx->dispatch.
 2531 		 *
 2532 		 * Simply queue flush rq to the front of hctx->dispatch so that
 2533 		 * intensive flush workloads can benefit in case of NCQ HW.
 2534 		 */
 2535 		blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD);
 2536 	} else if (q->elevator) {
 2537 		LIST_HEAD(list);
 2538 
 2539 		WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG);
 2540 
 2541 		list_add(&rq->queuelist, &list);
 2542 		q->elevator->type->ops.insert_requests(hctx, &list, flags);
 2543 	} else {
 2544 		trace_block_rq_insert(rq);
 2545 
 2546 		spin_lock(&ctx->lock);
 2547 		if (flags & BLK_MQ_INSERT_AT_HEAD)
 2548 			list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
 2549 		else
 2550 			list_add_tail(&rq->queuelist,
 2551 				      &ctx->rq_lists[hctx->type]);
 2552 		blk_mq_hctx_mark_pending(hctx, ctx);
 2553 		spin_unlock(&ctx->lock);
 2554 	}
 2555 }
 2556 
 2557 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
 2558 		unsigned int nr_segs)
 2559 {
 2560 	int err;
 2561 
 2562 	if (bio->bi_opf & REQ_RAHEAD)
 2563 		rq->cmd_flags |= REQ_FAILFAST_MASK;
 2564 
 2565 	rq->__sector = bio->bi_iter.bi_sector;
 2566 	blk_rq_bio_prep(rq, bio, nr_segs);
 2567 
 2568 	/* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
 2569 	err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
 2570 	WARN_ON_ONCE(err);
 2571 
 2572 	blk_account_io_start(rq);
 2573 }
 2574 
 2575 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
 2576 					    struct request *rq, bool last)
 2577 {
 2578 	struct request_queue *q = rq->q;
 2579 	struct blk_mq_queue_data bd = {
 2580 		.rq = rq,
 2581 		.last = last,
 2582 	};
 2583 	blk_status_t ret;
 2584 
 2585 	/*
 2586 	 * For OK queue, we are done. For error, caller may kill it.
 2587 	 * Any other error (busy), just add it to our list as we
 2588 	 * previously would have done.
 2589 	 */
 2590 	ret = q->mq_ops->queue_rq(hctx, &bd);
 2591 	switch (ret) {
 2592 	case BLK_STS_OK:
 2593 		blk_mq_update_dispatch_busy(hctx, false);
 2594 		break;
 2595 	case BLK_STS_RESOURCE:
 2596 	case BLK_STS_DEV_RESOURCE:
 2597 		blk_mq_update_dispatch_busy(hctx, true);
 2598 		__blk_mq_requeue_request(rq);
 2599 		break;
 2600 	default:
 2601 		blk_mq_update_dispatch_busy(hctx, false);
 2602 		break;
 2603 	}
 2604 
 2605 	return ret;
 2606 }
 2607 
 2608 static bool blk_mq_get_budget_and_tag(struct request *rq)
 2609 {
 2610 	int budget_token;
 2611 
 2612 	budget_token = blk_mq_get_dispatch_budget(rq->q);
 2613 	if (budget_token < 0)
 2614 		return false;
 2615 	blk_mq_set_rq_budget_token(rq, budget_token);
 2616 	if (!blk_mq_get_driver_tag(rq)) {
 2617 		blk_mq_put_dispatch_budget(rq->q, budget_token);
 2618 		return false;
 2619 	}
 2620 	return true;
 2621 }
 2622 
 2623 /**
 2624  * blk_mq_try_issue_directly - Try to send a request directly to device driver.
 2625  * @hctx: Pointer of the associated hardware queue.
 2626  * @rq: Pointer to request to be sent.
 2627  *
 2628  * If the device has enough resources to accept a new request now, send the
 2629  * request directly to device driver. Else, insert at hctx->dispatch queue, so
 2630  * we can try send it another time in the future. Requests inserted at this
 2631  * queue have higher priority.
 2632  */
 2633 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
 2634 		struct request *rq)
 2635 {
 2636 	blk_status_t ret;
 2637 
 2638 	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
 2639 		blk_mq_insert_request(rq, 0);
 2640 		return;
 2641 	}
 2642 
 2643 	if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) {
 2644 		blk_mq_insert_request(rq, 0);
 2645 		blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT);
 2646 		return;
 2647 	}
 2648 
 2649 	ret = __blk_mq_issue_directly(hctx, rq, true);
 2650 	switch (ret) {
 2651 	case BLK_STS_OK:
 2652 		break;
 2653 	case BLK_STS_RESOURCE:
 2654 	case BLK_STS_DEV_RESOURCE:
 2655 		blk_mq_request_bypass_insert(rq, 0);
 2656 		blk_mq_run_hw_queue(hctx, false);
 2657 		break;
 2658 	default:
 2659 		blk_mq_end_request(rq, ret);
 2660 		break;
 2661 	}
 2662 }
 2663 
 2664 static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
 2665 {
 2666 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 2667 
 2668 	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
 2669 		blk_mq_insert_request(rq, 0);
 2670 		return BLK_STS_OK;
 2671 	}
 2672 
 2673 	if (!blk_mq_get_budget_and_tag(rq))
 2674 		return BLK_STS_RESOURCE;
 2675 	return __blk_mq_issue_directly(hctx, rq, last);
 2676 }
 2677 
 2678 static void blk_mq_plug_issue_direct(struct blk_plug *plug)
 2679 {
 2680 	struct blk_mq_hw_ctx *hctx = NULL;
 2681 	struct request *rq;
 2682 	int queued = 0;
 2683 	blk_status_t ret = BLK_STS_OK;
 2684 
 2685 	while ((rq = rq_list_pop(&plug->mq_list))) {
 2686 		bool last = rq_list_empty(plug->mq_list);
 2687 
 2688 		if (hctx != rq->mq_hctx) {
 2689 			if (hctx) {
 2690 				blk_mq_commit_rqs(hctx, queued, false);
 2691 				queued = 0;
 2692 			}
 2693 			hctx = rq->mq_hctx;
 2694 		}
 2695 
 2696 		ret = blk_mq_request_issue_directly(rq, last);
 2697 		switch (ret) {
 2698 		case BLK_STS_OK:
 2699 			queued++;
 2700 			break;
 2701 		case BLK_STS_RESOURCE:
 2702 		case BLK_STS_DEV_RESOURCE:
 2703 			blk_mq_request_bypass_insert(rq, 0);
 2704 			blk_mq_run_hw_queue(hctx, false);
 2705 			goto out;
 2706 		default:
 2707 			blk_mq_end_request(rq, ret);
 2708 			break;
 2709 		}
 2710 	}
 2711 
 2712 out:
 2713 	if (ret != BLK_STS_OK)
 2714 		blk_mq_commit_rqs(hctx, queued, false);
 2715 }
 2716 
 2717 static void __blk_mq_flush_plug_list(struct request_queue *q,
 2718 				     struct blk_plug *plug)
 2719 {
 2720 	if (blk_queue_quiesced(q))
 2721 		return;
 2722 	q->mq_ops->queue_rqs(&plug->mq_list);
 2723 }
 2724 
 2725 static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched)
 2726 {
 2727 	struct blk_mq_hw_ctx *this_hctx = NULL;
 2728 	struct blk_mq_ctx *this_ctx = NULL;
 2729 	struct request *requeue_list = NULL;
 2730 	struct request **requeue_lastp = &requeue_list;
 2731 	unsigned int depth = 0;
 2732 	bool is_passthrough = false;
 2733 	LIST_HEAD(list);
 2734 
 2735 	do {
 2736 		struct request *rq = rq_list_pop(&plug->mq_list);
 2737 
 2738 		if (!this_hctx) {
 2739 			this_hctx = rq->mq_hctx;
 2740 			this_ctx = rq->mq_ctx;
 2741 			is_passthrough = blk_rq_is_passthrough(rq);
 2742 		} else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
 2743 			   is_passthrough != blk_rq_is_passthrough(rq)) {
 2744 			rq_list_add_tail(&requeue_lastp, rq);
 2745 			continue;
 2746 		}
 2747 		list_add(&rq->queuelist, &list);
 2748 		depth++;
 2749 	} while (!rq_list_empty(plug->mq_list));
 2750 
 2751 	plug->mq_list = requeue_list;
 2752 	trace_block_unplug(this_hctx->queue, depth, !from_sched);
 2753 
 2754 	percpu_ref_get(&this_hctx->queue->q_usage_counter);
 2755 	/* passthrough requests should never be issued to the I/O scheduler */
 2756 	if (is_passthrough) {
 2757 		spin_lock(&this_hctx->lock);
 2758 		list_splice_tail_init(&list, &this_hctx->dispatch);
 2759 		spin_unlock(&this_hctx->lock);
 2760 		blk_mq_run_hw_queue(this_hctx, from_sched);
 2761 	} else if (this_hctx->queue->elevator) {
 2762 		this_hctx->queue->elevator->type->ops.insert_requests(this_hctx,
 2763 				&list, 0);
 2764 		blk_mq_run_hw_queue(this_hctx, from_sched);
 2765 	} else {
 2766 		blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched);
 2767 	}
 2768 	percpu_ref_put(&this_hctx->queue->q_usage_counter);
 2769 }
 2770 
 2771 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 2772 {
 2773 	struct request *rq;
 2774 
 2775 	/*
 2776 	 * We may have been called recursively midway through handling
 2777 	 * plug->mq_list via a schedule() in the driver's queue_rq() callback.
 2778 	 * To avoid mq_list changing under our feet, clear rq_count early and
 2779 	 * bail out specifically if rq_count is 0 rather than checking
 2780 	 * whether the mq_list is empty.
 2781 	 */
 2782 	if (plug->rq_count == 0)
 2783 		return;
 2784 	plug->rq_count = 0;
 2785 
 2786 	if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) {
 2787 		struct request_queue *q;
 2788 
 2789 		rq = rq_list_peek(&plug->mq_list);
 2790 		q = rq->q;
 2791 
 2792 		/*
 2793 		 * Peek first request and see if we have a ->queue_rqs() hook.
 2794 		 * If we do, we can dispatch the whole plug list in one go. We
 2795 		 * already know at this point that all requests belong to the
 2796 		 * same queue, caller must ensure that's the case.
 2797 		 *
 2798 		 * Since we pass off the full list to the driver at this point,
 2799 		 * we do not increment the active request count for the queue.
 2800 		 * Bypass shared tags for now because of that.
 2801 		 */
 2802 		if (q->mq_ops->queue_rqs &&
 2803 		    !(rq->mq_hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
 2804 			blk_mq_run_dispatch_ops(q,
 2805 				__blk_mq_flush_plug_list(q, plug));
 2806 			if (rq_list_empty(plug->mq_list))
 2807 				return;
 2808 		}
 2809 
 2810 		blk_mq_run_dispatch_ops(q,
 2811 				blk_mq_plug_issue_direct(plug));
 2812 		if (rq_list_empty(plug->mq_list))
 2813 			return;
 2814 	}
 2815 
 2816 	do {
 2817 		blk_mq_dispatch_plug_list(plug, from_schedule);
 2818 	} while (!rq_list_empty(plug->mq_list));
 2819 }
 2820 
 2821 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
 2822 		struct list_head *list)
 2823 {
 2824 	int queued = 0;
 2825 	blk_status_t ret = BLK_STS_OK;
 2826 
 2827 	while (!list_empty(list)) {
 2828 		struct request *rq = list_first_entry(list, struct request,
 2829 				queuelist);
 2830 
 2831 		list_del_init(&rq->queuelist);
 2832 		ret = blk_mq_request_issue_directly(rq, list_empty(list));
 2833 		switch (ret) {
 2834 		case BLK_STS_OK:
 2835 			queued++;
 2836 			break;
 2837 		case BLK_STS_RESOURCE:
 2838 		case BLK_STS_DEV_RESOURCE:
 2839 			blk_mq_request_bypass_insert(rq, 0);
 2840 			if (list_empty(list))
 2841 				blk_mq_run_hw_queue(hctx, false);
 2842 			goto out;
 2843 		default:
 2844 			blk_mq_end_request(rq, ret);
 2845 			break;
 2846 		}
 2847 	}
 2848 
 2849 out:
 2850 	if (ret != BLK_STS_OK)
 2851 		blk_mq_commit_rqs(hctx, queued, false);
 2852 }
 2853 
 2854 static bool blk_mq_attempt_bio_merge(struct request_queue *q,
 2855 				     struct bio *bio, unsigned int nr_segs)
 2856 {
 2857 	if (!blk_queue_nomerges(q) && bio_mergeable(bio)) {
 2858 		if (blk_attempt_plug_merge(q, bio, nr_segs))
 2859 			return true;
 2860 		if (blk_mq_sched_bio_merge(q, bio, nr_segs))
 2861 			return true;
 2862 	}
 2863 	return false;
 2864 }
 2865 
 2866 static struct request *blk_mq_get_new_requests(struct request_queue *q,
 2867 					       struct blk_plug *plug,
 2868 					       struct bio *bio,
 2869 					       unsigned int nsegs)
 2870 {
 2871 	struct blk_mq_alloc_data data = {
 2872 		.q		= q,
 2873 		.nr_tags	= 1,
 2874 		.cmd_flags	= bio->bi_opf,
 2875 	};
 2876 	struct request *rq;
 2877 
 2878 	if (unlikely(bio_queue_enter(bio)))
 2879 		return NULL;
 2880 
 2881 	if (blk_mq_attempt_bio_merge(q, bio, nsegs))
 2882 		goto queue_exit;
 2883 
 2884 	rq_qos_throttle(q, bio);
 2885 
 2886 	if (plug) {
 2887 		data.nr_tags = plug->nr_ios;
 2888 		plug->nr_ios = 1;
 2889 		data.cached_rq = &plug->cached_rq;
 2890 	}
 2891 
 2892 	rq = __blk_mq_alloc_requests(&data);
 2893 	if (rq)
 2894 		return rq;
 2895 	rq_qos_cleanup(q, bio);
 2896 	if (bio->bi_opf & REQ_NOWAIT)
 2897 		bio_wouldblock_error(bio);
 2898 queue_exit:
 2899 	blk_queue_exit(q);
 2900 	return NULL;
 2901 }
 2902 
 2903 static inline struct request *blk_mq_get_cached_request(struct request_queue *q,
 2904 		struct blk_plug *plug, struct bio **bio, unsigned int nsegs)
 2905 {
 2906 	struct request *rq;
 2907 	enum hctx_type type, hctx_type;
 2908 
 2909 	if (!plug)
 2910 		return NULL;
 2911 	rq = rq_list_peek(&plug->cached_rq);
 2912 	if (!rq || rq->q != q)
 2913 		return NULL;
 2914 
 2915 	if (blk_mq_attempt_bio_merge(q, *bio, nsegs)) {
 2916 		*bio = NULL;
 2917 		return NULL;
 2918 	}
 2919 
 2920 	type = blk_mq_get_hctx_type((*bio)->bi_opf);
 2921 	hctx_type = rq->mq_hctx->type;
 2922 	if (type != hctx_type &&
 2923 	    !(type == HCTX_TYPE_READ && hctx_type == HCTX_TYPE_DEFAULT))
 2924 		return NULL;
 2925 	if (op_is_flush(rq->cmd_flags) != op_is_flush((*bio)->bi_opf))
 2926 		return NULL;
 2927 
 2928 	/*
 2929 	 * If any qos ->throttle() end up blocking, we will have flushed the
 2930 	 * plug and hence killed the cached_rq list as well. Pop this entry
 2931 	 * before we throttle.
 2932 	 */
 2933 	plug->cached_rq = rq_list_next(rq);
 2934 	rq_qos_throttle(q, *bio);
 2935 
 2936 	blk_mq_rq_time_init(rq, 0);
 2937 	rq->cmd_flags = (*bio)->bi_opf;
 2938 	INIT_LIST_HEAD(&rq->queuelist);
 2939 	return rq;
 2940 }
 2941 
 2942 static void bio_set_ioprio(struct bio *bio)
 2943 {
 2944 	/* Nobody set ioprio so far? Initialize it based on task's nice value */
 2945 	if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE)
 2946 		bio->bi_ioprio = get_current_ioprio();
 2947 	blkcg_set_ioprio(bio);
 2948 }
 2949 
 2950 /**
 2951  * blk_mq_submit_bio - Create and send a request to block device.
 2952  * @bio: Bio pointer.
 2953  *
 2954  * Builds up a request structure from @q and @bio and send to the device. The
 2955  * request may not be queued directly to hardware if:
 2956  * * This request can be merged with another one
 2957  * * We want to place request at plug queue for possible future merging
 2958  * * There is an IO scheduler active at this queue
 2959  *
 2960  * It will not queue the request if there is an error with the bio, or at the
 2961  * request creation.
 2962  */
 2963 void blk_mq_submit_bio(struct bio *bio)
 2964 {
 2965 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
 2966 	struct blk_plug *plug = blk_mq_plug(bio);
 2967 	const int is_sync = op_is_sync(bio->bi_opf);
 2968 	struct blk_mq_hw_ctx *hctx;
 2969 	struct request *rq;
 2970 	unsigned int nr_segs = 1;
 2971 	blk_status_t ret;
 2972 
 2973 	bio = blk_queue_bounce(bio, q);
 2974 	if (bio_may_exceed_limits(bio, &q->limits)) {
 2975 		bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
 2976 		if (!bio)
 2977 			return;
 2978 	}
 2979 
 2980 	if (!bio_integrity_prep(bio))
 2981 		return;
 2982 
 2983 	bio_set_ioprio(bio);
 2984 
 2985 	rq = blk_mq_get_cached_request(q, plug, &bio, nr_segs);
 2986 	if (!rq) {
 2987 		if (!bio)
 2988 			return;
 2989 		rq = blk_mq_get_new_requests(q, plug, bio, nr_segs);
 2990 		if (unlikely(!rq))
 2991 			return;
 2992 	}
 2993 
 2994 	trace_block_getrq(bio);
 2995 
 2996 	rq_qos_track(q, rq, bio);
 2997 
 2998 	blk_mq_bio_to_request(rq, bio, nr_segs);
 2999 
 3000 	ret = blk_crypto_rq_get_keyslot(rq);
 3001 	if (ret != BLK_STS_OK) {
 3002 		bio->bi_status = ret;
 3003 		bio_endio(bio);
 3004 		blk_mq_free_request(rq);
 3005 		return;
 3006 	}
 3007 
 3008 	if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
 3009 		return;
 3010 
 3011 	if (plug) {
 3012 		blk_add_rq_to_plug(plug, rq);
 3013 		return;
 3014 	}
 3015 
 3016 	hctx = rq->mq_hctx;
 3017 	if ((rq->rq_flags & RQF_USE_SCHED) ||
 3018 	    (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) {
 3019 		blk_mq_insert_request(rq, 0);
 3020 		blk_mq_run_hw_queue(hctx, true);
 3021 	} else {
 3022 		blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq));
 3023 	}
 3024 }
 3025 
 3026 #ifdef CONFIG_BLK_MQ_STACKING
 3027 /**
 3028  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
 3029  * @rq: the request being queued
 3030  */
 3031 blk_status_t blk_insert_cloned_request(struct request *rq)
 3032 {
 3033 	struct request_queue *q = rq->q;
 3034 	unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
 3035 	unsigned int max_segments = blk_rq_get_max_segments(rq);
 3036 	blk_status_t ret;
 3037 
 3038 	if (blk_rq_sectors(rq) > max_sectors) {
 3039 		/*
 3040 		 * SCSI device does not have a good way to return if
 3041 		 * Write Same/Zero is actually supported. If a device rejects
 3042 		 * a non-read/write command (discard, write same,etc.) the
 3043 		 * low-level device driver will set the relevant queue limit to
 3044 		 * 0 to prevent blk-lib from issuing more of the offending
 3045 		 * operations. Commands queued prior to the queue limit being
 3046 		 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
 3047 		 * errors being propagated to upper layers.
 3048 		 */
 3049 		if (max_sectors == 0)
 3050 			return BLK_STS_NOTSUPP;
 3051 
 3052 		printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
 3053 			__func__, blk_rq_sectors(rq), max_sectors);
 3054 		return BLK_STS_IOERR;
 3055 	}
 3056 
 3057 	/*
 3058 	 * The queue settings related to segment counting may differ from the
 3059 	 * original queue.
 3060 	 */
 3061 	rq->nr_phys_segments = blk_recalc_rq_segments(rq);
 3062 	if (rq->nr_phys_segments > max_segments) {
 3063 		printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n",
 3064 			__func__, rq->nr_phys_segments, max_segments);
 3065 		return BLK_STS_IOERR;
 3066 	}
 3067 
 3068 	if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
 3069 		return BLK_STS_IOERR;
 3070 
 3071 	ret = blk_crypto_rq_get_keyslot(rq);
 3072 	if (ret != BLK_STS_OK)
 3073 		return ret;
 3074 
 3075 	blk_account_io_start(rq);
 3076 
 3077 	/*
 3078 	 * Since we have a scheduler attached on the top device,
 3079 	 * bypass a potential scheduler on the bottom device for
 3080 	 * insert.
 3081 	 */
 3082 	blk_mq_run_dispatch_ops(q,
 3083 			ret = blk_mq_request_issue_directly(rq, true));
 3084 	if (ret)
 3085 		blk_account_io_done(rq, ktime_get_ns());
 3086 	return ret;
 3087 }
 3088 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
 3089 
 3090 /**
 3091  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
 3092  * @rq: the clone request to be cleaned up
 3093  *
 3094  * Description:
 3095  *     Free all bios in @rq for a cloned request.
 3096  */
 3097 void blk_rq_unprep_clone(struct request *rq)
 3098 {
 3099 	struct bio *bio;
 3100 
 3101 	while ((bio = rq->bio) != NULL) {
 3102 		rq->bio = bio->bi_next;
 3103 
 3104 		bio_put(bio);
 3105 	}
 3106 }
 3107 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
 3108 
 3109 /**
 3110  * blk_rq_prep_clone - Helper function to setup clone request
 3111  * @rq: the request to be setup
 3112  * @rq_src: original request to be cloned
 3113  * @bs: bio_set that bios for clone are allocated from
 3114  * @gfp_mask: memory allocation mask for bio
 3115  * @bio_ctr: setup function to be called for each clone bio.
 3116  *           Returns %0 for success, non %0 for failure.
 3117  * @data: private data to be passed to @bio_ctr
 3118  *
 3119  * Description:
 3120  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
 3121  *     Also, pages which the original bios are pointing to are not copied
 3122  *     and the cloned bios just point same pages.
 3123  *     So cloned bios must be completed before original bios, which means
 3124  *     the caller must complete @rq before @rq_src.
 3125  */
 3126 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
 3127 		      struct bio_set *bs, gfp_t gfp_mask,
 3128 		      int (*bio_ctr)(struct bio *, struct bio *, void *),
 3129 		      void *data)
 3130 {
 3131 	struct bio *bio, *bio_src;
 3132 
 3133 	if (!bs)
 3134 		bs = &fs_bio_set;
 3135 
 3136 	__rq_for_each_bio(bio_src, rq_src) {
 3137 		bio = bio_alloc_clone(rq->q->disk->part0, bio_src, gfp_mask,
 3138 				      bs);
 3139 		if (!bio)
 3140 			goto free_and_out;
 3141 
 3142 		if (bio_ctr && bio_ctr(bio, bio_src, data))
 3143 			goto free_and_out;
 3144 
 3145 		if (rq->bio) {
 3146 			rq->biotail->bi_next = bio;
 3147 			rq->biotail = bio;
 3148 		} else {
 3149 			rq->bio = rq->biotail = bio;
 3150 		}
 3151 		bio = NULL;
 3152 	}
 3153 
 3154 	/* Copy attributes of the original request to the clone request. */
 3155 	rq->__sector = blk_rq_pos(rq_src);
 3156 	rq->__data_len = blk_rq_bytes(rq_src);
 3157 	if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
 3158 		rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
 3159 		rq->special_vec = rq_src->special_vec;
 3160 	}
 3161 	rq->nr_phys_segments = rq_src->nr_phys_segments;
 3162 	rq->ioprio = rq_src->ioprio;
 3163 
 3164 	if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
 3165 		goto free_and_out;
 3166 
 3167 	return 0;
 3168 
 3169 free_and_out:
 3170 	if (bio)
 3171 		bio_put(bio);
 3172 	blk_rq_unprep_clone(rq);
 3173 
 3174 	return -ENOMEM;
 3175 }
 3176 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
 3177 #endif /* CONFIG_BLK_MQ_STACKING */
 3178 
 3179 /*
 3180  * Steal bios from a request and add them to a bio list.
 3181  * The request must not have been partially completed before.
 3182  */
 3183 void blk_steal_bios(struct bio_list *list, struct request *rq)
 3184 {
 3185 	if (rq->bio) {
 3186 		if (list->tail)
 3187 			list->tail->bi_next = rq->bio;
 3188 		else
 3189 			list->head = rq->bio;
 3190 		list->tail = rq->biotail;
 3191 
 3192 		rq->bio = NULL;
 3193 		rq->biotail = NULL;
 3194 	}
 3195 
 3196 	rq->__data_len = 0;
 3197 }
 3198 EXPORT_SYMBOL_GPL(blk_steal_bios);
 3199 
 3200 static size_t order_to_size(unsigned int order)
 3201 {
 3202 	return (size_t)PAGE_SIZE << order;
 3203 }
 3204 
 3205 /* called before freeing request pool in @tags */
 3206 static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
 3207 				    struct blk_mq_tags *tags)
 3208 {
 3209 	struct page *page;
 3210 	unsigned long flags;
 3211 
 3212 	/*
 3213 	 * There is no need to clear mapping if driver tags is not initialized
 3214 	 * or the mapping belongs to the driver tags.
 3215 	 */
 3216 	if (!drv_tags || drv_tags == tags)
 3217 		return;
 3218 
 3219 	list_for_each_entry(page, &tags->page_list, lru) {
 3220 		unsigned long start = (unsigned long)page_address(page);
 3221 		unsigned long end = start + order_to_size(page->private);
 3222 		int i;
 3223 
 3224 		for (i = 0; i < drv_tags->nr_tags; i++) {
 3225 			struct request *rq = drv_tags->rqs[i];
 3226 			unsigned long rq_addr = (unsigned long)rq;
 3227 
 3228 			if (rq_addr >= start && rq_addr < end) {
 3229 				WARN_ON_ONCE(req_ref_read(rq) != 0);
 3230 				cmpxchg(&drv_tags->rqs[i], rq, NULL);
 3231 			}
 3232 		}
 3233 	}
 3234 
 3235 	/*
 3236 	 * Wait until all pending iteration is done.
 3237 	 *
 3238 	 * Request reference is cleared and it is guaranteed to be observed
 3239 	 * after the ->lock is released.
 3240 	 */
 3241 	spin_lock_irqsave(&drv_tags->lock, flags);
 3242 	spin_unlock_irqrestore(&drv_tags->lock, flags);
 3243 }
 3244 
 3245 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
 3246 		     unsigned int hctx_idx)
 3247 {
 3248 	struct blk_mq_tags *drv_tags;
 3249 	struct page *page;
 3250 
 3251 	if (list_empty(&tags->page_list))
 3252 		return;
 3253 
 3254 	if (blk_mq_is_shared_tags(set->flags))
 3255 		drv_tags = set->shared_tags;
 3256 	else
 3257 		drv_tags = set->tags[hctx_idx];
 3258 
 3259 	if (tags->static_rqs && set->ops->exit_request) {
 3260 		int i;
 3261 
 3262 		for (i = 0; i < tags->nr_tags; i++) {
 3263 			struct request *rq = tags->static_rqs[i];
 3264 
 3265 			if (!rq)
 3266 				continue;
 3267 			set->ops->exit_request(set, rq, hctx_idx);
 3268 			tags->static_rqs[i] = NULL;
 3269 		}
 3270 	}
 3271 
 3272 	blk_mq_clear_rq_mapping(drv_tags, tags);
 3273 
 3274 	while (!list_empty(&tags->page_list)) {
 3275 		page = list_first_entry(&tags->page_list, struct page, lru);
 3276 		list_del_init(&page->lru);
 3277 		/*
 3278 		 * Remove kmemleak object previously allocated in
 3279 		 * blk_mq_alloc_rqs().
 3280 		 */
 3281 		kmemleak_free(page_address(page));
 3282 		__free_pages(page, page->private);
 3283 	}
 3284 }
 3285 
 3286 void blk_mq_free_rq_map(struct blk_mq_tags *tags)
 3287 {
 3288 	kfree(tags->rqs);
 3289 	tags->rqs = NULL;
 3290 	kfree(tags->static_rqs);
 3291 	tags->static_rqs = NULL;
 3292 
 3293 	blk_mq_free_tags(tags);
 3294 }
 3295 
 3296 static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
 3297 		unsigned int hctx_idx)
 3298 {
 3299 	int i;
 3300 
 3301 	for (i = 0; i < set->nr_maps; i++) {
 3302 		unsigned int start = set->map[i].queue_offset;
 3303 		unsigned int end = start + set->map[i].nr_queues;
 3304 
 3305 		if (hctx_idx >= start && hctx_idx < end)
 3306 			break;
 3307 	}
 3308 
 3309 	if (i >= set->nr_maps)
 3310 		i = HCTX_TYPE_DEFAULT;
 3311 
 3312 	return i;
 3313 }
 3314 
 3315 static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set,
 3316 		unsigned int hctx_idx)
 3317 {
 3318 	enum hctx_type type = hctx_idx_to_type(set, hctx_idx);
 3319 
 3320 	return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx);
 3321 }
 3322 
 3323 static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
 3324 					       unsigned int hctx_idx,
 3325 					       unsigned int nr_tags,
 3326 					       unsigned int reserved_tags)
 3327 {
 3328 	int node = blk_mq_get_hctx_node(set, hctx_idx);
 3329 	struct blk_mq_tags *tags;
 3330 
 3331 	if (node == NUMA_NO_NODE)
 3332 		node = set->numa_node;
 3333 
 3334 	tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
 3335 				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
 3336 	if (!tags)
 3337 		return NULL;
 3338 
 3339 	tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
 3340 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
 3341 				 node);
 3342 	if (!tags->rqs)
 3343 		goto err_free_tags;
 3344 
 3345 	tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
 3346 					GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
 3347 					node);
 3348 	if (!tags->static_rqs)
 3349 		goto err_free_rqs;
 3350 
 3351 	return tags;
 3352 
 3353 err_free_rqs:
 3354 	kfree(tags->rqs);
 3355 err_free_tags:
 3356 	blk_mq_free_tags(tags);
 3357 	return NULL;
 3358 }
 3359 
 3360 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
 3361 			       unsigned int hctx_idx, int node)
 3362 {
 3363 	int ret;
 3364 
 3365 	if (set->ops->init_request) {
 3366 		ret = set->ops->init_request(set, rq, hctx_idx, node);
 3367 		if (ret)
 3368 			return ret;
 3369 	}
 3370 
 3371 	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
 3372 	return 0;
 3373 }
 3374 
 3375 static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
 3376 			    struct blk_mq_tags *tags,
 3377 			    unsigned int hctx_idx, unsigned int depth)
 3378 {
 3379 	unsigned int i, j, entries_per_page, max_order = 4;
 3380 	int node = blk_mq_get_hctx_node(set, hctx_idx);
 3381 	size_t rq_size, left;
 3382 
 3383 	if (node == NUMA_NO_NODE)
 3384 		node = set->numa_node;
 3385 
 3386 	INIT_LIST_HEAD(&tags->page_list);
 3387 
 3388 	/*
 3389 	 * rq_size is the size of the request plus driver payload, rounded
 3390 	 * to the cacheline size
 3391 	 */
 3392 	rq_size = round_up(sizeof(struct request) + set->cmd_size,
 3393 				cache_line_size());
 3394 	left = rq_size * depth;
 3395 
 3396 	for (i = 0; i < depth; ) {
 3397 		int this_order = max_order;
 3398 		struct page *page;
 3399 		int to_do;
 3400 		void *p;
 3401 
 3402 		while (this_order && left < order_to_size(this_order - 1))
 3403 			this_order--;
 3404 
 3405 		do {
 3406 			page = alloc_pages_node(node,
 3407 				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
 3408 				this_order);
 3409 			if (page)
 3410 				break;
 3411 			if (!this_order--)
 3412 				break;
 3413 			if (order_to_size(this_order) < rq_size)
 3414 				break;
 3415 		} while (1);
 3416 
 3417 		if (!page)
 3418 			goto fail;
 3419 
 3420 		page->private = this_order;
 3421 		list_add_tail(&page->lru, &tags->page_list);
 3422 
 3423 		p = page_address(page);
 3424 		/*
 3425 		 * Allow kmemleak to scan these pages as they contain pointers
 3426 		 * to additional allocations like via ops->init_request().
 3427 		 */
 3428 		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
 3429 		entries_per_page = order_to_size(this_order) / rq_size;
 3430 		to_do = min(entries_per_page, depth - i);
 3431 		left -= to_do * rq_size;
 3432 		for (j = 0; j < to_do; j++) {
 3433 			struct request *rq = p;
 3434 
 3435 			tags->static_rqs[i] = rq;
 3436 			if (blk_mq_init_request(set, rq, hctx_idx, node)) {
 3437 				tags->static_rqs[i] = NULL;
 3438 				goto fail;
 3439 			}
 3440 
 3441 			p += rq_size;
 3442 			i++;
 3443 		}
 3444 	}
 3445 	return 0;
 3446 
 3447 fail:
 3448 	blk_mq_free_rqs(set, tags, hctx_idx);
 3449 	return -ENOMEM;
 3450 }
 3451 
 3452 struct rq_iter_data {
 3453 	struct blk_mq_hw_ctx *hctx;
 3454 	bool has_rq;
 3455 };
 3456 
 3457 static bool blk_mq_has_request(struct request *rq, void *data)
 3458 {
 3459 	struct rq_iter_data *iter_data = data;
 3460 
 3461 	if (rq->mq_hctx != iter_data->hctx)
 3462 		return true;
 3463 	iter_data->has_rq = true;
 3464 	return false;
 3465 }
 3466 
 3467 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
 3468 {
 3469 	struct blk_mq_tags *tags = hctx->sched_tags ?
 3470 			hctx->sched_tags : hctx->tags;
 3471 	struct rq_iter_data data = {
 3472 		.hctx	= hctx,
 3473 	};
 3474 
 3475 	blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
 3476 	return data.has_rq;
 3477 }
 3478 
 3479 static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
 3480 		struct blk_mq_hw_ctx *hctx)
 3481 {
 3482 	if (cpumask_first_and(hctx->cpumask, cpu_online_mask) != cpu)
 3483 		return false;
 3484 	if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
 3485 		return false;
 3486 	return true;
 3487 }
 3488 
 3489 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
 3490 {
 3491 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
 3492 			struct blk_mq_hw_ctx, cpuhp_online);
 3493 
 3494 	if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
 3495 	    !blk_mq_last_cpu_in_hctx(cpu, hctx))
 3496 		return 0;
 3497 
 3498 	/*
 3499 	 * Prevent new request from being allocated on the current hctx.
 3500 	 *
 3501 	 * The smp_mb__after_atomic() Pairs with the implied barrier in
 3502 	 * test_and_set_bit_lock in sbitmap_get().  Ensures the inactive flag is
 3503 	 * seen once we return from the tag allocator.
 3504 	 */
 3505 	set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
 3506 	smp_mb__after_atomic();
 3507 
 3508 	/*
 3509 	 * Try to grab a reference to the queue and wait for any outstanding
 3510 	 * requests.  If we could not grab a reference the queue has been
 3511 	 * frozen and there are no requests.
 3512 	 */
 3513 	if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
 3514 		while (blk_mq_hctx_has_requests(hctx))
 3515 			msleep(5);
 3516 		percpu_ref_put(&hctx->queue->q_usage_counter);
 3517 	}
 3518 
 3519 	return 0;
 3520 }
 3521 
 3522 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
 3523 {
 3524 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
 3525 			struct blk_mq_hw_ctx, cpuhp_online);
 3526 
 3527 	if (cpumask_test_cpu(cpu, hctx->cpumask))
 3528 		clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
 3529 	return 0;
 3530 }
 3531 
 3532 /*
 3533  * 'cpu' is going away. splice any existing rq_list entries from this
 3534  * software queue to the hw queue dispatch list, and ensure that it
 3535  * gets run.
 3536  */
 3537 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
 3538 {
 3539 	struct blk_mq_hw_ctx *hctx;
 3540 	struct blk_mq_ctx *ctx;
 3541 	LIST_HEAD(tmp);
 3542 	enum hctx_type type;
 3543 
 3544 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
 3545 	if (!cpumask_test_cpu(cpu, hctx->cpumask))
 3546 		return 0;
 3547 
 3548 	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
 3549 	type = hctx->type;
 3550 
 3551 	spin_lock(&ctx->lock);
 3552 	if (!list_empty(&ctx->rq_lists[type])) {
 3553 		list_splice_init(&ctx->rq_lists[type], &tmp);
 3554 		blk_mq_hctx_clear_pending(hctx, ctx);
 3555 	}
 3556 	spin_unlock(&ctx->lock);
 3557 
 3558 	if (list_empty(&tmp))
 3559 		return 0;
 3560 
 3561 	spin_lock(&hctx->lock);
 3562 	list_splice_tail_init(&tmp, &hctx->dispatch);
 3563 	spin_unlock(&hctx->lock);
 3564 
 3565 	blk_mq_run_hw_queue(hctx, true);
 3566 	return 0;
 3567 }
 3568 
 3569 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
 3570 {
 3571 	if (!(hctx->flags & BLK_MQ_F_STACKING))
 3572 		cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
 3573 						    &hctx->cpuhp_online);
 3574 	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
 3575 					    &hctx->cpuhp_dead);
 3576 }
 3577 
 3578 /*
 3579  * Before freeing hw queue, clearing the flush request reference in
 3580  * tags->rqs[] for avoiding potential UAF.
 3581  */
 3582 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
 3583 		unsigned int queue_depth, struct request *flush_rq)
 3584 {
 3585 	int i;
 3586 	unsigned long flags;
 3587 
 3588 	/* The hw queue may not be mapped yet */
 3589 	if (!tags)
 3590 		return;
 3591 
 3592 	WARN_ON_ONCE(req_ref_read(flush_rq) != 0);
 3593 
 3594 	for (i = 0; i < queue_depth; i++)
 3595 		cmpxchg(&tags->rqs[i], flush_rq, NULL);
 3596 
 3597 	/*
 3598 	 * Wait until all pending iteration is done.
 3599 	 *
 3600 	 * Request reference is cleared and it is guaranteed to be observed
 3601 	 * after the ->lock is released.
 3602 	 */
 3603 	spin_lock_irqsave(&tags->lock, flags);
 3604 	spin_unlock_irqrestore(&tags->lock, flags);
 3605 }
 3606 
 3607 /* hctx->ctxs will be freed in queue's release handler */
 3608 static void blk_mq_exit_hctx(struct request_queue *q,
 3609 		struct blk_mq_tag_set *set,
 3610 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 3611 {
 3612 	struct request *flush_rq = hctx->fq->flush_rq;
 3613 
 3614 	if (blk_mq_hw_queue_mapped(hctx))
 3615 		blk_mq_tag_idle(hctx);
 3616 
 3617 	if (blk_queue_init_done(q))
 3618 		blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
 3619 				set->queue_depth, flush_rq);
 3620 	if (set->ops->exit_request)
 3621 		set->ops->exit_request(set, flush_rq, hctx_idx);
 3622 
 3623 	if (set->ops->exit_hctx)
 3624 		set->ops->exit_hctx(hctx, hctx_idx);
 3625 
 3626 	blk_mq_remove_cpuhp(hctx);
 3627 
 3628 	xa_erase(&q->hctx_table, hctx_idx);
 3629 
 3630 	spin_lock(&q->unused_hctx_lock);
 3631 	list_add(&hctx->hctx_list, &q->unused_hctx_list);
 3632 	spin_unlock(&q->unused_hctx_lock);
 3633 }
 3634 
 3635 static void blk_mq_exit_hw_queues(struct request_queue *q,
 3636 		struct blk_mq_tag_set *set, int nr_queue)
 3637 {
 3638 	struct blk_mq_hw_ctx *hctx;
 3639 	unsigned long i;
 3640 
 3641 	queue_for_each_hw_ctx(q, hctx, i) {
 3642 		if (i == nr_queue)
 3643 			break;
 3644 		blk_mq_exit_hctx(q, set, hctx, i);
 3645 	}
 3646 }
 3647 
 3648 static int blk_mq_init_hctx(struct request_queue *q,
 3649 		struct blk_mq_tag_set *set,
 3650 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
 3651 {
 3652 	hctx->queue_num = hctx_idx;
 3653 
 3654 	if (!(hctx->flags & BLK_MQ_F_STACKING))
 3655 		cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
 3656 				&hctx->cpuhp_online);
 3657 	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
 3658 
 3659 	hctx->tags = set->tags[hctx_idx];
 3660 
 3661 	if (set->ops->init_hctx &&
 3662 	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
 3663 		goto unregister_cpu_notifier;
 3664 
 3665 	if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
 3666 				hctx->numa_node))
 3667 		goto exit_hctx;
 3668 
 3669 	if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL))
 3670 		goto exit_flush_rq;
 3671 
 3672 	return 0;
 3673 
 3674  exit_flush_rq:
 3675 	if (set->ops->exit_request)
 3676 		set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
 3677  exit_hctx:
 3678 	if (set->ops->exit_hctx)
 3679 		set->ops->exit_hctx(hctx, hctx_idx);
 3680  unregister_cpu_notifier:
 3681 	blk_mq_remove_cpuhp(hctx);
 3682 	return -1;
 3683 }
 3684 
 3685 static struct blk_mq_hw_ctx *
 3686 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
 3687 		int node)
 3688 {
 3689 	struct blk_mq_hw_ctx *hctx;
 3690 	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
 3691 
 3692 	hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);
 3693 	if (!hctx)
 3694 		goto fail_alloc_hctx;
 3695 
 3696 	if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
 3697 		goto free_hctx;
 3698 
 3699 	atomic_set(&hctx->nr_active, 0);
 3700 	if (node == NUMA_NO_NODE)
 3701 		node = set->numa_node;
 3702 	hctx->numa_node = node;
 3703 
 3704 	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
 3705 	spin_lock_init(&hctx->lock);
 3706 	INIT_LIST_HEAD(&hctx->dispatch);
 3707 	hctx->queue = q;
 3708 	hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
 3709 
 3710 	INIT_LIST_HEAD(&hctx->hctx_list);
 3711 
 3712 	/*
 3713 	 * Allocate space for all possible cpus to avoid allocation at
 3714 	 * runtime
 3715 	 */
 3716 	hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
 3717 			gfp, node);
 3718 	if (!hctx->ctxs)
 3719 		goto free_cpumask;
 3720 
 3721 	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
 3722 				gfp, node, false, false))
 3723 		goto free_ctxs;
 3724 	hctx->nr_ctx = 0;
 3725 
 3726 	spin_lock_init(&hctx->dispatch_wait_lock);
 3727 	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
 3728 	INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
 3729 
 3730 	hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
 3731 	if (!hctx->fq)
 3732 		goto free_bitmap;
 3733 
 3734 	blk_mq_hctx_kobj_init(hctx);
 3735 
 3736 	return hctx;
 3737 
 3738  free_bitmap:
 3739 	sbitmap_free(&hctx->ctx_map);
 3740  free_ctxs:
 3741 	kfree(hctx->ctxs);
 3742  free_cpumask:
 3743 	free_cpumask_var(hctx->cpumask);
 3744  free_hctx:
 3745 	kfree(hctx);
 3746  fail_alloc_hctx:
 3747 	return NULL;
 3748 }
 3749 
 3750 static void blk_mq_init_cpu_queues(struct request_queue *q,
 3751 				   unsigned int nr_hw_queues)
 3752 {
 3753 	struct blk_mq_tag_set *set = q->tag_set;
 3754 	unsigned int i, j;
 3755 
 3756 	for_each_possible_cpu(i) {
 3757 		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
 3758 		struct blk_mq_hw_ctx *hctx;
 3759 		int k;
 3760 
 3761 		__ctx->cpu = i;
 3762 		spin_lock_init(&__ctx->lock);
 3763 		for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
 3764 			INIT_LIST_HEAD(&__ctx->rq_lists[k]);
 3765 
 3766 		__ctx->queue = q;
 3767 
 3768 		/*
 3769 		 * Set local node, IFF we have more than one hw queue. If
 3770 		 * not, we remain on the home node of the device
 3771 		 */
 3772 		for (j = 0; j < set->nr_maps; j++) {
 3773 			hctx = blk_mq_map_queue_type(q, j, i);
 3774 			if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
 3775 				hctx->numa_node = cpu_to_node(i);
 3776 		}
 3777 	}
 3778 }
 3779 
 3780 struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
 3781 					     unsigned int hctx_idx,
 3782 					     unsigned int depth)
 3783 {
 3784 	struct blk_mq_tags *tags;
 3785 	int ret;
 3786 
 3787 	tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
 3788 	if (!tags)
 3789 		return NULL;
 3790 
 3791 	ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
 3792 	if (ret) {
 3793 		blk_mq_free_rq_map(tags);
 3794 		return NULL;
 3795 	}
 3796 
 3797 	return tags;
 3798 }
 3799 
 3800 static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
 3801 				       int hctx_idx)
 3802 {
 3803 	if (blk_mq_is_shared_tags(set->flags)) {
 3804 		set->tags[hctx_idx] = set->shared_tags;
 3805 
 3806 		return true;
 3807 	}
 3808 
 3809 	set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
 3810 						       set->queue_depth);
 3811 
 3812 	return set->tags[hctx_idx];
 3813 }
 3814 
 3815 void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
 3816 			     struct blk_mq_tags *tags,
 3817 			     unsigned int hctx_idx)
 3818 {
 3819 	if (tags) {
 3820 		blk_mq_free_rqs(set, tags, hctx_idx);
 3821 		blk_mq_free_rq_map(tags);
 3822 	}
 3823 }
 3824 
 3825 static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
 3826 				      unsigned int hctx_idx)
 3827 {
 3828 	if (!blk_mq_is_shared_tags(set->flags))
 3829 		blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx);
 3830 
 3831 	set->tags[hctx_idx] = NULL;
 3832 }
 3833 
 3834 static void blk_mq_map_swqueue(struct request_queue *q)
 3835 {
 3836 	unsigned int j, hctx_idx;
 3837 	unsigned long i;
 3838 	struct blk_mq_hw_ctx *hctx;
 3839 	struct blk_mq_ctx *ctx;
 3840 	struct blk_mq_tag_set *set = q->tag_set;
 3841 
 3842 	queue_for_each_hw_ctx(q, hctx, i) {
 3843 		cpumask_clear(hctx->cpumask);
 3844 		hctx->nr_ctx = 0;
 3845 		hctx->dispatch_from = NULL;
 3846 	}
 3847 
 3848 	/*
 3849 	 * Map software to hardware queues.
 3850 	 *
 3851 	 * If the cpu isn't present, the cpu is mapped to first hctx.
 3852 	 */
 3853 	for_each_possible_cpu(i) {
 3854 
 3855 		ctx = per_cpu_ptr(q->queue_ctx, i);
 3856 		for (j = 0; j < set->nr_maps; j++) {
 3857 			if (!set->map[j].nr_queues) {
 3858 				ctx->hctxs[j] = blk_mq_map_queue_type(q,
 3859 						HCTX_TYPE_DEFAULT, i);
 3860 				continue;
 3861 			}
 3862 			hctx_idx = set->map[j].mq_map[i];
 3863 			/* unmapped hw queue can be remapped after CPU topo changed */
 3864 			if (!set->tags[hctx_idx] &&
 3865 			    !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
 3866 				/*
 3867 				 * If tags initialization fail for some hctx,
 3868 				 * that hctx won't be brought online.  In this
 3869 				 * case, remap the current ctx to hctx[0] which
 3870 				 * is guaranteed to always have tags allocated
 3871 				 */
 3872 				set->map[j].mq_map[i] = 0;
 3873 			}
 3874 
 3875 			hctx = blk_mq_map_queue_type(q, j, i);
 3876 			ctx->hctxs[j] = hctx;
 3877 			/*
 3878 			 * If the CPU is already set in the mask, then we've
 3879 			 * mapped this one already. This can happen if
 3880 			 * devices share queues across queue maps.
 3881 			 */
 3882 			if (cpumask_test_cpu(i, hctx->cpumask))
 3883 				continue;
 3884 
 3885 			cpumask_set_cpu(i, hctx->cpumask);
 3886 			hctx->type = j;
 3887 			ctx->index_hw[hctx->type] = hctx->nr_ctx;
 3888 			hctx->ctxs[hctx->nr_ctx++] = ctx;
 3889 
 3890 			/*
 3891 			 * If the nr_ctx type overflows, we have exceeded the
 3892 			 * amount of sw queues we can support.
 3893 			 */
 3894 			BUG_ON(!hctx->nr_ctx);
 3895 		}
 3896 
 3897 		for (; j < HCTX_MAX_TYPES; j++)
 3898 			ctx->hctxs[j] = blk_mq_map_queue_type(q,
 3899 					HCTX_TYPE_DEFAULT, i);
 3900 	}
 3901 
 3902 	queue_for_each_hw_ctx(q, hctx, i) {
 3903 		/*
 3904 		 * If no software queues are mapped to this hardware queue,
 3905 		 * disable it and free the request entries.
 3906 		 */
 3907 		if (!hctx->nr_ctx) {
 3908 			/* Never unmap queue 0.  We need it as a
 3909 			 * fallback in case of a new remap fails
 3910 			 * allocation
 3911 			 */
 3912 			if (i)
 3913 				__blk_mq_free_map_and_rqs(set, i);
 3914 
 3915 			hctx->tags = NULL;
 3916 			continue;
 3917 		}
 3918 
 3919 		hctx->tags = set->tags[i];
 3920 		WARN_ON(!hctx->tags);
 3921 
 3922 		/*
 3923 		 * Set the map size to the number of mapped software queues.
 3924 		 * This is more accurate and more efficient than looping
 3925 		 * over all possibly mapped software queues.
 3926 		 */
 3927 		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
 3928 
 3929 		/*
 3930 		 * Initialize batch roundrobin counts
 3931 		 */
 3932 		hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
 3933 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 3934 	}
 3935 }
 3936 
 3937 /*
 3938  * Caller needs to ensure that we're either frozen/quiesced, or that
 3939  * the queue isn't live yet.
 3940  */
 3941 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
 3942 {
 3943 	struct blk_mq_hw_ctx *hctx;
 3944 	unsigned long i;
 3945 
 3946 	queue_for_each_hw_ctx(q, hctx, i) {
 3947 		if (shared) {
 3948 			hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
 3949 		} else {
 3950 			blk_mq_tag_idle(hctx);
 3951 			hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
 3952 		}
 3953 	}
 3954 }
 3955 
 3956 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
 3957 					 bool shared)
 3958 {
 3959 	struct request_queue *q;
 3960 
 3961 	lockdep_assert_held(&set->tag_list_lock);
 3962 
 3963 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
 3964 		blk_mq_freeze_queue(q);
 3965 		queue_set_hctx_shared(q, shared);
 3966 		blk_mq_unfreeze_queue(q);
 3967 	}
 3968 }
 3969 
 3970 static void blk_mq_del_queue_tag_set(struct request_queue *q)
 3971 {
 3972 	struct blk_mq_tag_set *set = q->tag_set;
 3973 
 3974 	mutex_lock(&set->tag_list_lock);
 3975 	list_del(&q->tag_set_list);
 3976 	if (list_is_singular(&set->tag_list)) {
 3977 		/* just transitioned to unshared */
 3978 		set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
 3979 		/* update existing queue */
 3980 		blk_mq_update_tag_set_shared(set, false);
 3981 	}
 3982 	mutex_unlock(&set->tag_list_lock);
 3983 	INIT_LIST_HEAD(&q->tag_set_list);
 3984 }
 3985 
 3986 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
 3987 				     struct request_queue *q)
 3988 {
 3989 	mutex_lock(&set->tag_list_lock);
 3990 
 3991 	/*
 3992 	 * Check to see if we're transitioning to shared (from 1 to 2 queues).
 3993 	 */
 3994 	if (!list_empty(&set->tag_list) &&
 3995 	    !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
 3996 		set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
 3997 		/* update existing queue */
 3998 		blk_mq_update_tag_set_shared(set, true);
 3999 	}
 4000 	if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
 4001 		queue_set_hctx_shared(q, true);
 4002 	list_add_tail(&q->tag_set_list, &set->tag_list);
 4003 
 4004 	mutex_unlock(&set->tag_list_lock);
 4005 }
 4006 
 4007 /* All allocations will be freed in release handler of q->mq_kobj */
 4008 static int blk_mq_alloc_ctxs(struct request_queue *q)
 4009 {
 4010 	struct blk_mq_ctxs *ctxs;
 4011 	int cpu;
 4012 
 4013 	ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
 4014 	if (!ctxs)
 4015 		return -ENOMEM;
 4016 
 4017 	ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
 4018 	if (!ctxs->queue_ctx)
 4019 		goto fail;
 4020 
 4021 	for_each_possible_cpu(cpu) {
 4022 		struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
 4023 		ctx->ctxs = ctxs;
 4024 	}
 4025 
 4026 	q->mq_kobj = &ctxs->kobj;
 4027 	q->queue_ctx = ctxs->queue_ctx;
 4028 
 4029 	return 0;
 4030  fail:
 4031 	kfree(ctxs);
 4032 	return -ENOMEM;
 4033 }
 4034 
 4035 /*
 4036  * It is the actual release handler for mq, but we do it from
 4037  * request queue's release handler for avoiding use-after-free
 4038  * and headache because q->mq_kobj shouldn't have been introduced,
 4039  * but we can't group ctx/kctx kobj without it.
 4040  */
 4041 void blk_mq_release(struct request_queue *q)
 4042 {
 4043 	struct blk_mq_hw_ctx *hctx, *next;
 4044 	unsigned long i;
 4045 
 4046 	queue_for_each_hw_ctx(q, hctx, i)
 4047 		WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
 4048 
 4049 	/* all hctx are in .unused_hctx_list now */
 4050 	list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
 4051 		list_del_init(&hctx->hctx_list);
 4052 		kobject_put(&hctx->kobj);
 4053 	}
 4054 
 4055 	xa_destroy(&q->hctx_table);
 4056 
 4057 	/*
 4058 	 * release .mq_kobj and sw queue's kobject now because
 4059 	 * both share lifetime with request queue.
 4060 	 */
 4061 	blk_mq_sysfs_deinit(q);
 4062 }
 4063 
 4064 static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
 4065 		void *queuedata)
 4066 {
 4067 	struct request_queue *q;
 4068 	int ret;
 4069 
 4070 	q = blk_alloc_queue(set->numa_node);
 4071 	if (!q)
 4072 		return ERR_PTR(-ENOMEM);
 4073 	q->queuedata = queuedata;
 4074 	ret = blk_mq_init_allocated_queue(set, q);
 4075 	if (ret) {
 4076 		blk_put_queue(q);
 4077 		return ERR_PTR(ret);
 4078 	}
 4079 	return q;
 4080 }
 4081 
 4082 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
 4083 {
 4084 	return blk_mq_init_queue_data(set, NULL);
 4085 }
 4086 EXPORT_SYMBOL(blk_mq_init_queue);
 4087 
 4088 /**
 4089  * blk_mq_destroy_queue - shutdown a request queue
 4090  * @q: request queue to shutdown
 4091  *
 4092  * This shuts down a request queue allocated by blk_mq_init_queue(). All future
 4093  * requests will be failed with -ENODEV. The caller is responsible for dropping
 4094  * the reference from blk_mq_init_queue() by calling blk_put_queue().
 4095  *
 4096  * Context: can sleep
 4097  */
 4098 void blk_mq_destroy_queue(struct request_queue *q)
 4099 {
 4100 	WARN_ON_ONCE(!queue_is_mq(q));
 4101 	WARN_ON_ONCE(blk_queue_registered(q));
 4102 
 4103 	might_sleep();
 4104 
 4105 	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
 4106 	blk_queue_start_drain(q);
 4107 	blk_mq_freeze_queue_wait(q);
 4108 
 4109 	blk_sync_queue(q);
 4110 	blk_mq_cancel_work_sync(q);
 4111 	blk_mq_exit_queue(q);
 4112 }
 4113 EXPORT_SYMBOL(blk_mq_destroy_queue);
 4114 
 4115 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
 4116 		struct lock_class_key *lkclass)
 4117 {
 4118 	struct request_queue *q;
 4119 	struct gendisk *disk;
 4120 
 4121 	q = blk_mq_init_queue_data(set, queuedata);
 4122 	if (IS_ERR(q))
 4123 		return ERR_CAST(q);
 4124 
 4125 	disk = __alloc_disk_node(q, set->numa_node, lkclass);
 4126 	if (!disk) {
 4127 		blk_mq_destroy_queue(q);
 4128 		blk_put_queue(q);
 4129 		return ERR_PTR(-ENOMEM);
 4130 	}
 4131 	set_bit(GD_OWNS_QUEUE, &disk->state);
 4132 	return disk;
 4133 }
 4134 EXPORT_SYMBOL(__blk_mq_alloc_disk);
 4135 
 4136 struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
 4137 		struct lock_class_key *lkclass)
 4138 {
 4139 	struct gendisk *disk;
 4140 
 4141 	if (!blk_get_queue(q))
 4142 		return NULL;
 4143 	disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
 4144 	if (!disk)
 4145 		blk_put_queue(q);
 4146 	return disk;
 4147 }
 4148 EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
 4149 
 4150 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
 4151 		struct blk_mq_tag_set *set, struct request_queue *q,
 4152 		int hctx_idx, int node)
 4153 {
 4154 	struct blk_mq_hw_ctx *hctx = NULL, *tmp;
 4155 
 4156 	/* reuse dead hctx first */
 4157 	spin_lock(&q->unused_hctx_lock);
 4158 	list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
 4159 		if (tmp->numa_node == node) {
 4160 			hctx = tmp;
 4161 			break;
 4162 		}
 4163 	}
 4164 	if (hctx)
 4165 		list_del_init(&hctx->hctx_list);
 4166 	spin_unlock(&q->unused_hctx_lock);
 4167 
 4168 	if (!hctx)
 4169 		hctx = blk_mq_alloc_hctx(q, set, node);
 4170 	if (!hctx)
 4171 		goto fail;
 4172 
 4173 	if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
 4174 		goto free_hctx;
 4175 
 4176 	return hctx;
 4177 
 4178  free_hctx:
 4179 	kobject_put(&hctx->kobj);
 4180  fail:
 4181 	return NULL;
 4182 }
 4183 
 4184 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
 4185 						struct request_queue *q)
 4186 {
 4187 	struct blk_mq_hw_ctx *hctx;
 4188 	unsigned long i, j;
 4189 
 4190 	/* protect against switching io scheduler  */
 4191 	mutex_lock(&q->sysfs_lock);
 4192 	for (i = 0; i < set->nr_hw_queues; i++) {
 4193 		int old_node;
 4194 		int node = blk_mq_get_hctx_node(set, i);
 4195 		struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i);
 4196 
 4197 		if (old_hctx) {
 4198 			old_node = old_hctx->numa_node;
 4199 			blk_mq_exit_hctx(q, set, old_hctx, i);
 4200 		}
 4201 
 4202 		if (!blk_mq_alloc_and_init_hctx(set, q, i, node)) {
 4203 			if (!old_hctx)
 4204 				break;
 4205 			pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n",
 4206 					node, old_node);
 4207 			hctx = blk_mq_alloc_and_init_hctx(set, q, i, old_node);
 4208 			WARN_ON_ONCE(!hctx);
 4209 		}
 4210 	}
 4211 	/*
 4212 	 * Increasing nr_hw_queues fails. Free the newly allocated
 4213 	 * hctxs and keep the previous q->nr_hw_queues.
 4214 	 */
 4215 	if (i != set->nr_hw_queues) {
 4216 		j = q->nr_hw_queues;
 4217 	} else {
 4218 		j = i;
 4219 		q->nr_hw_queues = set->nr_hw_queues;
 4220 	}
 4221 
 4222 	xa_for_each_start(&q->hctx_table, j, hctx, j)
 4223 		blk_mq_exit_hctx(q, set, hctx, j);
 4224 	mutex_unlock(&q->sysfs_lock);
 4225 }
 4226 
 4227 static void blk_mq_update_poll_flag(struct request_queue *q)
 4228 {
 4229 	struct blk_mq_tag_set *set = q->tag_set;
 4230 
 4231 	if (set->nr_maps > HCTX_TYPE_POLL &&
 4232 	    set->map[HCTX_TYPE_POLL].nr_queues)
 4233 		blk_queue_flag_set(QUEUE_FLAG_POLL, q);
 4234 	else
 4235 		blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
 4236 }
 4237 
 4238 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
 4239 		struct request_queue *q)
 4240 {
 4241 	/* mark the queue as mq asap */
 4242 	q->mq_ops = set->ops;
 4243 
 4244 	if (blk_mq_alloc_ctxs(q))
 4245 		goto err_exit;
 4246 
 4247 	/* init q->mq_kobj and sw queues' kobjects */
 4248 	blk_mq_sysfs_init(q);
 4249 
 4250 	INIT_LIST_HEAD(&q->unused_hctx_list);
 4251 	spin_lock_init(&q->unused_hctx_lock);
 4252 
 4253 	xa_init(&q->hctx_table);
 4254 
 4255 	blk_mq_realloc_hw_ctxs(set, q);
 4256 	if (!q->nr_hw_queues)
 4257 		goto err_hctxs;
 4258 
 4259 	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
 4260 	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
 4261 
 4262 	q->tag_set = set;
 4263 
 4264 	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
 4265 	blk_mq_update_poll_flag(q);
 4266 
 4267 	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
 4268 	INIT_LIST_HEAD(&q->flush_list);
 4269 	INIT_LIST_HEAD(&q->requeue_list);
 4270 	spin_lock_init(&q->requeue_lock);
 4271 
 4272 	q->nr_requests = set->queue_depth;
 4273 
 4274 	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
 4275 	blk_mq_add_queue_tag_set(set, q);
 4276 	blk_mq_map_swqueue(q);
 4277 	return 0;
 4278 
 4279 err_hctxs:
 4280 	blk_mq_release(q);
 4281 err_exit:
 4282 	q->mq_ops = NULL;
 4283 	return -ENOMEM;
 4284 }
 4285 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
 4286 
 4287 /* tags can _not_ be used after returning from blk_mq_exit_queue */
 4288 void blk_mq_exit_queue(struct request_queue *q)
 4289 {
 4290 	struct blk_mq_tag_set *set = q->tag_set;
 4291 
 4292 	/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
 4293 	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
 4294 	/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
 4295 	blk_mq_del_queue_tag_set(q);
 4296 }
 4297 
 4298 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
 4299 {
 4300 	int i;
 4301 
 4302 	if (blk_mq_is_shared_tags(set->flags)) {
 4303 		set->shared_tags = blk_mq_alloc_map_and_rqs(set,
 4304 						BLK_MQ_NO_HCTX_IDX,
 4305 						set->queue_depth);
 4306 		if (!set->shared_tags)
 4307 			return -ENOMEM;
 4308 	}
 4309 
 4310 	for (i = 0; i < set->nr_hw_queues; i++) {
 4311 		if (!__blk_mq_alloc_map_and_rqs(set, i))
 4312 			goto out_unwind;
 4313 		cond_resched();
 4314 	}
 4315 
 4316 	return 0;
 4317 
 4318 out_unwind:
 4319 	while (--i >= 0)
 4320 		__blk_mq_free_map_and_rqs(set, i);
 4321 
 4322 	if (blk_mq_is_shared_tags(set->flags)) {
 4323 		blk_mq_free_map_and_rqs(set, set->shared_tags,
 4324 					BLK_MQ_NO_HCTX_IDX);
 4325 	}
 4326 
 4327 	return -ENOMEM;
 4328 }
 4329 
 4330 /*
 4331  * Allocate the request maps associated with this tag_set. Note that this
 4332  * may reduce the depth asked for, if memory is tight. set->queue_depth
 4333  * will be updated to reflect the allocated depth.
 4334  */
 4335 static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
 4336 {
 4337 	unsigned int depth;
 4338 	int err;
 4339 
 4340 	depth = set->queue_depth;
 4341 	do {
 4342 		err = __blk_mq_alloc_rq_maps(set);
 4343 		if (!err)
 4344 			break;
 4345 
 4346 		set->queue_depth >>= 1;
 4347 		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
 4348 			err = -ENOMEM;
 4349 			break;
 4350 		}
 4351 	} while (set->queue_depth);
 4352 
 4353 	if (!set->queue_depth || err) {
 4354 		pr_err("blk-mq: failed to allocate request map\n");
 4355 		return -ENOMEM;
 4356 	}
 4357 
 4358 	if (depth != set->queue_depth)
 4359 		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
 4360 						depth, set->queue_depth);
 4361 
 4362 	return 0;
 4363 }
 4364 
 4365 static void blk_mq_update_queue_map(struct blk_mq_tag_set *set)
 4366 {
 4367 	/*
 4368 	 * blk_mq_map_queues() and multiple .map_queues() implementations
 4369 	 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
 4370 	 * number of hardware queues.
 4371 	 */
 4372 	if (set->nr_maps == 1)
 4373 		set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
 4374 
 4375 	if (set->ops->map_queues && !is_kdump_kernel()) {
 4376 		int i;
 4377 
 4378 		/*
 4379 		 * transport .map_queues is usually done in the following
 4380 		 * way:
 4381 		 *
 4382 		 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
 4383 		 * 	mask = get_cpu_mask(queue)
 4384 		 * 	for_each_cpu(cpu, mask)
 4385 		 * 		set->map[x].mq_map[cpu] = queue;
 4386 		 * }
 4387 		 *
 4388 		 * When we need to remap, the table has to be cleared for
 4389 		 * killing stale mapping since one CPU may not be mapped
 4390 		 * to any hw queue.
 4391 		 */
 4392 		for (i = 0; i < set->nr_maps; i++)
 4393 			blk_mq_clear_mq_map(&set->map[i]);
 4394 
 4395 		set->ops->map_queues(set);
 4396 	} else {
 4397 		BUG_ON(set->nr_maps > 1);
 4398 		blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
 4399 	}
 4400 }
 4401 
 4402 static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
 4403 				       int new_nr_hw_queues)
 4404 {
 4405 	struct blk_mq_tags **new_tags;
 4406 	int i;
 4407 
 4408 	if (set->nr_hw_queues >= new_nr_hw_queues)
 4409 		goto done;
 4410 
 4411 	new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
 4412 				GFP_KERNEL, set->numa_node);
 4413 	if (!new_tags)
 4414 		return -ENOMEM;
 4415 
 4416 	if (set->tags)
 4417 		memcpy(new_tags, set->tags, set->nr_hw_queues *
 4418 		       sizeof(*set->tags));
 4419 	kfree(set->tags);
 4420 	set->tags = new_tags;
 4421 
 4422 	for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) {
 4423 		if (!__blk_mq_alloc_map_and_rqs(set, i)) {
 4424 			while (--i >= set->nr_hw_queues)
 4425 				__blk_mq_free_map_and_rqs(set, i);
 4426 			return -ENOMEM;
 4427 		}
 4428 		cond_resched();
 4429 	}
 4430 
 4431 done:
 4432 	set->nr_hw_queues = new_nr_hw_queues;
 4433 	return 0;
 4434 }
 4435 
 4436 /*
 4437  * Alloc a tag set to be associated with one or more request queues.
 4438  * May fail with EINVAL for various error conditions. May adjust the
 4439  * requested depth down, if it's too large. In that case, the set
 4440  * value will be stored in set->queue_depth.
 4441  */
 4442 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
 4443 {
 4444 	int i, ret;
 4445 
 4446 	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
 4447 
 4448 	if (!set->nr_hw_queues)
 4449 		return -EINVAL;
 4450 	if (!set->queue_depth)
 4451 		return -EINVAL;
 4452 	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
 4453 		return -EINVAL;
 4454 
 4455 	if (!set->ops->queue_rq)
 4456 		return -EINVAL;
 4457 
 4458 	if (!set->ops->get_budget ^ !set->ops->put_budget)
 4459 		return -EINVAL;
 4460 
 4461 	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
 4462 		pr_info("blk-mq: reduced tag depth to %u\n",
 4463 			BLK_MQ_MAX_DEPTH);
 4464 		set->queue_depth = BLK_MQ_MAX_DEPTH;
 4465 	}
 4466 
 4467 	if (!set->nr_maps)
 4468 		set->nr_maps = 1;
 4469 	else if (set->nr_maps > HCTX_MAX_TYPES)
 4470 		return -EINVAL;
 4471 
 4472 	/*
 4473 	 * If a crashdump is active, then we are potentially in a very
 4474 	 * memory constrained environment. Limit us to 1 queue and
 4475 	 * 64 tags to prevent using too much memory.
 4476 	 */
 4477 	if (is_kdump_kernel()) {
 4478 		set->nr_hw_queues = 1;
 4479 		set->nr_maps = 1;
 4480 		set->queue_depth = min(64U, set->queue_depth);
 4481 	}
 4482 	/*
 4483 	 * There is no use for more h/w queues than cpus if we just have
 4484 	 * a single map
 4485 	 */
 4486 	if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
 4487 		set->nr_hw_queues = nr_cpu_ids;
 4488 
 4489 	if (set->flags & BLK_MQ_F_BLOCKING) {
 4490 		set->srcu = kmalloc(sizeof(*set->srcu), GFP_KERNEL);
 4491 		if (!set->srcu)
 4492 			return -ENOMEM;
 4493 		ret = init_srcu_struct(set->srcu);
 4494 		if (ret)
 4495 			goto out_free_srcu;
 4496 	}
 4497 
 4498 	ret = -ENOMEM;
 4499 	set->tags = kcalloc_node(set->nr_hw_queues,
 4500 				 sizeof(struct blk_mq_tags *), GFP_KERNEL,
 4501 				 set->numa_node);
 4502 	if (!set->tags)
 4503 		goto out_cleanup_srcu;
 4504 
 4505 	for (i = 0; i < set->nr_maps; i++) {
 4506 		set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
 4507 						  sizeof(set->map[i].mq_map[0]),
 4508 						  GFP_KERNEL, set->numa_node);
 4509 		if (!set->map[i].mq_map)
 4510 			goto out_free_mq_map;
 4511 		set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues;
 4512 	}
 4513 
 4514 	blk_mq_update_queue_map(set);
 4515 
 4516 	ret = blk_mq_alloc_set_map_and_rqs(set);
 4517 	if (ret)
 4518 		goto out_free_mq_map;
 4519 
 4520 	mutex_init(&set->tag_list_lock);
 4521 	INIT_LIST_HEAD(&set->tag_list);
 4522 
 4523 	return 0;
 4524 
 4525 out_free_mq_map:
 4526 	for (i = 0; i < set->nr_maps; i++) {
 4527 		kfree(set->map[i].mq_map);
 4528 		set->map[i].mq_map = NULL;
 4529 	}
 4530 	kfree(set->tags);
 4531 	set->tags = NULL;
 4532 out_cleanup_srcu:
 4533 	if (set->flags & BLK_MQ_F_BLOCKING)
 4534 		cleanup_srcu_struct(set->srcu);
 4535 out_free_srcu:
 4536 	if (set->flags & BLK_MQ_F_BLOCKING)
 4537 		kfree(set->srcu);
 4538 	return ret;
 4539 }
 4540 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
 4541 
 4542 /* allocate and initialize a tagset for a simple single-queue device */
 4543 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
 4544 		const struct blk_mq_ops *ops, unsigned int queue_depth,
 4545 		unsigned int set_flags)
 4546 {
 4547 	memset(set, 0, sizeof(*set));
 4548 	set->ops = ops;
 4549 	set->nr_hw_queues = 1;
 4550 	set->nr_maps = 1;
 4551 	set->queue_depth = queue_depth;
 4552 	set->numa_node = NUMA_NO_NODE;
 4553 	set->flags = set_flags;
 4554 	return blk_mq_alloc_tag_set(set);
 4555 }
 4556 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
 4557 
 4558 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
 4559 {
 4560 	int i, j;
 4561 
 4562 	for (i = 0; i < set->nr_hw_queues; i++)
 4563 		__blk_mq_free_map_and_rqs(set, i);
 4564 
 4565 	if (blk_mq_is_shared_tags(set->flags)) {
 4566 		blk_mq_free_map_and_rqs(set, set->shared_tags,
 4567 					BLK_MQ_NO_HCTX_IDX);
 4568 	}
 4569 
 4570 	for (j = 0; j < set->nr_maps; j++) {
 4571 		kfree(set->map[j].mq_map);
 4572 		set->map[j].mq_map = NULL;
 4573 	}
 4574 
 4575 	kfree(set->tags);
 4576 	set->tags = NULL;
 4577 	if (set->flags & BLK_MQ_F_BLOCKING) {
 4578 		cleanup_srcu_struct(set->srcu);
 4579 		kfree(set->srcu);
 4580 	}
 4581 }
 4582 EXPORT_SYMBOL(blk_mq_free_tag_set);
 4583 
 4584 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
 4585 {
 4586 	struct blk_mq_tag_set *set = q->tag_set;
 4587 	struct blk_mq_hw_ctx *hctx;
 4588 	int ret;
 4589 	unsigned long i;
 4590 
 4591 	if (!set)
 4592 		return -EINVAL;
 4593 
 4594 	if (q->nr_requests == nr)
 4595 		return 0;
 4596 
 4597 	blk_mq_freeze_queue(q);
 4598 	blk_mq_quiesce_queue(q);
 4599 
 4600 	ret = 0;
 4601 	queue_for_each_hw_ctx(q, hctx, i) {
 4602 		if (!hctx->tags)
 4603 			continue;
 4604 		/*
 4605 		 * If we're using an MQ scheduler, just update the scheduler
 4606 		 * queue depth. This is similar to what the old code would do.
 4607 		 */
 4608 		if (hctx->sched_tags) {
 4609 			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
 4610 						      nr, true);
 4611 		} else {
 4612 			ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
 4613 						      false);
 4614 		}
 4615 		if (ret)
 4616 			break;
 4617 		if (q->elevator && q->elevator->type->ops.depth_updated)
 4618 			q->elevator->type->ops.depth_updated(hctx);
 4619 	}
 4620 	if (!ret) {
 4621 		q->nr_requests = nr;
 4622 		if (blk_mq_is_shared_tags(set->flags)) {
 4623 			if (q->elevator)
 4624 				blk_mq_tag_update_sched_shared_tags(q);
 4625 			else
 4626 				blk_mq_tag_resize_shared_tags(set, nr);
 4627 		}
 4628 	}
 4629 
 4630 	blk_mq_unquiesce_queue(q);
 4631 	blk_mq_unfreeze_queue(q);
 4632 
 4633 	return ret;
 4634 }
 4635 
 4636 /*
 4637  * request_queue and elevator_type pair.
 4638  * It is just used by __blk_mq_update_nr_hw_queues to cache
 4639  * the elevator_type associated with a request_queue.
 4640  */
 4641 struct blk_mq_qe_pair {
 4642 	struct list_head node;
 4643 	struct request_queue *q;
 4644 	struct elevator_type *type;
 4645 };
 4646 
 4647 /*
 4648  * Cache the elevator_type in qe pair list and switch the
 4649  * io scheduler to 'none'
 4650  */
 4651 static bool blk_mq_elv_switch_none(struct list_head *head,
 4652 		struct request_queue *q)
 4653 {
 4654 	struct blk_mq_qe_pair *qe;
 4655 
 4656 	qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
 4657 	if (!qe)
 4658 		return false;
 4659 
 4660 	/* q->elevator needs protection from ->sysfs_lock */
 4661 	mutex_lock(&q->sysfs_lock);
 4662 
 4663 	/* the check has to be done with holding sysfs_lock */
 4664 	if (!q->elevator) {
 4665 		kfree(qe);
 4666 		goto unlock;
 4667 	}
 4668 
 4669 	INIT_LIST_HEAD(&qe->node);
 4670 	qe->q = q;
 4671 	qe->type = q->elevator->type;
 4672 	/* keep a reference to the elevator module as we'll switch back */
 4673 	__elevator_get(qe->type);
 4674 	list_add(&qe->node, head);
 4675 	elevator_disable(q);
 4676 unlock:
 4677 	mutex_unlock(&q->sysfs_lock);
 4678 
 4679 	return true;
 4680 }
 4681 
 4682 static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head,
 4683 						struct request_queue *q)
 4684 {
 4685 	struct blk_mq_qe_pair *qe;
 4686 
 4687 	list_for_each_entry(qe, head, node)
 4688 		if (qe->q == q)
 4689 			return qe;
 4690 
 4691 	return NULL;
 4692 }
 4693 
 4694 static void blk_mq_elv_switch_back(struct list_head *head,
 4695 				  struct request_queue *q)
 4696 {
 4697 	struct blk_mq_qe_pair *qe;
 4698 	struct elevator_type *t;
 4699 
 4700 	qe = blk_lookup_qe_pair(head, q);
 4701 	if (!qe)
 4702 		return;
 4703 	t = qe->type;
 4704 	list_del(&qe->node);
 4705 	kfree(qe);
 4706 
 4707 	mutex_lock(&q->sysfs_lock);
 4708 	elevator_switch(q, t);
 4709 	/* drop the reference acquired in blk_mq_elv_switch_none */
 4710 	elevator_put(t);
 4711 	mutex_unlock(&q->sysfs_lock);
 4712 }
 4713 
 4714 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
 4715 							int nr_hw_queues)
 4716 {
 4717 	struct request_queue *q;
 4718 	LIST_HEAD(head);
 4719 	int prev_nr_hw_queues = set->nr_hw_queues;
 4720 	int i;
 4721 
 4722 	lockdep_assert_held(&set->tag_list_lock);
 4723 
 4724 	if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
 4725 		nr_hw_queues = nr_cpu_ids;
 4726 	if (nr_hw_queues < 1)
 4727 		return;
 4728 	if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
 4729 		return;
 4730 
 4731 	list_for_each_entry(q, &set->tag_list, tag_set_list)
 4732 		blk_mq_freeze_queue(q);
 4733 	/*
 4734 	 * Switch IO scheduler to 'none', cleaning up the data associated
 4735 	 * with the previous scheduler. We will switch back once we are done
 4736 	 * updating the new sw to hw queue mappings.
 4737 	 */
 4738 	list_for_each_entry(q, &set->tag_list, tag_set_list)
 4739 		if (!blk_mq_elv_switch_none(&head, q))
 4740 			goto switch_back;
 4741 
 4742 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
 4743 		blk_mq_debugfs_unregister_hctxs(q);
 4744 		blk_mq_sysfs_unregister_hctxs(q);
 4745 	}
 4746 
 4747 	if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0)
 4748 		goto reregister;
 4749 
 4750 fallback:
 4751 	blk_mq_update_queue_map(set);
 4752 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
 4753 		blk_mq_realloc_hw_ctxs(set, q);
 4754 		blk_mq_update_poll_flag(q);
 4755 		if (q->nr_hw_queues != set->nr_hw_queues) {
 4756 			int i = prev_nr_hw_queues;
 4757 
 4758 			pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
 4759 					nr_hw_queues, prev_nr_hw_queues);
 4760 			for (; i < set->nr_hw_queues; i++)
 4761 				__blk_mq_free_map_and_rqs(set, i);
 4762 
 4763 			set->nr_hw_queues = prev_nr_hw_queues;
 4764 			goto fallback;
 4765 		}
 4766 		blk_mq_map_swqueue(q);
 4767 	}
 4768 
 4769 reregister:
 4770 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
 4771 		blk_mq_sysfs_register_hctxs(q);
 4772 		blk_mq_debugfs_register_hctxs(q);
 4773 	}
 4774 
 4775 switch_back:
 4776 	list_for_each_entry(q, &set->tag_list, tag_set_list)
 4777 		blk_mq_elv_switch_back(&head, q);
 4778 
 4779 	list_for_each_entry(q, &set->tag_list, tag_set_list)
 4780 		blk_mq_unfreeze_queue(q);
 4781 
 4782 	/* Free the excess tags when nr_hw_queues shrink. */
 4783 	for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++)
 4784 		__blk_mq_free_map_and_rqs(set, i);
 4785 }
 4786 
 4787 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
 4788 {
 4789 	mutex_lock(&set->tag_list_lock);
 4790 	__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
 4791 	mutex_unlock(&set->tag_list_lock);
 4792 }
 4793 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
 4794 
 4795 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
 4796 			 struct io_comp_batch *iob, unsigned int flags)
 4797 {
 4798 	long state = get_current_state();
 4799 	int ret;
 4800 
 4801 	do {
 4802 		ret = q->mq_ops->poll(hctx, iob);
 4803 		if (ret > 0) {
 4804 			__set_current_state(TASK_RUNNING);
 4805 			return ret;
 4806 		}
 4807 
 4808 		if (signal_pending_state(state, current))
 4809 			__set_current_state(TASK_RUNNING);
 4810 		if (task_is_running(current))
 4811 			return 1;
 4812 
 4813 		if (ret < 0 || (flags & BLK_POLL_ONESHOT))
 4814 			break;
 4815 		cpu_relax();
 4816 	} while (!need_resched());
 4817 
 4818 	__set_current_state(TASK_RUNNING);
 4819 	return 0;
 4820 }
 4821 
 4822 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
 4823 		struct io_comp_batch *iob, unsigned int flags)
 4824 {
 4825 	struct blk_mq_hw_ctx *hctx = xa_load(&q->hctx_table, cookie);
 4826 
 4827 	return blk_hctx_poll(q, hctx, iob, flags);
 4828 }
 4829 
 4830 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
 4831 		unsigned int poll_flags)
 4832 {
 4833 	struct request_queue *q = rq->q;
 4834 	int ret;
 4835 
 4836 	if (!blk_rq_is_poll(rq))
 4837 		return 0;
 4838 	if (!percpu_ref_tryget(&q->q_usage_counter))
 4839 		return 0;
 4840 
 4841 	ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags);
 4842 	blk_queue_exit(q);
 4843 
 4844 	return ret;
 4845 }
 4846 EXPORT_SYMBOL_GPL(blk_rq_poll);
 4847 
 4848 unsigned int blk_mq_rq_cpu(struct request *rq)
 4849 {
 4850 	return rq->mq_ctx->cpu;
 4851 }
 4852 EXPORT_SYMBOL(blk_mq_rq_cpu);
 4853 
 4854 void blk_mq_cancel_work_sync(struct request_queue *q)
 4855 {
 4856 	struct blk_mq_hw_ctx *hctx;
 4857 	unsigned long i;
 4858 
 4859 	cancel_delayed_work_sync(&q->requeue_work);
 4860 
 4861 	queue_for_each_hw_ctx(q, hctx, i)
 4862 		cancel_delayed_work_sync(&hctx->run_work);
 4863 }
 4864 
 4865 static int __init blk_mq_init(void)
 4866 {
 4867 	int i;
 4868 
 4869 	for_each_possible_cpu(i)
 4870 		init_llist_head(&per_cpu(blk_cpu_done, i));
 4871 	for_each_possible_cpu(i)
 4872 		INIT_CSD(&per_cpu(blk_cpu_csd, i),
 4873 			 __blk_mq_complete_request_remote, NULL);
 4874 	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
 4875 
 4876 	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
 4877 				  "block/softirq:dead", NULL,
 4878 				  blk_softirq_cpu_dead);
 4879 	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
 4880 				blk_mq_hctx_notify_dead);
 4881 	cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
 4882 				blk_mq_hctx_notify_online,
 4883 				blk_mq_hctx_notify_offline);
 4884 	return 0;
 4885 }
 4886 subsys_initcall(blk_mq_init);