개념 설명 전체
block/blk-mq.c · v6.18.37
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Block multiqueue core code
4 *
5 * Copyright (C) 2013-2014 Jens Axboe
6 * Copyright (C) 2013-2014 Christoph Hellwig
7 */
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/backing-dev.h>
11 #include <linux/bio.h>
12 #include <linux/blkdev.h>
13 #include <linux/blk-integrity.h>
14 #include <linux/kmemleak.h>
15 #include <linux/mm.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/workqueue.h>
19 #include <linux/smp.h>
20 #include <linux/interrupt.h>
21 #include <linux/llist.h>
22 #include <linux/cpu.h>
23 #include <linux/cache.h>
24 #include <linux/sched/topology.h>
25 #include <linux/sched/signal.h>
26 #include <linux/suspend.h>
27 #include <linux/delay.h>
28 #include <linux/crash_dump.h>
29 #include <linux/prefetch.h>
30 #include <linux/blk-crypto.h>
31 #include <linux/part_stat.h>
32 #include <linux/sched/isolation.h>
33
34 #include <trace/events/block.h>
35
36 #include <linux/t10-pi.h>
37 #include "blk.h"
38 #include "blk-mq.h"
39 #include "blk-mq-debugfs.h"
40 #include "blk-pm.h"
41 #include "blk-stat.h"
42 #include "blk-mq-sched.h"
43 #include "blk-rq-qos.h"
44
45 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
46 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
47 static DEFINE_MUTEX(blk_mq_cpuhp_lock);
48
49 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags);
50 static void blk_mq_request_bypass_insert(struct request *rq,
51 blk_insert_t flags);
52 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
53 struct list_head *list);
54 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
55 struct io_comp_batch *iob, unsigned int flags);
56
57 /*
58 * Check if any of the ctx, dispatch list or elevator
59 * have pending work in this hardware queue.
60 */
61 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
62 {
63 return !list_empty_careful(&hctx->dispatch) ||
64 sbitmap_any_bit_set(&hctx->ctx_map) ||
65 blk_mq_sched_has_work(hctx);
66 }
67
68 /*
69 * Mark this ctx as having pending work in this hardware queue
70 */
71 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
72 struct blk_mq_ctx *ctx)
73 {
74 const int bit = ctx->index_hw[hctx->type];
75
76 if (!sbitmap_test_bit(&hctx->ctx_map, bit))
77 sbitmap_set_bit(&hctx->ctx_map, bit);
78 }
79
80 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
81 struct blk_mq_ctx *ctx)
82 {
83 const int bit = ctx->index_hw[hctx->type];
84
85 sbitmap_clear_bit(&hctx->ctx_map, bit);
86 }
87
88 struct mq_inflight {
89 struct block_device *part;
90 unsigned int inflight[2];
91 };
92
93 static bool blk_mq_check_in_driver(struct request *rq, void *priv)
94 {
95 struct mq_inflight *mi = priv;
96
97 if (rq->rq_flags & RQF_IO_STAT &&
98 (!bdev_is_partition(mi->part) || rq->part == mi->part) &&
99 blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
100 mi->inflight[rq_data_dir(rq)]++;
101
102 return true;
103 }
104
105 void blk_mq_in_driver_rw(struct block_device *part, unsigned int inflight[2])
106 {
107 struct mq_inflight mi = { .part = part };
108
109 blk_mq_queue_tag_busy_iter(bdev_get_queue(part), blk_mq_check_in_driver,
110 &mi);
111 inflight[READ] = mi.inflight[READ];
112 inflight[WRITE] = mi.inflight[WRITE];
113 }
114
115 #ifdef CONFIG_LOCKDEP
116 static bool blk_freeze_set_owner(struct request_queue *q,
117 struct task_struct *owner)
118 {
119 if (!owner)
120 return false;
121
122 if (!q->mq_freeze_depth) {
123 q->mq_freeze_owner = owner;
124 q->mq_freeze_owner_depth = 1;
125 q->mq_freeze_disk_dead = !q->disk ||
126 test_bit(GD_DEAD, &q->disk->state) ||
127 !blk_queue_registered(q);
128 q->mq_freeze_queue_dying = blk_queue_dying(q);
129 return true;
130 }
131
132 if (owner == q->mq_freeze_owner)
133 q->mq_freeze_owner_depth += 1;
134 return false;
135 }
136
137 /* verify the last unfreeze in owner context */
138 static bool blk_unfreeze_check_owner(struct request_queue *q)
139 {
140 if (q->mq_freeze_owner != current)
141 return false;
142 if (--q->mq_freeze_owner_depth == 0) {
143 q->mq_freeze_owner = NULL;
144 return true;
145 }
146 return false;
147 }
148
149 #else
150
151 static bool blk_freeze_set_owner(struct request_queue *q,
152 struct task_struct *owner)
153 {
154 return false;
155 }
156
157 static bool blk_unfreeze_check_owner(struct request_queue *q)
158 {
159 return false;
160 }
161 #endif
162
163 bool __blk_freeze_queue_start(struct request_queue *q,
164 struct task_struct *owner)
165 {
166 bool freeze;
167
168 mutex_lock(&q->mq_freeze_lock);
169 freeze = blk_freeze_set_owner(q, owner);
170 if (++q->mq_freeze_depth == 1) {
171 percpu_ref_kill(&q->q_usage_counter);
172 mutex_unlock(&q->mq_freeze_lock);
173 if (queue_is_mq(q))
174 blk_mq_run_hw_queues(q, false);
175 } else {
176 mutex_unlock(&q->mq_freeze_lock);
177 }
178
179 return freeze;
180 }
181
182 void blk_freeze_queue_start(struct request_queue *q)
183 {
184 if (__blk_freeze_queue_start(q, current))
185 blk_freeze_acquire_lock(q);
186 }
187 EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
188
189 void blk_mq_freeze_queue_wait(struct request_queue *q)
190 {
191 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
192 }
193 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
194
195 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
196 unsigned long timeout)
197 {
198 return wait_event_timeout(q->mq_freeze_wq,
199 percpu_ref_is_zero(&q->q_usage_counter),
200 timeout);
201 }
202 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
203
204 void blk_mq_freeze_queue_nomemsave(struct request_queue *q)
205 {
206 blk_freeze_queue_start(q);
207 blk_mq_freeze_queue_wait(q);
208 }
209 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_nomemsave);
210
211 bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
212 {
213 bool unfreeze;
214
215 mutex_lock(&q->mq_freeze_lock);
216 if (force_atomic)
217 q->q_usage_counter.data->force_atomic = true;
218 q->mq_freeze_depth--;
219 WARN_ON_ONCE(q->mq_freeze_depth < 0);
220 if (!q->mq_freeze_depth) {
221 percpu_ref_resurrect(&q->q_usage_counter);
222 wake_up_all(&q->mq_freeze_wq);
223 }
224 unfreeze = blk_unfreeze_check_owner(q);
225 mutex_unlock(&q->mq_freeze_lock);
226
227 return unfreeze;
228 }
229
230 void blk_mq_unfreeze_queue_nomemrestore(struct request_queue *q)
231 {
232 if (__blk_mq_unfreeze_queue(q, false))
233 blk_unfreeze_release_lock(q);
234 }
235 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_nomemrestore);
236
237 /*
238 * non_owner variant of blk_freeze_queue_start
239 *
240 * Unlike blk_freeze_queue_start, the queue doesn't need to be unfrozen
241 * by the same task. This is fragile and should not be used if at all
242 * possible.
243 */
244 void blk_freeze_queue_start_non_owner(struct request_queue *q)
245 {
246 __blk_freeze_queue_start(q, NULL);
247 }
248 EXPORT_SYMBOL_GPL(blk_freeze_queue_start_non_owner);
249
250 /* non_owner variant of blk_mq_unfreeze_queue */
251 void blk_mq_unfreeze_queue_non_owner(struct request_queue *q)
252 {
253 __blk_mq_unfreeze_queue(q, false);
254 }
255 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_non_owner);
256
257 /*
258 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
259 * mpt3sas driver such that this function can be removed.
260 */
261 void blk_mq_quiesce_queue_nowait(struct request_queue *q)
262 {
263 unsigned long flags;
264
265 spin_lock_irqsave(&q->queue_lock, flags);
266 if (!q->quiesce_depth++)
267 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
268 spin_unlock_irqrestore(&q->queue_lock, flags);
269 }
270 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
271
272 /**
273 * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done
274 * @set: tag_set to wait on
275 *
276 * Note: it is driver's responsibility for making sure that quiesce has
277 * been started on or more of the request_queues of the tag_set. This
278 * function only waits for the quiesce on those request_queues that had
279 * the quiesce flag set using blk_mq_quiesce_queue_nowait.
280 */
281 void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set)
282 {
283 if (set->flags & BLK_MQ_F_BLOCKING)
284 synchronize_srcu(set->srcu);
285 else
286 synchronize_rcu();
287 }
288 EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done);
289
290 /**
291 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
292 * @q: request queue.
293 *
294 * Note: this function does not prevent that the struct request end_io()
295 * callback function is invoked. Once this function is returned, we make
296 * sure no dispatch can happen until the queue is unquiesced via
297 * blk_mq_unquiesce_queue().
298 */
299 void blk_mq_quiesce_queue(struct request_queue *q)
300 {
301 blk_mq_quiesce_queue_nowait(q);
302 /* nothing to wait for non-mq queues */
303 if (queue_is_mq(q))
304 blk_mq_wait_quiesce_done(q->tag_set);
305 }
306 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
307
308 /*
309 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
310 * @q: request queue.
311 *
312 * This function recovers queue into the state before quiescing
313 * which is done by blk_mq_quiesce_queue.
314 */
315 void blk_mq_unquiesce_queue(struct request_queue *q)
316 {
317 unsigned long flags;
318 bool run_queue = false;
319
320 spin_lock_irqsave(&q->queue_lock, flags);
321 if (WARN_ON_ONCE(q->quiesce_depth <= 0)) {
322 ;
323 } else if (!--q->quiesce_depth) {
324 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
325 run_queue = true;
326 }
327 spin_unlock_irqrestore(&q->queue_lock, flags);
328
329 /* dispatch requests which are inserted during quiescing */
330 if (run_queue)
331 blk_mq_run_hw_queues(q, true);
332 }
333 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
334
335 void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
336 {
337 struct request_queue *q;
338
339 rcu_read_lock();
340 list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
341 if (!blk_queue_skip_tagset_quiesce(q))
342 blk_mq_quiesce_queue_nowait(q);
343 }
344 rcu_read_unlock();
345
346 blk_mq_wait_quiesce_done(set);
347 }
348 EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
349
350 void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
351 {
352 struct request_queue *q;
353
354 rcu_read_lock();
355 list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
356 if (!blk_queue_skip_tagset_quiesce(q))
357 blk_mq_unquiesce_queue(q);
358 }
359 rcu_read_unlock();
360 }
361 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
362
363 void blk_mq_wake_waiters(struct request_queue *q)
364 {
365 struct blk_mq_hw_ctx *hctx;
366 unsigned long i;
367
368 queue_for_each_hw_ctx(q, hctx, i)
369 if (blk_mq_hw_queue_mapped(hctx))
370 blk_mq_tag_wakeup_all(hctx->tags, true);
371 }
372
373 void blk_rq_init(struct request_queue *q, struct request *rq)
374 {
375 memset(rq, 0, sizeof(*rq));
376
377 INIT_LIST_HEAD(&rq->queuelist);
378 rq->q = q;
379 rq->__sector = (sector_t) -1;
380 INIT_HLIST_NODE(&rq->hash);
381 RB_CLEAR_NODE(&rq->rb_node);
382 rq->tag = BLK_MQ_NO_TAG;
383 rq->internal_tag = BLK_MQ_NO_TAG;
384 rq->start_time_ns = blk_time_get_ns();
385 blk_crypto_rq_set_defaults(rq);
386 }
387 EXPORT_SYMBOL(blk_rq_init);
388
389 /* Set start and alloc time when the allocated request is actually used */
390 static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
391 {
392 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
393 if (blk_queue_rq_alloc_time(rq->q))
394 rq->alloc_time_ns = alloc_time_ns;
395 else
396 rq->alloc_time_ns = 0;
397 #endif
398 }
399
400 static inline void blk_mq_bio_issue_init(struct request_queue *q,
401 struct bio *bio)
402 {
403 #ifdef CONFIG_BLK_CGROUP
404 if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags))
405 bio->issue_time_ns = blk_time_get_ns();
406 #endif
407 }
408
409 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
410 struct blk_mq_tags *tags, unsigned int tag)
411 {
412 struct blk_mq_ctx *ctx = data->ctx;
413 struct blk_mq_hw_ctx *hctx = data->hctx;
414 struct request_queue *q = data->q;
415 struct request *rq = tags->static_rqs[tag];
416
417 rq->q = q;
418 rq->mq_ctx = ctx;
419 rq->mq_hctx = hctx;
420 rq->cmd_flags = data->cmd_flags;
421
422 if (data->flags & BLK_MQ_REQ_PM)
423 data->rq_flags |= RQF_PM;
424 rq->rq_flags = data->rq_flags;
425
426 if (data->rq_flags & RQF_SCHED_TAGS) {
427 rq->tag = BLK_MQ_NO_TAG;
428 rq->internal_tag = tag;
429 } else {
430 rq->tag = tag;
431 rq->internal_tag = BLK_MQ_NO_TAG;
432 }
433 rq->timeout = 0;
434
435 rq->part = NULL;
436 rq->io_start_time_ns = 0;
437 rq->stats_sectors = 0;
438 rq->nr_phys_segments = 0;
439 rq->nr_integrity_segments = 0;
440 rq->end_io = NULL;
441 rq->end_io_data = NULL;
442
443 blk_crypto_rq_set_defaults(rq);
444 INIT_LIST_HEAD(&rq->queuelist);
445 /* tag was already set */
446 WRITE_ONCE(rq->deadline, 0);
447 req_ref_set(rq, 1);
448
449 if (rq->rq_flags & RQF_USE_SCHED) {
450 struct elevator_queue *e = data->q->elevator;
451
452 INIT_HLIST_NODE(&rq->hash);
453 RB_CLEAR_NODE(&rq->rb_node);
454
455 if (e->type->ops.prepare_request)
456 e->type->ops.prepare_request(rq);
457 }
458
459 return rq;
460 }
461
462 static inline struct request *
463 __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
464 {
465 unsigned int tag, tag_offset;
466 struct blk_mq_tags *tags;
467 struct request *rq;
468 unsigned long tag_mask;
469 int i, nr = 0;
470
471 tag_mask = blk_mq_get_tags(data, data->nr_tags, &tag_offset);
472 if (unlikely(!tag_mask))
473 return NULL;
474
475 tags = blk_mq_tags_from_data(data);
476 for (i = 0; tag_mask; i++) {
477 if (!(tag_mask & (1UL << i)))
478 continue;
479 tag = tag_offset + i;
480 prefetch(tags->static_rqs[tag]);
481 tag_mask &= ~(1UL << i);
482 rq = blk_mq_rq_ctx_init(data, tags, tag);
483 rq_list_add_head(data->cached_rqs, rq);
484 nr++;
485 }
486 if (!(data->rq_flags & RQF_SCHED_TAGS))
487 blk_mq_add_active_requests(data->hctx, nr);
488 /* caller already holds a reference, add for remainder */
489 percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
490 data->nr_tags -= nr;
491
492 return rq_list_pop(data->cached_rqs);
493 }
494
495 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
496 {
497 struct request_queue *q = data->q;
498 u64 alloc_time_ns = 0;
499 struct request *rq;
500 unsigned int tag;
501
502 /* alloc_time includes depth and tag waits */
503 if (blk_queue_rq_alloc_time(q))
504 alloc_time_ns = blk_time_get_ns();
505
506 if (data->cmd_flags & REQ_NOWAIT)
507 data->flags |= BLK_MQ_REQ_NOWAIT;
508
509 retry:
510 data->ctx = blk_mq_get_ctx(q);
511 data->hctx = blk_mq_map_queue(data->cmd_flags, data->ctx);
512
513 if (q->elevator) {
514 /*
515 * All requests use scheduler tags when an I/O scheduler is
516 * enabled for the queue.
517 */
518 data->rq_flags |= RQF_SCHED_TAGS;
519
520 /*
521 * Flush/passthrough requests are special and go directly to the
522 * dispatch list.
523 */
524 if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH &&
525 !blk_op_is_passthrough(data->cmd_flags)) {
526 struct elevator_mq_ops *ops = &q->elevator->type->ops;
527
528 WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
529
530 data->rq_flags |= RQF_USE_SCHED;
531 if (ops->limit_depth)
532 ops->limit_depth(data->cmd_flags, data);
533 }
534 } else {
535 blk_mq_tag_busy(data->hctx);
536 }
537
538 if (data->flags & BLK_MQ_REQ_RESERVED)
539 data->rq_flags |= RQF_RESV;
540
541 /*
542 * Try batched alloc if we want more than 1 tag.
543 */
544 if (data->nr_tags > 1) {
545 rq = __blk_mq_alloc_requests_batch(data);
546 if (rq) {
547 blk_mq_rq_time_init(rq, alloc_time_ns);
548 return rq;
549 }
550 data->nr_tags = 1;
551 }
552
553 /*
554 * Waiting allocations only fail because of an inactive hctx. In that
555 * case just retry the hctx assignment and tag allocation as CPU hotplug
556 * should have migrated us to an online CPU by now.
557 */
558 tag = blk_mq_get_tag(data);
559 if (tag == BLK_MQ_NO_TAG) {
560 if (data->flags & BLK_MQ_REQ_NOWAIT)
561 return NULL;
562 /*
563 * Give up the CPU and sleep for a random short time to
564 * ensure that thread using a realtime scheduling class
565 * are migrated off the CPU, and thus off the hctx that
566 * is going away.
567 */
568 msleep(3);
569 goto retry;
570 }
571
572 if (!(data->rq_flags & RQF_SCHED_TAGS))
573 blk_mq_inc_active_requests(data->hctx);
574 rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
575 blk_mq_rq_time_init(rq, alloc_time_ns);
576 return rq;
577 }
578
579 static struct request *blk_mq_rq_cache_fill(struct request_queue *q,
580 struct blk_plug *plug,
581 blk_opf_t opf,
582 blk_mq_req_flags_t flags)
583 {
584 struct blk_mq_alloc_data data = {
585 .q = q,
586 .flags = flags,
587 .shallow_depth = 0,
588 .cmd_flags = opf,
589 .rq_flags = 0,
590 .nr_tags = plug->nr_ios,
591 .cached_rqs = &plug->cached_rqs,
592 .ctx = NULL,
593 .hctx = NULL
594 };
595 struct request *rq;
596
597 if (blk_queue_enter(q, flags))
598 return NULL;
599
600 plug->nr_ios = 1;
601
602 rq = __blk_mq_alloc_requests(&data);
603 if (unlikely(!rq))
604 blk_queue_exit(q);
605 return rq;
606 }
607
608 static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
609 blk_opf_t opf,
610 blk_mq_req_flags_t flags)
611 {
612 struct blk_plug *plug = current->plug;
613 struct request *rq;
614
615 if (!plug)
616 return NULL;
617
618 if (rq_list_empty(&plug->cached_rqs)) {
619 if (plug->nr_ios == 1)
620 return NULL;
621 rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
622 if (!rq)
623 return NULL;
624 } else {
625 rq = rq_list_peek(&plug->cached_rqs);
626 if (!rq || rq->q != q)
627 return NULL;
628
629 if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
630 return NULL;
631 if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
632 return NULL;
633
634 rq_list_pop(&plug->cached_rqs);
635 blk_mq_rq_time_init(rq, blk_time_get_ns());
636 }
637
638 rq->cmd_flags = opf;
639 INIT_LIST_HEAD(&rq->queuelist);
640 return rq;
641 }
642
643 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
644 blk_mq_req_flags_t flags)
645 {
646 struct request *rq;
647
648 rq = blk_mq_alloc_cached_request(q, opf, flags);
649 if (!rq) {
650 struct blk_mq_alloc_data data = {
651 .q = q,
652 .flags = flags,
653 .shallow_depth = 0,
654 .cmd_flags = opf,
655 .rq_flags = 0,
656 .nr_tags = 1,
657 .cached_rqs = NULL,
658 .ctx = NULL,
659 .hctx = NULL
660 };
661 int ret;
662
663 ret = blk_queue_enter(q, flags);
664 if (ret)
665 return ERR_PTR(ret);
666
667 rq = __blk_mq_alloc_requests(&data);
668 if (!rq)
669 goto out_queue_exit;
670 }
671 rq->__data_len = 0;
672 rq->__sector = (sector_t) -1;
673 rq->bio = rq->biotail = NULL;
674 return rq;
675 out_queue_exit:
676 blk_queue_exit(q);
677 return ERR_PTR(-EWOULDBLOCK);
678 }
679 EXPORT_SYMBOL(blk_mq_alloc_request);
680
681 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
682 blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
683 {
684 struct blk_mq_alloc_data data = {
685 .q = q,
686 .flags = flags,
687 .shallow_depth = 0,
688 .cmd_flags = opf,
689 .rq_flags = 0,
690 .nr_tags = 1,
691 .cached_rqs = NULL,
692 .ctx = NULL,
693 .hctx = NULL
694 };
695 u64 alloc_time_ns = 0;
696 struct request *rq;
697 unsigned int cpu;
698 unsigned int tag;
699 int ret;
700
701 /* alloc_time includes depth and tag waits */
702 if (blk_queue_rq_alloc_time(q))
703 alloc_time_ns = blk_time_get_ns();
704
705 /*
706 * If the tag allocator sleeps we could get an allocation for a
707 * different hardware context. No need to complicate the low level
708 * allocator for this for the rare use case of a command tied to
709 * a specific queue.
710 */
711 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) ||
712 WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED)))
713 return ERR_PTR(-EINVAL);
714
715 if (hctx_idx >= q->nr_hw_queues)
716 return ERR_PTR(-EIO);
717
718 ret = blk_queue_enter(q, flags);
719 if (ret)
720 return ERR_PTR(ret);
721
722 /*
723 * Check if the hardware context is actually mapped to anything.
724 * If not tell the caller that it should skip this queue.
725 */
726 ret = -EXDEV;
727 data.hctx = xa_load(&q->hctx_table, hctx_idx);
728 if (!blk_mq_hw_queue_mapped(data.hctx))
729 goto out_queue_exit;
730 cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
731 if (cpu >= nr_cpu_ids)
732 goto out_queue_exit;
733 data.ctx = __blk_mq_get_ctx(q, cpu);
734
735 if (q->elevator)
736 data.rq_flags |= RQF_SCHED_TAGS;
737 else
738 blk_mq_tag_busy(data.hctx);
739
740 if (flags & BLK_MQ_REQ_RESERVED)
741 data.rq_flags |= RQF_RESV;
742
743 ret = -EWOULDBLOCK;
744 tag = blk_mq_get_tag(&data);
745 if (tag == BLK_MQ_NO_TAG)
746 goto out_queue_exit;
747 if (!(data.rq_flags & RQF_SCHED_TAGS))
748 blk_mq_inc_active_requests(data.hctx);
749 rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
750 blk_mq_rq_time_init(rq, alloc_time_ns);
751 rq->__data_len = 0;
752 rq->__sector = (sector_t) -1;
753 rq->bio = rq->biotail = NULL;
754 return rq;
755
756 out_queue_exit:
757 blk_queue_exit(q);
758 return ERR_PTR(ret);
759 }
760 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
761
762 static void blk_mq_finish_request(struct request *rq)
763 {
764 struct request_queue *q = rq->q;
765
766 blk_zone_finish_request(rq);
767
768 if (rq->rq_flags & RQF_USE_SCHED) {
769 q->elevator->type->ops.finish_request(rq);
770 /*
771 * For postflush request that may need to be
772 * completed twice, we should clear this flag
773 * to avoid double finish_request() on the rq.
774 */
775 rq->rq_flags &= ~RQF_USE_SCHED;
776 }
777 }
778
779 static void __blk_mq_free_request(struct request *rq)
780 {
781 struct request_queue *q = rq->q;
782 struct blk_mq_ctx *ctx = rq->mq_ctx;
783 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
784 const int sched_tag = rq->internal_tag;
785
786 blk_crypto_free_request(rq);
787 blk_pm_mark_last_busy(rq);
788 rq->mq_hctx = NULL;
789
790 if (rq->tag != BLK_MQ_NO_TAG) {
791 blk_mq_dec_active_requests(hctx);
792 blk_mq_put_tag(hctx->tags, ctx, rq->tag);
793 }
794 if (sched_tag != BLK_MQ_NO_TAG)
795 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
796 blk_mq_sched_restart(hctx);
797 blk_queue_exit(q);
798 }
799
800 void blk_mq_free_request(struct request *rq)
801 {
802 struct request_queue *q = rq->q;
803
804 blk_mq_finish_request(rq);
805
806 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
807 laptop_io_completion(q->disk->bdi);
808
809 rq_qos_done(q, rq);
810
811 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
812 if (req_ref_put_and_test(rq))
813 __blk_mq_free_request(rq);
814 }
815 EXPORT_SYMBOL_GPL(blk_mq_free_request);
816
817 void blk_mq_free_plug_rqs(struct blk_plug *plug)
818 {
819 struct request *rq;
820
821 while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL)
822 blk_mq_free_request(rq);
823 }
824
825 void blk_dump_rq_flags(struct request *rq, char *msg)
826 {
827 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
828 rq->q->disk ? rq->q->disk->disk_name : "?",
829 (__force unsigned long long) rq->cmd_flags);
830
831 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
832 (unsigned long long)blk_rq_pos(rq),
833 blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
834 printk(KERN_INFO " bio %p, biotail %p, len %u\n",
835 rq->bio, rq->biotail, blk_rq_bytes(rq));
836 }
837 EXPORT_SYMBOL(blk_dump_rq_flags);
838
839 static void blk_account_io_completion(struct request *req, unsigned int bytes)
840 {
841 if (req->rq_flags & RQF_IO_STAT) {
842 const int sgrp = op_stat_group(req_op(req));
843
844 part_stat_lock();
845 part_stat_add(req->part, sectors[sgrp], bytes >> 9);
846 part_stat_unlock();
847 }
848 }
849
850 static void blk_print_req_error(struct request *req, blk_status_t status)
851 {
852 printk_ratelimited(KERN_ERR
853 "%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
854 "phys_seg %u prio class %u\n",
855 blk_status_to_str(status),
856 req->q->disk ? req->q->disk->disk_name : "?",
857 blk_rq_pos(req), (__force u32)req_op(req),
858 blk_op_str(req_op(req)),
859 (__force u32)(req->cmd_flags & ~REQ_OP_MASK),
860 req->nr_phys_segments,
861 IOPRIO_PRIO_CLASS(req_get_ioprio(req)));
862 }
863
864 /*
865 * Fully end IO on a request. Does not support partial completions, or
866 * errors.
867 */
868 static void blk_complete_request(struct request *req)
869 {
870 const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0;
871 int total_bytes = blk_rq_bytes(req);
872 struct bio *bio = req->bio;
873
874 trace_block_rq_complete(req, BLK_STS_OK, total_bytes);
875
876 if (!bio)
877 return;
878
879 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ)
880 blk_integrity_complete(req, total_bytes);
881
882 /*
883 * Upper layers may call blk_crypto_evict_key() anytime after the last
884 * bio_endio(). Therefore, the keyslot must be released before that.
885 */
886 blk_crypto_rq_put_keyslot(req);
887
888 blk_account_io_completion(req, total_bytes);
889
890 do {
891 struct bio *next = bio->bi_next;
892
893 /* Completion has already been traced */
894 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
895
896 if (blk_req_bio_is_zone_append(req, bio))
897 blk_zone_append_update_request_bio(req, bio);
898
899 if (!is_flush)
900 bio_endio(bio);
901 bio = next;
902 } while (bio);
903
904 /*
905 * Reset counters so that the request stacking driver
906 * can find how many bytes remain in the request
907 * later.
908 */
909 if (!req->end_io) {
910 req->bio = NULL;
911 req->__data_len = 0;
912 }
913 }
914
915 /**
916 * blk_update_request - Complete multiple bytes without completing the request
917 * @req: the request being processed
918 * @error: block status code
919 * @nr_bytes: number of bytes to complete for @req
920 *
921 * Description:
922 * Ends I/O on a number of bytes attached to @req, but doesn't complete
923 * the request structure even if @req doesn't have leftover.
924 * If @req has leftover, sets it up for the next range of segments.
925 *
926 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
927 * %false return from this function.
928 *
929 * Note:
930 * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
931 * except in the consistency check at the end of this function.
932 *
933 * Return:
934 * %false - this request doesn't have any more data
935 * %true - this request has more data
936 **/
937 bool blk_update_request(struct request *req, blk_status_t error,
938 unsigned int nr_bytes)
939 {
940 bool is_flush = req->rq_flags & RQF_FLUSH_SEQ;
941 bool quiet = req->rq_flags & RQF_QUIET;
942 int total_bytes;
943
944 trace_block_rq_complete(req, error, nr_bytes);
945
946 if (!req->bio)
947 return false;
948
949 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
950 error == BLK_STS_OK)
951 blk_integrity_complete(req, nr_bytes);
952
953 /*
954 * Upper layers may call blk_crypto_evict_key() anytime after the last
955 * bio_endio(). Therefore, the keyslot must be released before that.
956 */
957 if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))
958 __blk_crypto_rq_put_keyslot(req);
959
960 if (unlikely(error && !blk_rq_is_passthrough(req) && !quiet) &&
961 !test_bit(GD_DEAD, &req->q->disk->state)) {
962 blk_print_req_error(req, error);
963 trace_block_rq_error(req, error, nr_bytes);
964 }
965
966 blk_account_io_completion(req, nr_bytes);
967
968 total_bytes = 0;
969 while (req->bio) {
970 struct bio *bio = req->bio;
971 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
972
973 if (unlikely(error))
974 bio->bi_status = error;
975
976 if (bio_bytes == bio->bi_iter.bi_size) {
977 req->bio = bio->bi_next;
978 } else if (bio_is_zone_append(bio) && error == BLK_STS_OK) {
979 /*
980 * Partial zone append completions cannot be supported
981 * as the BIO fragments may end up not being written
982 * sequentially.
983 */
984 bio->bi_status = BLK_STS_IOERR;
985 }
986
987 /* Completion has already been traced */
988 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
989 if (unlikely(quiet))
990 bio_set_flag(bio, BIO_QUIET);
991
992 bio_advance(bio, bio_bytes);
993
994 /* Don't actually finish bio if it's part of flush sequence */
995 if (!bio->bi_iter.bi_size) {
996 if (blk_req_bio_is_zone_append(req, bio))
997 blk_zone_append_update_request_bio(req, bio);
998 if (!is_flush)
999 bio_endio(bio);
1000 }
1001
1002 total_bytes += bio_bytes;
1003 nr_bytes -= bio_bytes;
1004
1005 if (!nr_bytes)
1006 break;
1007 }
1008
1009 /*
1010 * completely done
1011 */
1012 if (!req->bio) {
1013 /*
1014 * Reset counters so that the request stacking driver
1015 * can find how many bytes remain in the request
1016 * later.
1017 */
1018 req->__data_len = 0;
1019 return false;
1020 }
1021
1022 req->__data_len -= total_bytes;
1023
1024 /* update sector only for requests with clear definition of sector */
1025 if (!blk_rq_is_passthrough(req))
1026 req->__sector += total_bytes >> 9;
1027
1028 /* mixed attributes always follow the first bio */
1029 if (req->rq_flags & RQF_MIXED_MERGE) {
1030 req->cmd_flags &= ~REQ_FAILFAST_MASK;
1031 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
1032 }
1033
1034 if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
1035 /*
1036 * If total number of sectors is less than the first segment
1037 * size, something has gone terribly wrong.
1038 */
1039 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
1040 blk_dump_rq_flags(req, "request botched");
1041 req->__data_len = blk_rq_cur_bytes(req);
1042 }
1043
1044 /* recalculate the number of segments */
1045 req->nr_phys_segments = blk_recalc_rq_segments(req);
1046 }
1047
1048 return true;
1049 }
1050 EXPORT_SYMBOL_GPL(blk_update_request);
1051
1052 static inline void blk_account_io_done(struct request *req, u64 now)
1053 {
1054 trace_block_io_done(req);
1055
1056 /*
1057 * Account IO completion. flush_rq isn't accounted as a
1058 * normal IO on queueing nor completion. Accounting the
1059 * containing request is enough.
1060 */
1061 if ((req->rq_flags & (RQF_IO_STAT|RQF_FLUSH_SEQ)) == RQF_IO_STAT) {
1062 const int sgrp = op_stat_group(req_op(req));
1063
1064 part_stat_lock();
1065 update_io_ticks(req->part, jiffies, true);
1066 part_stat_inc(req->part, ios[sgrp]);
1067 part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
1068 part_stat_local_dec(req->part,
1069 in_flight[op_is_write(req_op(req))]);
1070 part_stat_unlock();
1071 }
1072 }
1073
1074 static inline bool blk_rq_passthrough_stats(struct request *req)
1075 {
1076 struct bio *bio = req->bio;
1077
1078 if (!blk_queue_passthrough_stat(req->q))
1079 return false;
1080
1081 /* Requests without a bio do not transfer data. */
1082 if (!bio)
1083 return false;
1084
1085 /*
1086 * Stats are accumulated in the bdev, so must have one attached to a
1087 * bio to track stats. Most drivers do not set the bdev for passthrough
1088 * requests, but nvme is one that will set it.
1089 */
1090 if (!bio->bi_bdev)
1091 return false;
1092
1093 /*
1094 * We don't know what a passthrough command does, but we know the
1095 * payload size and data direction. Ensuring the size is aligned to the
1096 * block size filters out most commands with payloads that don't
1097 * represent sector access.
1098 */
1099 if (blk_rq_bytes(req) & (bdev_logical_block_size(bio->bi_bdev) - 1))
1100 return false;
1101 return true;
1102 }
1103
1104 static inline void blk_account_io_start(struct request *req)
1105 {
1106 trace_block_io_start(req);
1107
1108 if (!blk_queue_io_stat(req->q))
1109 return;
1110 if (blk_rq_is_passthrough(req) && !blk_rq_passthrough_stats(req))
1111 return;
1112
1113 req->rq_flags |= RQF_IO_STAT;
1114 req->start_time_ns = blk_time_get_ns();
1115
1116 /*
1117 * All non-passthrough requests are created from a bio with one
1118 * exception: when a flush command that is part of a flush sequence
1119 * generated by the state machine in blk-flush.c is cloned onto the
1120 * lower device by dm-multipath we can get here without a bio.
1121 */
1122 if (req->bio)
1123 req->part = req->bio->bi_bdev;
1124 else
1125 req->part = req->q->disk->part0;
1126
1127 part_stat_lock();
1128 update_io_ticks(req->part, jiffies, false);
1129 part_stat_local_inc(req->part, in_flight[op_is_write(req_op(req))]);
1130 part_stat_unlock();
1131 }
1132
1133 static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
1134 {
1135 if (rq->rq_flags & RQF_STATS)
1136 blk_stat_add(rq, now);
1137
1138 blk_mq_sched_completed_request(rq, now);
1139 blk_account_io_done(rq, now);
1140 }
1141
1142 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
1143 {
1144 if (blk_mq_need_time_stamp(rq))
1145 __blk_mq_end_request_acct(rq, blk_time_get_ns());
1146
1147 blk_mq_finish_request(rq);
1148
1149 if (rq->end_io) {
1150 rq_qos_done(rq->q, rq);
1151 if (rq->end_io(rq, error) == RQ_END_IO_FREE)
1152 blk_mq_free_request(rq);
1153 } else {
1154 blk_mq_free_request(rq);
1155 }
1156 }
1157 EXPORT_SYMBOL(__blk_mq_end_request);
1158
1159 void blk_mq_end_request(struct request *rq, blk_status_t error)
1160 {
1161 if (blk_update_request(rq, error, blk_rq_bytes(rq)))
1162 BUG();
1163 __blk_mq_end_request(rq, error);
1164 }
1165 EXPORT_SYMBOL(blk_mq_end_request);
1166
1167 #define TAG_COMP_BATCH 32
1168
1169 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
1170 int *tag_array, int nr_tags)
1171 {
1172 struct request_queue *q = hctx->queue;
1173
1174 blk_mq_sub_active_requests(hctx, nr_tags);
1175
1176 blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
1177 percpu_ref_put_many(&q->q_usage_counter, nr_tags);
1178 }
1179
1180 void blk_mq_end_request_batch(struct io_comp_batch *iob)
1181 {
1182 int tags[TAG_COMP_BATCH], nr_tags = 0;
1183 struct blk_mq_hw_ctx *cur_hctx = NULL;
1184 struct request *rq;
1185 u64 now = 0;
1186
1187 if (iob->need_ts)
1188 now = blk_time_get_ns();
1189
1190 while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
1191 prefetch(rq->bio);
1192 prefetch(rq->rq_next);
1193
1194 blk_complete_request(rq);
1195 if (iob->need_ts)
1196 __blk_mq_end_request_acct(rq, now);
1197
1198 blk_mq_finish_request(rq);
1199
1200 rq_qos_done(rq->q, rq);
1201
1202 /*
1203 * If end_io handler returns NONE, then it still has
1204 * ownership of the request.
1205 */
1206 if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE)
1207 continue;
1208
1209 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1210 if (!req_ref_put_and_test(rq))
1211 continue;
1212
1213 blk_crypto_free_request(rq);
1214 blk_pm_mark_last_busy(rq);
1215
1216 if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) {
1217 if (cur_hctx)
1218 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1219 nr_tags = 0;
1220 cur_hctx = rq->mq_hctx;
1221 }
1222 tags[nr_tags++] = rq->tag;
1223 }
1224
1225 if (nr_tags)
1226 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1227 }
1228 EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
1229
1230 static void blk_complete_reqs(struct llist_head *list)
1231 {
1232 struct llist_node *entry = llist_reverse_order(llist_del_all(list));
1233 struct request *rq, *next;
1234
1235 llist_for_each_entry_safe(rq, next, entry, ipi_list)
1236 rq->q->mq_ops->complete(rq);
1237 }
1238
1239 static __latent_entropy void blk_done_softirq(void)
1240 {
1241 blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
1242 }
1243
1244 static int blk_softirq_cpu_dead(unsigned int cpu)
1245 {
1246 blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
1247 return 0;
1248 }
1249
1250 static void __blk_mq_complete_request_remote(void *data)
1251 {
1252 __raise_softirq_irqoff(BLOCK_SOFTIRQ);
1253 }
1254
1255 static inline bool blk_mq_complete_need_ipi(struct request *rq)
1256 {
1257 int cpu = raw_smp_processor_id();
1258
1259 if (!IS_ENABLED(CONFIG_SMP) ||
1260 !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
1261 return false;
1262 /*
1263 * With force threaded interrupts enabled, raising softirq from an SMP
1264 * function call will always result in waking the ksoftirqd thread.
1265 * This is probably worse than completing the request on a different
1266 * cache domain.
1267 */
1268 if (force_irqthreads())
1269 return false;
1270
1271 /* same CPU or cache domain and capacity? Complete locally */
1272 if (cpu == rq->mq_ctx->cpu ||
1273 (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
1274 cpus_share_cache(cpu, rq->mq_ctx->cpu) &&
1275 cpus_equal_capacity(cpu, rq->mq_ctx->cpu)))
1276 return false;
1277
1278 /* don't try to IPI to an offline CPU */
1279 return cpu_online(rq->mq_ctx->cpu);
1280 }
1281
1282 static void blk_mq_complete_send_ipi(struct request *rq)
1283 {
1284 unsigned int cpu;
1285
1286 cpu = rq->mq_ctx->cpu;
1287 if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu)))
1288 smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu));
1289 }
1290
1291 static void blk_mq_raise_softirq(struct request *rq)
1292 {
1293 struct llist_head *list;
1294
1295 preempt_disable();
1296 list = this_cpu_ptr(&blk_cpu_done);
1297 if (llist_add(&rq->ipi_list, list))
1298 raise_softirq(BLOCK_SOFTIRQ);
1299 preempt_enable();
1300 }
1301
1302 bool blk_mq_complete_request_remote(struct request *rq)
1303 {
1304 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
1305
1306 /*
1307 * For request which hctx has only one ctx mapping,
1308 * or a polled request, always complete locally,
1309 * it's pointless to redirect the completion.
1310 */
1311 if ((rq->mq_hctx->nr_ctx == 1 &&
1312 rq->mq_ctx->cpu == raw_smp_processor_id()) ||
1313 rq->cmd_flags & REQ_POLLED)
1314 return false;
1315
1316 if (blk_mq_complete_need_ipi(rq)) {
1317 blk_mq_complete_send_ipi(rq);
1318 return true;
1319 }
1320
1321 if (rq->q->nr_hw_queues == 1) {
1322 blk_mq_raise_softirq(rq);
1323 return true;
1324 }
1325 return false;
1326 }
1327 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
1328
1329 /**
1330 * blk_mq_complete_request - end I/O on a request
1331 * @rq: the request being processed
1332 *
1333 * Description:
1334 * Complete a request by scheduling the ->complete_rq operation.
1335 **/
1336 void blk_mq_complete_request(struct request *rq)
1337 {
1338 if (!blk_mq_complete_request_remote(rq))
1339 rq->q->mq_ops->complete(rq);
1340 }
1341 EXPORT_SYMBOL(blk_mq_complete_request);
1342
1343 /**
1344 * blk_mq_start_request - Start processing a request
1345 * @rq: Pointer to request to be started
1346 *
1347 * Function used by device drivers to notify the block layer that a request
1348 * is going to be processed now, so blk layer can do proper initializations
1349 * such as starting the timeout timer.
1350 */
1351 void blk_mq_start_request(struct request *rq)
1352 {
1353 struct request_queue *q = rq->q;
1354
1355 trace_block_rq_issue(rq);
1356
1357 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags) &&
1358 !blk_rq_is_passthrough(rq)) {
1359 rq->io_start_time_ns = blk_time_get_ns();
1360 rq->stats_sectors = blk_rq_sectors(rq);
1361 rq->rq_flags |= RQF_STATS;
1362 rq_qos_issue(q, rq);
1363 }
1364
1365 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
1366
1367 blk_add_timer(rq);
1368 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
1369 rq->mq_hctx->tags->rqs[rq->tag] = rq;
1370
1371 if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
1372 blk_integrity_prepare(rq);
1373
1374 if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
1375 WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num);
1376 }
1377 EXPORT_SYMBOL(blk_mq_start_request);
1378
1379 /*
1380 * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
1381 * queues. This is important for md arrays to benefit from merging
1382 * requests.
1383 */
1384 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
1385 {
1386 if (plug->multiple_queues)
1387 return BLK_MAX_REQUEST_COUNT * 2;
1388 return BLK_MAX_REQUEST_COUNT;
1389 }
1390
1391 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
1392 {
1393 struct request *last = rq_list_peek(&plug->mq_list);
1394
1395 if (!plug->rq_count) {
1396 trace_block_plug(rq->q);
1397 } else if (plug->rq_count >= blk_plug_max_rq_count(plug) ||
1398 (!blk_queue_nomerges(rq->q) &&
1399 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1400 blk_mq_flush_plug_list(plug, false);
1401 last = NULL;
1402 trace_block_plug(rq->q);
1403 }
1404
1405 if (!plug->multiple_queues && last && last->q != rq->q)
1406 plug->multiple_queues = true;
1407 /*
1408 * Any request allocated from sched tags can't be issued to
1409 * ->queue_rqs() directly
1410 */
1411 if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
1412 plug->has_elevator = true;
1413 rq_list_add_tail(&plug->mq_list, rq);
1414 plug->rq_count++;
1415 }
1416
1417 /**
1418 * blk_execute_rq_nowait - insert a request to I/O scheduler for execution
1419 * @rq: request to insert
1420 * @at_head: insert request at head or tail of queue
1421 *
1422 * Description:
1423 * Insert a fully prepared request at the back of the I/O scheduler queue
1424 * for execution. Don't wait for completion.
1425 *
1426 * Note:
1427 * This function will invoke @done directly if the queue is dead.
1428 */
1429 void blk_execute_rq_nowait(struct request *rq, bool at_head)
1430 {
1431 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1432
1433 WARN_ON(irqs_disabled());
1434 WARN_ON(!blk_rq_is_passthrough(rq));
1435
1436 blk_account_io_start(rq);
1437
1438 if (current->plug && !at_head) {
1439 blk_add_rq_to_plug(current->plug, rq);
1440 return;
1441 }
1442
1443 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1444 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
1445 }
1446 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
1447
1448 struct blk_rq_wait {
1449 struct completion done;
1450 blk_status_t ret;
1451 };
1452
1453 static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret)
1454 {
1455 struct blk_rq_wait *wait = rq->end_io_data;
1456
1457 wait->ret = ret;
1458 complete(&wait->done);
1459 return RQ_END_IO_NONE;
1460 }
1461
1462 bool blk_rq_is_poll(struct request *rq)
1463 {
1464 if (!rq->mq_hctx)
1465 return false;
1466 if (rq->mq_hctx->type != HCTX_TYPE_POLL)
1467 return false;
1468 return true;
1469 }
1470 EXPORT_SYMBOL_GPL(blk_rq_is_poll);
1471
1472 static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
1473 {
1474 do {
1475 blk_hctx_poll(rq->q, rq->mq_hctx, NULL, 0);
1476 cond_resched();
1477 } while (!completion_done(wait));
1478 }
1479
1480 /**
1481 * blk_execute_rq - insert a request into queue for execution
1482 * @rq: request to insert
1483 * @at_head: insert request at head or tail of queue
1484 *
1485 * Description:
1486 * Insert a fully prepared request at the back of the I/O scheduler queue
1487 * for execution and wait for completion.
1488 * Return: The blk_status_t result provided to blk_mq_end_request().
1489 */
1490 blk_status_t blk_execute_rq(struct request *rq, bool at_head)
1491 {
1492 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1493 struct blk_rq_wait wait = {
1494 .done = COMPLETION_INITIALIZER_ONSTACK(wait.done),
1495 };
1496
1497 WARN_ON(irqs_disabled());
1498 WARN_ON(!blk_rq_is_passthrough(rq));
1499
1500 rq->end_io_data = &wait;
1501 rq->end_io = blk_end_sync_rq;
1502
1503 blk_account_io_start(rq);
1504 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1505 blk_mq_run_hw_queue(hctx, false);
1506
1507 if (blk_rq_is_poll(rq))
1508 blk_rq_poll_completion(rq, &wait.done);
1509 else
1510 blk_wait_io(&wait.done);
1511
1512 return wait.ret;
1513 }
1514 EXPORT_SYMBOL(blk_execute_rq);
1515
1516 static void __blk_mq_requeue_request(struct request *rq)
1517 {
1518 struct request_queue *q = rq->q;
1519
1520 blk_mq_put_driver_tag(rq);
1521
1522 trace_block_rq_requeue(rq);
1523 rq_qos_requeue(q, rq);
1524
1525 if (blk_mq_request_started(rq)) {
1526 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1527 rq->rq_flags &= ~RQF_TIMED_OUT;
1528 }
1529 }
1530
1531 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
1532 {
1533 struct request_queue *q = rq->q;
1534 unsigned long flags;
1535
1536 __blk_mq_requeue_request(rq);
1537
1538 /* this request will be re-inserted to io scheduler queue */
1539 blk_mq_sched_requeue_request(rq);
1540
1541 spin_lock_irqsave(&q->requeue_lock, flags);
1542 list_add_tail(&rq->queuelist, &q->requeue_list);
1543 spin_unlock_irqrestore(&q->requeue_lock, flags);
1544
1545 if (kick_requeue_list)
1546 blk_mq_kick_requeue_list(q);
1547 }
1548 EXPORT_SYMBOL(blk_mq_requeue_request);
1549
1550 static void blk_mq_requeue_work(struct work_struct *work)
1551 {
1552 struct request_queue *q =
1553 container_of(work, struct request_queue, requeue_work.work);
1554 LIST_HEAD(rq_list);
1555 LIST_HEAD(flush_list);
1556 struct request *rq;
1557
1558 spin_lock_irq(&q->requeue_lock);
1559 list_splice_init(&q->requeue_list, &rq_list);
1560 list_splice_init(&q->flush_list, &flush_list);
1561 spin_unlock_irq(&q->requeue_lock);
1562
1563 while (!list_empty(&rq_list)) {
1564 rq = list_entry(rq_list.next, struct request, queuelist);
1565 list_del_init(&rq->queuelist);
1566 /*
1567 * If RQF_DONTPREP is set, the request has been started by the
1568 * driver already and might have driver-specific data allocated
1569 * already. Insert it into the hctx dispatch list to avoid
1570 * block layer merges for the request.
1571 */
1572 if (rq->rq_flags & RQF_DONTPREP)
1573 blk_mq_request_bypass_insert(rq, 0);
1574 else
1575 blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD);
1576 }
1577
1578 while (!list_empty(&flush_list)) {
1579 rq = list_entry(flush_list.next, struct request, queuelist);
1580 list_del_init(&rq->queuelist);
1581 blk_mq_insert_request(rq, 0);
1582 }
1583
1584 blk_mq_run_hw_queues(q, false);
1585 }
1586
1587 void blk_mq_kick_requeue_list(struct request_queue *q)
1588 {
1589 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
1590 }
1591 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
1592
1593 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
1594 unsigned long msecs)
1595 {
1596 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
1597 msecs_to_jiffies(msecs));
1598 }
1599 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
1600
1601 static bool blk_is_flush_data_rq(struct request *rq)
1602 {
1603 return (rq->rq_flags & RQF_FLUSH_SEQ) && !is_flush_rq(rq);
1604 }
1605
1606 static bool blk_mq_rq_inflight(struct request *rq, void *priv)
1607 {
1608 /*
1609 * If we find a request that isn't idle we know the queue is busy
1610 * as it's checked in the iter.
1611 * Return false to stop the iteration.
1612 *
1613 * In case of queue quiesce, if one flush data request is completed,
1614 * don't count it as inflight given the flush sequence is suspended,
1615 * and the original flush data request is invisible to driver, just
1616 * like other pending requests because of quiesce
1617 */
1618 if (blk_mq_request_started(rq) && !(blk_queue_quiesced(rq->q) &&
1619 blk_is_flush_data_rq(rq) &&
1620 blk_mq_request_completed(rq))) {
1621 bool *busy = priv;
1622
1623 *busy = true;
1624 return false;
1625 }
1626
1627 return true;
1628 }
1629
1630 bool blk_mq_queue_inflight(struct request_queue *q)
1631 {
1632 bool busy = false;
1633
1634 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
1635 return busy;
1636 }
1637 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
1638
1639 static void blk_mq_rq_timed_out(struct request *req)
1640 {
1641 req->rq_flags |= RQF_TIMED_OUT;
1642 if (req->q->mq_ops->timeout) {
1643 enum blk_eh_timer_return ret;
1644
1645 ret = req->q->mq_ops->timeout(req);
1646 if (ret == BLK_EH_DONE)
1647 return;
1648 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
1649 }
1650
1651 blk_add_timer(req);
1652 }
1653
1654 struct blk_expired_data {
1655 bool has_timedout_rq;
1656 unsigned long next;
1657 unsigned long timeout_start;
1658 };
1659
1660 static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired)
1661 {
1662 unsigned long deadline;
1663
1664 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
1665 return false;
1666 if (rq->rq_flags & RQF_TIMED_OUT)
1667 return false;
1668
1669 deadline = READ_ONCE(rq->deadline);
1670 if (time_after_eq(expired->timeout_start, deadline))
1671 return true;
1672
1673 if (expired->next == 0)
1674 expired->next = deadline;
1675 else if (time_after(expired->next, deadline))
1676 expired->next = deadline;
1677 return false;
1678 }
1679
1680 void blk_mq_put_rq_ref(struct request *rq)
1681 {
1682 if (is_flush_rq(rq)) {
1683 if (rq->end_io(rq, 0) == RQ_END_IO_FREE)
1684 blk_mq_free_request(rq);
1685 } else if (req_ref_put_and_test(rq)) {
1686 __blk_mq_free_request(rq);
1687 }
1688 }
1689
1690 static bool blk_mq_check_expired(struct request *rq, void *priv)
1691 {
1692 struct blk_expired_data *expired = priv;
1693
1694 /*
1695 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
1696 * be reallocated underneath the timeout handler's processing, then
1697 * the expire check is reliable. If the request is not expired, then
1698 * it was completed and reallocated as a new request after returning
1699 * from blk_mq_check_expired().
1700 */
1701 if (blk_mq_req_expired(rq, expired)) {
1702 expired->has_timedout_rq = true;
1703 return false;
1704 }
1705 return true;
1706 }
1707
1708 static bool blk_mq_handle_expired(struct request *rq, void *priv)
1709 {
1710 struct blk_expired_data *expired = priv;
1711
1712 if (blk_mq_req_expired(rq, expired))
1713 blk_mq_rq_timed_out(rq);
1714 return true;
1715 }
1716
1717 static void blk_mq_timeout_work(struct work_struct *work)
1718 {
1719 struct request_queue *q =
1720 container_of(work, struct request_queue, timeout_work);
1721 struct blk_expired_data expired = {
1722 .timeout_start = jiffies,
1723 };
1724 struct blk_mq_hw_ctx *hctx;
1725 unsigned long i;
1726
1727 /* A deadlock might occur if a request is stuck requiring a
1728 * timeout at the same time a queue freeze is waiting
1729 * completion, since the timeout code would not be able to
1730 * acquire the queue reference here.
1731 *
1732 * That's why we don't use blk_queue_enter here; instead, we use
1733 * percpu_ref_tryget directly, because we need to be able to
1734 * obtain a reference even in the short window between the queue
1735 * starting to freeze, by dropping the first reference in
1736 * blk_freeze_queue_start, and the moment the last request is
1737 * consumed, marked by the instant q_usage_counter reaches
1738 * zero.
1739 */
1740 if (!percpu_ref_tryget(&q->q_usage_counter))
1741 return;
1742
1743 /* check if there is any timed-out request */
1744 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
1745 if (expired.has_timedout_rq) {
1746 /*
1747 * Before walking tags, we must ensure any submit started
1748 * before the current time has finished. Since the submit
1749 * uses srcu or rcu, wait for a synchronization point to
1750 * ensure all running submits have finished
1751 */
1752 blk_mq_wait_quiesce_done(q->tag_set);
1753
1754 expired.next = 0;
1755 blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired);
1756 }
1757
1758 if (expired.next != 0) {
1759 mod_timer(&q->timeout, expired.next);
1760 } else {
1761 /*
1762 * Request timeouts are handled as a forward rolling timer. If
1763 * we end up here it means that no requests are pending and
1764 * also that no request has been pending for a while. Mark
1765 * each hctx as idle.
1766 */
1767 queue_for_each_hw_ctx(q, hctx, i) {
1768 /* the hctx may be unmapped, so check it here */
1769 if (blk_mq_hw_queue_mapped(hctx))
1770 blk_mq_tag_idle(hctx);
1771 }
1772 }
1773 blk_queue_exit(q);
1774 }
1775
1776 struct flush_busy_ctx_data {
1777 struct blk_mq_hw_ctx *hctx;
1778 struct list_head *list;
1779 };
1780
1781 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
1782 {
1783 struct flush_busy_ctx_data *flush_data = data;
1784 struct blk_mq_hw_ctx *hctx = flush_data->hctx;
1785 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1786 enum hctx_type type = hctx->type;
1787
1788 spin_lock(&ctx->lock);
1789 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
1790 sbitmap_clear_bit(sb, bitnr);
1791 spin_unlock(&ctx->lock);
1792 return true;
1793 }
1794
1795 /*
1796 * Process software queues that have been marked busy, splicing them
1797 * to the for-dispatch
1798 */
1799 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1800 {
1801 struct flush_busy_ctx_data data = {
1802 .hctx = hctx,
1803 .list = list,
1804 };
1805
1806 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1807 }
1808
1809 struct dispatch_rq_data {
1810 struct blk_mq_hw_ctx *hctx;
1811 struct request *rq;
1812 };
1813
1814 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
1815 void *data)
1816 {
1817 struct dispatch_rq_data *dispatch_data = data;
1818 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
1819 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1820 enum hctx_type type = hctx->type;
1821
1822 spin_lock(&ctx->lock);
1823 if (!list_empty(&ctx->rq_lists[type])) {
1824 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1825 list_del_init(&dispatch_data->rq->queuelist);
1826 if (list_empty(&ctx->rq_lists[type]))
1827 sbitmap_clear_bit(sb, bitnr);
1828 }
1829 spin_unlock(&ctx->lock);
1830
1831 return !dispatch_data->rq;
1832 }
1833
1834 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
1835 struct blk_mq_ctx *start)
1836 {
1837 unsigned off = start ? start->index_hw[hctx->type] : 0;
1838 struct dispatch_rq_data data = {
1839 .hctx = hctx,
1840 .rq = NULL,
1841 };
1842
1843 __sbitmap_for_each_set(&hctx->ctx_map, off,
1844 dispatch_rq_from_ctx, &data);
1845
1846 return data.rq;
1847 }
1848
1849 bool __blk_mq_alloc_driver_tag(struct request *rq)
1850 {
1851 struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
1852 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1853 int tag;
1854
1855 blk_mq_tag_busy(rq->mq_hctx);
1856
1857 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1858 bt = &rq->mq_hctx->tags->breserved_tags;
1859 tag_offset = 0;
1860 } else {
1861 if (!hctx_may_queue(rq->mq_hctx, bt))
1862 return false;
1863 }
1864
1865 tag = __sbitmap_queue_get(bt);
1866 if (tag == BLK_MQ_NO_TAG)
1867 return false;
1868
1869 rq->tag = tag + tag_offset;
1870 blk_mq_inc_active_requests(rq->mq_hctx);
1871 return true;
1872 }
1873
1874 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1875 int flags, void *key)
1876 {
1877 struct blk_mq_hw_ctx *hctx;
1878
1879 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1880
1881 spin_lock(&hctx->dispatch_wait_lock);
1882 if (!list_empty(&wait->entry)) {
1883 struct sbitmap_queue *sbq;
1884
1885 list_del_init(&wait->entry);
1886 sbq = &hctx->tags->bitmap_tags;
1887 atomic_dec(&sbq->ws_active);
1888 }
1889 spin_unlock(&hctx->dispatch_wait_lock);
1890
1891 blk_mq_run_hw_queue(hctx, true);
1892 return 1;
1893 }
1894
1895 /*
1896 * Mark us waiting for a tag. For shared tags, this involves hooking us into
1897 * the tag wakeups. For non-shared tags, we can simply mark us needing a
1898 * restart. For both cases, take care to check the condition again after
1899 * marking us as waiting.
1900 */
1901 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1902 struct request *rq)
1903 {
1904 struct sbitmap_queue *sbq;
1905 struct wait_queue_head *wq;
1906 wait_queue_entry_t *wait;
1907 bool ret;
1908
1909 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1910 !(blk_mq_is_shared_tags(hctx->flags))) {
1911 blk_mq_sched_mark_restart_hctx(hctx);
1912
1913 /*
1914 * It's possible that a tag was freed in the window between the
1915 * allocation failure and adding the hardware queue to the wait
1916 * queue.
1917 *
1918 * Don't clear RESTART here, someone else could have set it.
1919 * At most this will cost an extra queue run.
1920 */
1921 return blk_mq_get_driver_tag(rq);
1922 }
1923
1924 wait = &hctx->dispatch_wait;
1925 if (!list_empty_careful(&wait->entry))
1926 return false;
1927
1928 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag))
1929 sbq = &hctx->tags->breserved_tags;
1930 else
1931 sbq = &hctx->tags->bitmap_tags;
1932 wq = &bt_wait_ptr(sbq, hctx)->wait;
1933
1934 spin_lock_irq(&wq->lock);
1935 spin_lock(&hctx->dispatch_wait_lock);
1936 if (!list_empty(&wait->entry)) {
1937 spin_unlock(&hctx->dispatch_wait_lock);
1938 spin_unlock_irq(&wq->lock);
1939 return false;
1940 }
1941
1942 atomic_inc(&sbq->ws_active);
1943 wait->flags &= ~WQ_FLAG_EXCLUSIVE;
1944 __add_wait_queue(wq, wait);
1945
1946 /*
1947 * Add one explicit barrier since blk_mq_get_driver_tag() may
1948 * not imply barrier in case of failure.
1949 *
1950 * Order adding us to wait queue and allocating driver tag.
1951 *
1952 * The pair is the one implied in sbitmap_queue_wake_up() which
1953 * orders clearing sbitmap tag bits and waitqueue_active() in
1954 * __sbitmap_queue_wake_up(), since waitqueue_active() is lockless
1955 *
1956 * Otherwise, re-order of adding wait queue and getting driver tag
1957 * may cause __sbitmap_queue_wake_up() to wake up nothing because
1958 * the waitqueue_active() may not observe us in wait queue.
1959 */
1960 smp_mb();
1961
1962 /*
1963 * It's possible that a tag was freed in the window between the
1964 * allocation failure and adding the hardware queue to the wait
1965 * queue.
1966 */
1967 ret = blk_mq_get_driver_tag(rq);
1968 if (!ret) {
1969 spin_unlock(&hctx->dispatch_wait_lock);
1970 spin_unlock_irq(&wq->lock);
1971 return false;
1972 }
1973
1974 /*
1975 * We got a tag, remove ourselves from the wait queue to ensure
1976 * someone else gets the wakeup.
1977 */
1978 list_del_init(&wait->entry);
1979 atomic_dec(&sbq->ws_active);
1980 spin_unlock(&hctx->dispatch_wait_lock);
1981 spin_unlock_irq(&wq->lock);
1982
1983 return true;
1984 }
1985
1986 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8
1987 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4
1988 /*
1989 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
1990 * - EWMA is one simple way to compute running average value
1991 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
1992 * - take 4 as factor for avoiding to get too small(0) result, and this
1993 * factor doesn't matter because EWMA decreases exponentially
1994 */
1995 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
1996 {
1997 unsigned int ewma;
1998
1999 ewma = hctx->dispatch_busy;
2000
2001 if (!ewma && !busy)
2002 return;
2003
2004 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
2005 if (busy)
2006 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
2007 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
2008
2009 hctx->dispatch_busy = ewma;
2010 }
2011
2012 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */
2013
2014 static void blk_mq_handle_dev_resource(struct request *rq,
2015 struct list_head *list)
2016 {
2017 list_add(&rq->queuelist, list);
2018 __blk_mq_requeue_request(rq);
2019 }
2020
2021 enum prep_dispatch {
2022 PREP_DISPATCH_OK,
2023 PREP_DISPATCH_NO_TAG,
2024 PREP_DISPATCH_NO_BUDGET,
2025 };
2026
2027 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
2028 bool need_budget)
2029 {
2030 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2031 int budget_token = -1;
2032
2033 if (need_budget) {
2034 budget_token = blk_mq_get_dispatch_budget(rq->q);
2035 if (budget_token < 0) {
2036 blk_mq_put_driver_tag(rq);
2037 return PREP_DISPATCH_NO_BUDGET;
2038 }
2039 blk_mq_set_rq_budget_token(rq, budget_token);
2040 }
2041
2042 if (!blk_mq_get_driver_tag(rq)) {
2043 /*
2044 * The initial allocation attempt failed, so we need to
2045 * rerun the hardware queue when a tag is freed. The
2046 * waitqueue takes care of that. If the queue is run
2047 * before we add this entry back on the dispatch list,
2048 * we'll re-run it below.
2049 */
2050 if (!blk_mq_mark_tag_wait(hctx, rq)) {
2051 /*
2052 * All budgets not got from this function will be put
2053 * together during handling partial dispatch
2054 */
2055 if (need_budget)
2056 blk_mq_put_dispatch_budget(rq->q, budget_token);
2057 return PREP_DISPATCH_NO_TAG;
2058 }
2059 }
2060
2061 return PREP_DISPATCH_OK;
2062 }
2063
2064 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
2065 static void blk_mq_release_budgets(struct request_queue *q,
2066 struct list_head *list)
2067 {
2068 struct request *rq;
2069
2070 list_for_each_entry(rq, list, queuelist) {
2071 int budget_token = blk_mq_get_rq_budget_token(rq);
2072
2073 if (budget_token >= 0)
2074 blk_mq_put_dispatch_budget(q, budget_token);
2075 }
2076 }
2077
2078 /*
2079 * blk_mq_commit_rqs will notify driver using bd->last that there is no
2080 * more requests. (See comment in struct blk_mq_ops for commit_rqs for
2081 * details)
2082 * Attention, we should explicitly call this in unusual cases:
2083 * 1) did not queue everything initially scheduled to queue
2084 * 2) the last attempt to queue a request failed
2085 */
2086 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,
2087 bool from_schedule)
2088 {
2089 if (hctx->queue->mq_ops->commit_rqs && queued) {
2090 trace_block_unplug(hctx->queue, queued, !from_schedule);
2091 hctx->queue->mq_ops->commit_rqs(hctx);
2092 }
2093 }
2094
2095 /*
2096 * Returns true if we did some work AND can potentially do more.
2097 */
2098 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
2099 bool get_budget)
2100 {
2101 enum prep_dispatch prep;
2102 struct request_queue *q = hctx->queue;
2103 struct request *rq;
2104 int queued;
2105 blk_status_t ret = BLK_STS_OK;
2106 bool needs_resource = false;
2107
2108 if (list_empty(list))
2109 return false;
2110
2111 /*
2112 * Now process all the entries, sending them to the driver.
2113 */
2114 queued = 0;
2115 do {
2116 struct blk_mq_queue_data bd;
2117
2118 rq = list_first_entry(list, struct request, queuelist);
2119
2120 WARN_ON_ONCE(hctx != rq->mq_hctx);
2121 prep = blk_mq_prep_dispatch_rq(rq, get_budget);
2122 if (prep != PREP_DISPATCH_OK)
2123 break;
2124
2125 list_del_init(&rq->queuelist);
2126
2127 bd.rq = rq;
2128 bd.last = list_empty(list);
2129
2130 ret = q->mq_ops->queue_rq(hctx, &bd);
2131 switch (ret) {
2132 case BLK_STS_OK:
2133 queued++;
2134 break;
2135 case BLK_STS_RESOURCE:
2136 needs_resource = true;
2137 fallthrough;
2138 case BLK_STS_DEV_RESOURCE:
2139 blk_mq_handle_dev_resource(rq, list);
2140 goto out;
2141 default:
2142 blk_mq_end_request(rq, ret);
2143 }
2144 } while (!list_empty(list));
2145 out:
2146 /* If we didn't flush the entire list, we could have told the driver
2147 * there was more coming, but that turned out to be a lie.
2148 */
2149 if (!list_empty(list) || ret != BLK_STS_OK)
2150 blk_mq_commit_rqs(hctx, queued, false);
2151
2152 /*
2153 * Any items that need requeuing? Stuff them into hctx->dispatch,
2154 * that is where we will continue on next queue run.
2155 */
2156 if (!list_empty(list)) {
2157 bool needs_restart;
2158 /* For non-shared tags, the RESTART check will suffice */
2159 bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
2160 ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) ||
2161 blk_mq_is_shared_tags(hctx->flags));
2162
2163 /*
2164 * If the caller allocated budgets, free the budgets of the
2165 * requests that have not yet been passed to the block driver.
2166 */
2167 if (!get_budget)
2168 blk_mq_release_budgets(q, list);
2169
2170 spin_lock(&hctx->lock);
2171 list_splice_tail_init(list, &hctx->dispatch);
2172 spin_unlock(&hctx->lock);
2173
2174 /*
2175 * Order adding requests to hctx->dispatch and checking
2176 * SCHED_RESTART flag. The pair of this smp_mb() is the one
2177 * in blk_mq_sched_restart(). Avoid restart code path to
2178 * miss the new added requests to hctx->dispatch, meantime
2179 * SCHED_RESTART is observed here.
2180 */
2181 smp_mb();
2182
2183 /*
2184 * If SCHED_RESTART was set by the caller of this function and
2185 * it is no longer set that means that it was cleared by another
2186 * thread and hence that a queue rerun is needed.
2187 *
2188 * If 'no_tag' is set, that means that we failed getting
2189 * a driver tag with an I/O scheduler attached. If our dispatch
2190 * waitqueue is no longer active, ensure that we run the queue
2191 * AFTER adding our entries back to the list.
2192 *
2193 * If no I/O scheduler has been configured it is possible that
2194 * the hardware queue got stopped and restarted before requests
2195 * were pushed back onto the dispatch list. Rerun the queue to
2196 * avoid starvation. Notes:
2197 * - blk_mq_run_hw_queue() checks whether or not a queue has
2198 * been stopped before rerunning a queue.
2199 * - Some but not all block drivers stop a queue before
2200 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
2201 * and dm-rq.
2202 *
2203 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
2204 * bit is set, run queue after a delay to avoid IO stalls
2205 * that could otherwise occur if the queue is idle. We'll do
2206 * similar if we couldn't get budget or couldn't lock a zone
2207 * and SCHED_RESTART is set.
2208 */
2209 needs_restart = blk_mq_sched_needs_restart(hctx);
2210 if (prep == PREP_DISPATCH_NO_BUDGET)
2211 needs_resource = true;
2212 if (!needs_restart ||
2213 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
2214 blk_mq_run_hw_queue(hctx, true);
2215 else if (needs_resource)
2216 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
2217
2218 blk_mq_update_dispatch_busy(hctx, true);
2219 return false;
2220 }
2221
2222 blk_mq_update_dispatch_busy(hctx, false);
2223 return true;
2224 }
2225
2226 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
2227 {
2228 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
2229
2230 if (cpu >= nr_cpu_ids)
2231 cpu = cpumask_first(hctx->cpumask);
2232 return cpu;
2233 }
2234
2235 /*
2236 * ->next_cpu is always calculated from hctx->cpumask, so simply use
2237 * it for speeding up the check
2238 */
2239 static bool blk_mq_hctx_empty_cpumask(struct blk_mq_hw_ctx *hctx)
2240 {
2241 return hctx->next_cpu >= nr_cpu_ids;
2242 }
2243
2244 /*
2245 * It'd be great if the workqueue API had a way to pass
2246 * in a mask and had some smarts for more clever placement.
2247 * For now we just round-robin here, switching for every
2248 * BLK_MQ_CPU_WORK_BATCH queued items.
2249 */
2250 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
2251 {
2252 bool tried = false;
2253 int next_cpu = hctx->next_cpu;
2254
2255 /* Switch to unbound if no allowable CPUs in this hctx */
2256 if (hctx->queue->nr_hw_queues == 1 || blk_mq_hctx_empty_cpumask(hctx))
2257 return WORK_CPU_UNBOUND;
2258
2259 if (--hctx->next_cpu_batch <= 0) {
2260 select_cpu:
2261 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
2262 cpu_online_mask);
2263 if (next_cpu >= nr_cpu_ids)
2264 next_cpu = blk_mq_first_mapped_cpu(hctx);
2265 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2266 }
2267
2268 /*
2269 * Do unbound schedule if we can't find a online CPU for this hctx,
2270 * and it should only happen in the path of handling CPU DEAD.
2271 */
2272 if (!cpu_online(next_cpu)) {
2273 if (!tried) {
2274 tried = true;
2275 goto select_cpu;
2276 }
2277
2278 /*
2279 * Make sure to re-select CPU next time once after CPUs
2280 * in hctx->cpumask become online again.
2281 */
2282 hctx->next_cpu = next_cpu;
2283 hctx->next_cpu_batch = 1;
2284 return WORK_CPU_UNBOUND;
2285 }
2286
2287 hctx->next_cpu = next_cpu;
2288 return next_cpu;
2289 }
2290
2291 /**
2292 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
2293 * @hctx: Pointer to the hardware queue to run.
2294 * @msecs: Milliseconds of delay to wait before running the queue.
2295 *
2296 * Run a hardware queue asynchronously with a delay of @msecs.
2297 */
2298 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
2299 {
2300 if (unlikely(blk_mq_hctx_stopped(hctx)))
2301 return;
2302 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
2303 msecs_to_jiffies(msecs));
2304 }
2305 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
2306
2307 static inline bool blk_mq_hw_queue_need_run(struct blk_mq_hw_ctx *hctx)
2308 {
2309 bool need_run;
2310
2311 /*
2312 * When queue is quiesced, we may be switching io scheduler, or
2313 * updating nr_hw_queues, or other things, and we can't run queue
2314 * any more, even blk_mq_hctx_has_pending() can't be called safely.
2315 *
2316 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
2317 * quiesced.
2318 */
2319 __blk_mq_run_dispatch_ops(hctx->queue, false,
2320 need_run = !blk_queue_quiesced(hctx->queue) &&
2321 blk_mq_hctx_has_pending(hctx));
2322 return need_run;
2323 }
2324
2325 /**
2326 * blk_mq_run_hw_queue - Start to run a hardware queue.
2327 * @hctx: Pointer to the hardware queue to run.
2328 * @async: If we want to run the queue asynchronously.
2329 *
2330 * Check if the request queue is not in a quiesced state and if there are
2331 * pending requests to be sent. If this is true, run the queue to send requests
2332 * to hardware.
2333 */
2334 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2335 {
2336 bool need_run;
2337
2338 /*
2339 * We can't run the queue inline with interrupts disabled.
2340 */
2341 WARN_ON_ONCE(!async && in_interrupt());
2342
2343 might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING);
2344
2345 need_run = blk_mq_hw_queue_need_run(hctx);
2346 if (!need_run) {
2347 unsigned long flags;
2348
2349 /*
2350 * Synchronize with blk_mq_unquiesce_queue(), because we check
2351 * if hw queue is quiesced locklessly above, we need the use
2352 * ->queue_lock to make sure we see the up-to-date status to
2353 * not miss rerunning the hw queue.
2354 */
2355 spin_lock_irqsave(&hctx->queue->queue_lock, flags);
2356 need_run = blk_mq_hw_queue_need_run(hctx);
2357 spin_unlock_irqrestore(&hctx->queue->queue_lock, flags);
2358
2359 if (!need_run)
2360 return;
2361 }
2362
2363 if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
2364 blk_mq_delay_run_hw_queue(hctx, 0);
2365 return;
2366 }
2367
2368 blk_mq_run_dispatch_ops(hctx->queue,
2369 blk_mq_sched_dispatch_requests(hctx));
2370 }
2371 EXPORT_SYMBOL(blk_mq_run_hw_queue);
2372
2373 /*
2374 * Return prefered queue to dispatch from (if any) for non-mq aware IO
2375 * scheduler.
2376 */
2377 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
2378 {
2379 struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
2380 /*
2381 * If the IO scheduler does not respect hardware queues when
2382 * dispatching, we just don't bother with multiple HW queues and
2383 * dispatch from hctx for the current CPU since running multiple queues
2384 * just causes lock contention inside the scheduler and pointless cache
2385 * bouncing.
2386 */
2387 struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
2388
2389 if (!blk_mq_hctx_stopped(hctx))
2390 return hctx;
2391 return NULL;
2392 }
2393
2394 /**
2395 * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
2396 * @q: Pointer to the request queue to run.
2397 * @async: If we want to run the queue asynchronously.
2398 */
2399 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
2400 {
2401 struct blk_mq_hw_ctx *hctx, *sq_hctx;
2402 unsigned long i;
2403
2404 sq_hctx = NULL;
2405 if (blk_queue_sq_sched(q))
2406 sq_hctx = blk_mq_get_sq_hctx(q);
2407 queue_for_each_hw_ctx(q, hctx, i) {
2408 if (blk_mq_hctx_stopped(hctx))
2409 continue;
2410 /*
2411 * Dispatch from this hctx either if there's no hctx preferred
2412 * by IO scheduler or if it has requests that bypass the
2413 * scheduler.
2414 */
2415 if (!sq_hctx || sq_hctx == hctx ||
2416 !list_empty_careful(&hctx->dispatch))
2417 blk_mq_run_hw_queue(hctx, async);
2418 }
2419 }
2420 EXPORT_SYMBOL(blk_mq_run_hw_queues);
2421
2422 /**
2423 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
2424 * @q: Pointer to the request queue to run.
2425 * @msecs: Milliseconds of delay to wait before running the queues.
2426 */
2427 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
2428 {
2429 struct blk_mq_hw_ctx *hctx, *sq_hctx;
2430 unsigned long i;
2431
2432 sq_hctx = NULL;
2433 if (blk_queue_sq_sched(q))
2434 sq_hctx = blk_mq_get_sq_hctx(q);
2435 queue_for_each_hw_ctx(q, hctx, i) {
2436 if (blk_mq_hctx_stopped(hctx))
2437 continue;
2438 /*
2439 * If there is already a run_work pending, leave the
2440 * pending delay untouched. Otherwise, a hctx can stall
2441 * if another hctx is re-delaying the other's work
2442 * before the work executes.
2443 */
2444 if (delayed_work_pending(&hctx->run_work))
2445 continue;
2446 /*
2447 * Dispatch from this hctx either if there's no hctx preferred
2448 * by IO scheduler or if it has requests that bypass the
2449 * scheduler.
2450 */
2451 if (!sq_hctx || sq_hctx == hctx ||
2452 !list_empty_careful(&hctx->dispatch))
2453 blk_mq_delay_run_hw_queue(hctx, msecs);
2454 }
2455 }
2456 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
2457
2458 /*
2459 * This function is often used for pausing .queue_rq() by driver when
2460 * there isn't enough resource or some conditions aren't satisfied, and
2461 * BLK_STS_RESOURCE is usually returned.
2462 *
2463 * We do not guarantee that dispatch can be drained or blocked
2464 * after blk_mq_stop_hw_queue() returns. Please use
2465 * blk_mq_quiesce_queue() for that requirement.
2466 */
2467 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
2468 {
2469 cancel_delayed_work(&hctx->run_work);
2470
2471 set_bit(BLK_MQ_S_STOPPED, &hctx->state);
2472 }
2473 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
2474
2475 /*
2476 * This function is often used for pausing .queue_rq() by driver when
2477 * there isn't enough resource or some conditions aren't satisfied, and
2478 * BLK_STS_RESOURCE is usually returned.
2479 *
2480 * We do not guarantee that dispatch can be drained or blocked
2481 * after blk_mq_stop_hw_queues() returns. Please use
2482 * blk_mq_quiesce_queue() for that requirement.
2483 */
2484 void blk_mq_stop_hw_queues(struct request_queue *q)
2485 {
2486 struct blk_mq_hw_ctx *hctx;
2487 unsigned long i;
2488
2489 queue_for_each_hw_ctx(q, hctx, i)
2490 blk_mq_stop_hw_queue(hctx);
2491 }
2492 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
2493
2494 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
2495 {
2496 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2497
2498 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
2499 }
2500 EXPORT_SYMBOL(blk_mq_start_hw_queue);
2501
2502 void blk_mq_start_hw_queues(struct request_queue *q)
2503 {
2504 struct blk_mq_hw_ctx *hctx;
2505 unsigned long i;
2506
2507 queue_for_each_hw_ctx(q, hctx, i)
2508 blk_mq_start_hw_queue(hctx);
2509 }
2510 EXPORT_SYMBOL(blk_mq_start_hw_queues);
2511
2512 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2513 {
2514 if (!blk_mq_hctx_stopped(hctx))
2515 return;
2516
2517 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2518 /*
2519 * Pairs with the smp_mb() in blk_mq_hctx_stopped() to order the
2520 * clearing of BLK_MQ_S_STOPPED above and the checking of dispatch
2521 * list in the subsequent routine.
2522 */
2523 smp_mb__after_atomic();
2524 blk_mq_run_hw_queue(hctx, async);
2525 }
2526 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
2527
2528 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
2529 {
2530 struct blk_mq_hw_ctx *hctx;
2531 unsigned long i;
2532
2533 queue_for_each_hw_ctx(q, hctx, i)
2534 blk_mq_start_stopped_hw_queue(hctx, async ||
2535 (hctx->flags & BLK_MQ_F_BLOCKING));
2536 }
2537 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
2538
2539 static void blk_mq_run_work_fn(struct work_struct *work)
2540 {
2541 struct blk_mq_hw_ctx *hctx =
2542 container_of(work, struct blk_mq_hw_ctx, run_work.work);
2543
2544 blk_mq_run_dispatch_ops(hctx->queue,
2545 blk_mq_sched_dispatch_requests(hctx));
2546 }
2547
2548 /**
2549 * blk_mq_request_bypass_insert - Insert a request at dispatch list.
2550 * @rq: Pointer to request to be inserted.
2551 * @flags: BLK_MQ_INSERT_*
2552 *
2553 * Should only be used carefully, when the caller knows we want to
2554 * bypass a potential IO scheduler on the target device.
2555 */
2556 static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags)
2557 {
2558 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2559
2560 spin_lock(&hctx->lock);
2561 if (flags & BLK_MQ_INSERT_AT_HEAD)
2562 list_add(&rq->queuelist, &hctx->dispatch);
2563 else
2564 list_add_tail(&rq->queuelist, &hctx->dispatch);
2565 spin_unlock(&hctx->lock);
2566 }
2567
2568 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
2569 struct blk_mq_ctx *ctx, struct list_head *list,
2570 bool run_queue_async)
2571 {
2572 struct request *rq;
2573 enum hctx_type type = hctx->type;
2574
2575 /*
2576 * Try to issue requests directly if the hw queue isn't busy to save an
2577 * extra enqueue & dequeue to the sw queue.
2578 */
2579 if (!hctx->dispatch_busy && !run_queue_async) {
2580 blk_mq_run_dispatch_ops(hctx->queue,
2581 blk_mq_try_issue_list_directly(hctx, list));
2582 if (list_empty(list))
2583 goto out;
2584 }
2585
2586 /*
2587 * preemption doesn't flush plug list, so it's possible ctx->cpu is
2588 * offline now
2589 */
2590 list_for_each_entry(rq, list, queuelist) {
2591 BUG_ON(rq->mq_ctx != ctx);
2592 trace_block_rq_insert(rq);
2593 if (rq->cmd_flags & REQ_NOWAIT)
2594 run_queue_async = true;
2595 }
2596
2597 spin_lock(&ctx->lock);
2598 list_splice_tail_init(list, &ctx->rq_lists[type]);
2599 blk_mq_hctx_mark_pending(hctx, ctx);
2600 spin_unlock(&ctx->lock);
2601 out:
2602 blk_mq_run_hw_queue(hctx, run_queue_async);
2603 }
2604
2605 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)
2606 {
2607 struct request_queue *q = rq->q;
2608 struct blk_mq_ctx *ctx = rq->mq_ctx;
2609 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2610
2611 if (blk_rq_is_passthrough(rq)) {
2612 /*
2613 * Passthrough request have to be added to hctx->dispatch
2614 * directly. The device may be in a situation where it can't
2615 * handle FS request, and always returns BLK_STS_RESOURCE for
2616 * them, which gets them added to hctx->dispatch.
2617 *
2618 * If a passthrough request is required to unblock the queues,
2619 * and it is added to the scheduler queue, there is no chance to
2620 * dispatch it given we prioritize requests in hctx->dispatch.
2621 */
2622 blk_mq_request_bypass_insert(rq, flags);
2623 } else if (req_op(rq) == REQ_OP_FLUSH) {
2624 /*
2625 * Firstly normal IO request is inserted to scheduler queue or
2626 * sw queue, meantime we add flush request to dispatch queue(
2627 * hctx->dispatch) directly and there is at most one in-flight
2628 * flush request for each hw queue, so it doesn't matter to add
2629 * flush request to tail or front of the dispatch queue.
2630 *
2631 * Secondly in case of NCQ, flush request belongs to non-NCQ
2632 * command, and queueing it will fail when there is any
2633 * in-flight normal IO request(NCQ command). When adding flush
2634 * rq to the front of hctx->dispatch, it is easier to introduce
2635 * extra time to flush rq's latency because of S_SCHED_RESTART
2636 * compared with adding to the tail of dispatch queue, then
2637 * chance of flush merge is increased, and less flush requests
2638 * will be issued to controller. It is observed that ~10% time
2639 * is saved in blktests block/004 on disk attached to AHCI/NCQ
2640 * drive when adding flush rq to the front of hctx->dispatch.
2641 *
2642 * Simply queue flush rq to the front of hctx->dispatch so that
2643 * intensive flush workloads can benefit in case of NCQ HW.
2644 */
2645 blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD);
2646 } else if (q->elevator) {
2647 LIST_HEAD(list);
2648
2649 WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG);
2650
2651 list_add(&rq->queuelist, &list);
2652 q->elevator->type->ops.insert_requests(hctx, &list, flags);
2653 } else {
2654 trace_block_rq_insert(rq);
2655
2656 spin_lock(&ctx->lock);
2657 if (flags & BLK_MQ_INSERT_AT_HEAD)
2658 list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
2659 else
2660 list_add_tail(&rq->queuelist,
2661 &ctx->rq_lists[hctx->type]);
2662 blk_mq_hctx_mark_pending(hctx, ctx);
2663 spin_unlock(&ctx->lock);
2664 }
2665 }
2666
2667 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
2668 unsigned int nr_segs)
2669 {
2670 int err;
2671
2672 if (bio->bi_opf & REQ_RAHEAD)
2673 rq->cmd_flags |= REQ_FAILFAST_MASK;
2674
2675 rq->bio = rq->biotail = bio;
2676 rq->__sector = bio->bi_iter.bi_sector;
2677 rq->__data_len = bio->bi_iter.bi_size;
2678 rq->nr_phys_segments = nr_segs;
2679 if (bio_integrity(bio))
2680 rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q,
2681 bio);
2682
2683 /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
2684 err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
2685 WARN_ON_ONCE(err);
2686
2687 blk_account_io_start(rq);
2688 }
2689
2690 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
2691 struct request *rq, bool last)
2692 {
2693 struct request_queue *q = rq->q;
2694 struct blk_mq_queue_data bd = {
2695 .rq = rq,
2696 .last = last,
2697 };
2698 blk_status_t ret;
2699
2700 /*
2701 * For OK queue, we are done. For error, caller may kill it.
2702 * Any other error (busy), just add it to our list as we
2703 * previously would have done.
2704 */
2705 ret = q->mq_ops->queue_rq(hctx, &bd);
2706 switch (ret) {
2707 case BLK_STS_OK:
2708 blk_mq_update_dispatch_busy(hctx, false);
2709 break;
2710 case BLK_STS_RESOURCE:
2711 case BLK_STS_DEV_RESOURCE:
2712 blk_mq_update_dispatch_busy(hctx, true);
2713 __blk_mq_requeue_request(rq);
2714 break;
2715 default:
2716 blk_mq_update_dispatch_busy(hctx, false);
2717 break;
2718 }
2719
2720 return ret;
2721 }
2722
2723 static bool blk_mq_get_budget_and_tag(struct request *rq)
2724 {
2725 int budget_token;
2726
2727 budget_token = blk_mq_get_dispatch_budget(rq->q);
2728 if (budget_token < 0)
2729 return false;
2730 blk_mq_set_rq_budget_token(rq, budget_token);
2731 if (!blk_mq_get_driver_tag(rq)) {
2732 blk_mq_put_dispatch_budget(rq->q, budget_token);
2733 return false;
2734 }
2735 return true;
2736 }
2737
2738 /**
2739 * blk_mq_try_issue_directly - Try to send a request directly to device driver.
2740 * @hctx: Pointer of the associated hardware queue.
2741 * @rq: Pointer to request to be sent.
2742 *
2743 * If the device has enough resources to accept a new request now, send the
2744 * request directly to device driver. Else, insert at hctx->dispatch queue, so
2745 * we can try send it another time in the future. Requests inserted at this
2746 * queue have higher priority.
2747 */
2748 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2749 struct request *rq)
2750 {
2751 blk_status_t ret;
2752
2753 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2754 blk_mq_insert_request(rq, 0);
2755 blk_mq_run_hw_queue(hctx, false);
2756 return;
2757 }
2758
2759 if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) {
2760 blk_mq_insert_request(rq, 0);
2761 blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT);
2762 return;
2763 }
2764
2765 ret = __blk_mq_issue_directly(hctx, rq, true);
2766 switch (ret) {
2767 case BLK_STS_OK:
2768 break;
2769 case BLK_STS_RESOURCE:
2770 case BLK_STS_DEV_RESOURCE:
2771 blk_mq_request_bypass_insert(rq, 0);
2772 blk_mq_run_hw_queue(hctx, false);
2773 break;
2774 default:
2775 blk_mq_end_request(rq, ret);
2776 break;
2777 }
2778 }
2779
2780 static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
2781 {
2782 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2783
2784 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2785 blk_mq_insert_request(rq, 0);
2786 blk_mq_run_hw_queue(hctx, false);
2787 return BLK_STS_OK;
2788 }
2789
2790 if (!blk_mq_get_budget_and_tag(rq))
2791 return BLK_STS_RESOURCE;
2792 return __blk_mq_issue_directly(hctx, rq, last);
2793 }
2794
2795 static void blk_mq_issue_direct(struct rq_list *rqs)
2796 {
2797 struct blk_mq_hw_ctx *hctx = NULL;
2798 struct request *rq;
2799 int queued = 0;
2800 blk_status_t ret = BLK_STS_OK;
2801
2802 while ((rq = rq_list_pop(rqs))) {
2803 bool last = rq_list_empty(rqs);
2804
2805 if (hctx != rq->mq_hctx) {
2806 if (hctx) {
2807 blk_mq_commit_rqs(hctx, queued, false);
2808 queued = 0;
2809 }
2810 hctx = rq->mq_hctx;
2811 }
2812
2813 ret = blk_mq_request_issue_directly(rq, last);
2814 switch (ret) {
2815 case BLK_STS_OK:
2816 queued++;
2817 break;
2818 case BLK_STS_RESOURCE:
2819 case BLK_STS_DEV_RESOURCE:
2820 blk_mq_request_bypass_insert(rq, 0);
2821 blk_mq_run_hw_queue(hctx, false);
2822 goto out;
2823 default:
2824 blk_mq_end_request(rq, ret);
2825 break;
2826 }
2827 }
2828
2829 out:
2830 if (ret != BLK_STS_OK)
2831 blk_mq_commit_rqs(hctx, queued, false);
2832 }
2833
2834 static void __blk_mq_flush_list(struct request_queue *q, struct rq_list *rqs)
2835 {
2836 if (blk_queue_quiesced(q))
2837 return;
2838 q->mq_ops->queue_rqs(rqs);
2839 }
2840
2841 static unsigned blk_mq_extract_queue_requests(struct rq_list *rqs,
2842 struct rq_list *queue_rqs)
2843 {
2844 struct request *rq = rq_list_pop(rqs);
2845 struct request_queue *this_q = rq->q;
2846 struct request **prev = &rqs->head;
2847 struct rq_list matched_rqs = {};
2848 struct request *last = NULL;
2849 unsigned depth = 1;
2850
2851 rq_list_add_tail(&matched_rqs, rq);
2852 while ((rq = *prev)) {
2853 if (rq->q == this_q) {
2854 /* move rq from rqs to matched_rqs */
2855 *prev = rq->rq_next;
2856 rq_list_add_tail(&matched_rqs, rq);
2857 depth++;
2858 } else {
2859 /* leave rq in rqs */
2860 prev = &rq->rq_next;
2861 last = rq;
2862 }
2863 }
2864
2865 rqs->tail = last;
2866 *queue_rqs = matched_rqs;
2867 return depth;
2868 }
2869
2870 static void blk_mq_dispatch_queue_requests(struct rq_list *rqs, unsigned depth)
2871 {
2872 struct request_queue *q = rq_list_peek(rqs)->q;
2873
2874 trace_block_unplug(q, depth, true);
2875
2876 /*
2877 * Peek first request and see if we have a ->queue_rqs() hook.
2878 * If we do, we can dispatch the whole list in one go.
2879 * We already know at this point that all requests belong to the
2880 * same queue, caller must ensure that's the case.
2881 */
2882 if (q->mq_ops->queue_rqs) {
2883 blk_mq_run_dispatch_ops(q, __blk_mq_flush_list(q, rqs));
2884 if (rq_list_empty(rqs))
2885 return;
2886 }
2887
2888 blk_mq_run_dispatch_ops(q, blk_mq_issue_direct(rqs));
2889 }
2890
2891 static void blk_mq_dispatch_list(struct rq_list *rqs, bool from_sched)
2892 {
2893 struct blk_mq_hw_ctx *this_hctx = NULL;
2894 struct blk_mq_ctx *this_ctx = NULL;
2895 struct rq_list requeue_list = {};
2896 unsigned int depth = 0;
2897 bool is_passthrough = false;
2898 LIST_HEAD(list);
2899
2900 do {
2901 struct request *rq = rq_list_pop(rqs);
2902
2903 if (!this_hctx) {
2904 this_hctx = rq->mq_hctx;
2905 this_ctx = rq->mq_ctx;
2906 is_passthrough = blk_rq_is_passthrough(rq);
2907 } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
2908 is_passthrough != blk_rq_is_passthrough(rq)) {
2909 rq_list_add_tail(&requeue_list, rq);
2910 continue;
2911 }
2912 list_add_tail(&rq->queuelist, &list);
2913 depth++;
2914 } while (!rq_list_empty(rqs));
2915
2916 *rqs = requeue_list;
2917 trace_block_unplug(this_hctx->queue, depth, !from_sched);
2918
2919 percpu_ref_get(&this_hctx->queue->q_usage_counter);
2920 /* passthrough requests should never be issued to the I/O scheduler */
2921 if (is_passthrough) {
2922 spin_lock(&this_hctx->lock);
2923 list_splice_tail_init(&list, &this_hctx->dispatch);
2924 spin_unlock(&this_hctx->lock);
2925 blk_mq_run_hw_queue(this_hctx, from_sched);
2926 } else if (this_hctx->queue->elevator) {
2927 this_hctx->queue->elevator->type->ops.insert_requests(this_hctx,
2928 &list, 0);
2929 blk_mq_run_hw_queue(this_hctx, from_sched);
2930 } else {
2931 blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched);
2932 }
2933 percpu_ref_put(&this_hctx->queue->q_usage_counter);
2934 }
2935
2936 static void blk_mq_dispatch_multiple_queue_requests(struct rq_list *rqs)
2937 {
2938 do {
2939 struct rq_list queue_rqs;
2940 unsigned depth;
2941
2942 depth = blk_mq_extract_queue_requests(rqs, &queue_rqs);
2943 blk_mq_dispatch_queue_requests(&queue_rqs, depth);
2944 while (!rq_list_empty(&queue_rqs))
2945 blk_mq_dispatch_list(&queue_rqs, false);
2946 } while (!rq_list_empty(rqs));
2947 }
2948
2949 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2950 {
2951 unsigned int depth;
2952
2953 /*
2954 * We may have been called recursively midway through handling
2955 * plug->mq_list via a schedule() in the driver's queue_rq() callback.
2956 * To avoid mq_list changing under our feet, clear rq_count early and
2957 * bail out specifically if rq_count is 0 rather than checking
2958 * whether the mq_list is empty.
2959 */
2960 if (plug->rq_count == 0)
2961 return;
2962 depth = plug->rq_count;
2963 plug->rq_count = 0;
2964
2965 if (!plug->has_elevator && !from_schedule) {
2966 if (plug->multiple_queues) {
2967 blk_mq_dispatch_multiple_queue_requests(&plug->mq_list);
2968 return;
2969 }
2970
2971 blk_mq_dispatch_queue_requests(&plug->mq_list, depth);
2972 if (rq_list_empty(&plug->mq_list))
2973 return;
2974 }
2975
2976 do {
2977 blk_mq_dispatch_list(&plug->mq_list, from_schedule);
2978 } while (!rq_list_empty(&plug->mq_list));
2979 }
2980
2981 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2982 struct list_head *list)
2983 {
2984 int queued = 0;
2985 blk_status_t ret = BLK_STS_OK;
2986
2987 while (!list_empty(list)) {
2988 struct request *rq = list_first_entry(list, struct request,
2989 queuelist);
2990
2991 list_del_init(&rq->queuelist);
2992 ret = blk_mq_request_issue_directly(rq, list_empty(list));
2993 switch (ret) {
2994 case BLK_STS_OK:
2995 queued++;
2996 break;
2997 case BLK_STS_RESOURCE:
2998 case BLK_STS_DEV_RESOURCE:
2999 blk_mq_request_bypass_insert(rq, 0);
3000 if (list_empty(list))
3001 blk_mq_run_hw_queue(hctx, false);
3002 goto out;
3003 default:
3004 blk_mq_end_request(rq, ret);
3005 break;
3006 }
3007 }
3008
3009 out:
3010 if (ret != BLK_STS_OK)
3011 blk_mq_commit_rqs(hctx, queued, false);
3012 }
3013
3014 static bool blk_mq_attempt_bio_merge(struct request_queue *q,
3015 struct bio *bio, unsigned int nr_segs)
3016 {
3017 if (!blk_queue_nomerges(q) && bio_mergeable(bio)) {
3018 if (blk_attempt_plug_merge(q, bio, nr_segs))
3019 return true;
3020 if (blk_mq_sched_bio_merge(q, bio, nr_segs))
3021 return true;
3022 }
3023 return false;
3024 }
3025
3026 static struct request *blk_mq_get_new_requests(struct request_queue *q,
3027 struct blk_plug *plug,
3028 struct bio *bio)
3029 {
3030 struct blk_mq_alloc_data data = {
3031 .q = q,
3032 .flags = 0,
3033 .shallow_depth = 0,
3034 .cmd_flags = bio->bi_opf,
3035 .rq_flags = 0,
3036 .nr_tags = 1,
3037 .cached_rqs = NULL,
3038 .ctx = NULL,
3039 .hctx = NULL
3040 };
3041 struct request *rq;
3042
3043 rq_qos_throttle(q, bio);
3044
3045 if (plug) {
3046 data.nr_tags = plug->nr_ios;
3047 plug->nr_ios = 1;
3048 data.cached_rqs = &plug->cached_rqs;
3049 }
3050
3051 rq = __blk_mq_alloc_requests(&data);
3052 if (unlikely(!rq))
3053 rq_qos_cleanup(q, bio);
3054 return rq;
3055 }
3056
3057 /*
3058 * Check if there is a suitable cached request and return it.
3059 */
3060 static struct request *blk_mq_peek_cached_request(struct blk_plug *plug,
3061 struct request_queue *q, blk_opf_t opf)
3062 {
3063 enum hctx_type type = blk_mq_get_hctx_type(opf);
3064 struct request *rq;
3065
3066 if (!plug)
3067 return NULL;
3068 rq = rq_list_peek(&plug->cached_rqs);
3069 if (!rq || rq->q != q)
3070 return NULL;
3071 if (type != rq->mq_hctx->type &&
3072 (type != HCTX_TYPE_READ || rq->mq_hctx->type != HCTX_TYPE_DEFAULT))
3073 return NULL;
3074 if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
3075 return NULL;
3076 return rq;
3077 }
3078
3079 static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,
3080 struct bio *bio)
3081 {
3082 if (rq_list_pop(&plug->cached_rqs) != rq)
3083 WARN_ON_ONCE(1);
3084
3085 /*
3086 * If any qos ->throttle() end up blocking, we will have flushed the
3087 * plug and hence killed the cached_rq list as well. Pop this entry
3088 * before we throttle.
3089 */
3090 rq_qos_throttle(rq->q, bio);
3091
3092 blk_mq_rq_time_init(rq, blk_time_get_ns());
3093 rq->cmd_flags = bio->bi_opf;
3094 INIT_LIST_HEAD(&rq->queuelist);
3095 }
3096
3097 static bool bio_unaligned(const struct bio *bio, struct request_queue *q)
3098 {
3099 unsigned int bs_mask = queue_logical_block_size(q) - 1;
3100
3101 /* .bi_sector of any zero sized bio need to be initialized */
3102 if ((bio->bi_iter.bi_size & bs_mask) ||
3103 ((bio->bi_iter.bi_sector << SECTOR_SHIFT) & bs_mask))
3104 return true;
3105 return false;
3106 }
3107
3108 /**
3109 * blk_mq_submit_bio - Create and send a request to block device.
3110 * @bio: Bio pointer.
3111 *
3112 * Builds up a request structure from @q and @bio and send to the device. The
3113 * request may not be queued directly to hardware if:
3114 * * This request can be merged with another one
3115 * * We want to place request at plug queue for possible future merging
3116 * * There is an IO scheduler active at this queue
3117 *
3118 * It will not queue the request if there is an error with the bio, or at the
3119 * request creation.
3120 */
3121 void blk_mq_submit_bio(struct bio *bio)
3122 {
3123 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
3124 struct blk_plug *plug = current->plug;
3125 const int is_sync = op_is_sync(bio->bi_opf);
3126 struct blk_mq_hw_ctx *hctx;
3127 unsigned int nr_segs;
3128 struct request *rq;
3129 blk_status_t ret;
3130
3131 /*
3132 * If the plug has a cached request for this queue, try to use it.
3133 */
3134 rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf);
3135
3136 /*
3137 * A BIO that was released from a zone write plug has already been
3138 * through the preparation in this function, already holds a reference
3139 * on the queue usage counter, and is the only write BIO in-flight for
3140 * the target zone. Go straight to preparing a request for it.
3141 */
3142 if (bio_zone_write_plugging(bio)) {
3143 nr_segs = bio->__bi_nr_segments;
3144 if (rq)
3145 blk_queue_exit(q);
3146 goto new_request;
3147 }
3148
3149 /*
3150 * The cached request already holds a q_usage_counter reference and we
3151 * don't have to acquire a new one if we use it.
3152 */
3153 if (!rq) {
3154 if (unlikely(bio_queue_enter(bio)))
3155 return;
3156 }
3157
3158 /*
3159 * Device reconfiguration may change logical block size or reduce the
3160 * number of poll queues, so the checks for alignment and poll support
3161 * have to be done with queue usage counter held.
3162 */
3163 if (unlikely(bio_unaligned(bio, q))) {
3164 bio_io_error(bio);
3165 goto queue_exit;
3166 }
3167
3168 if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) {
3169 bio->bi_status = BLK_STS_NOTSUPP;
3170 bio_endio(bio);
3171 goto queue_exit;
3172 }
3173
3174 bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
3175 if (!bio)
3176 goto queue_exit;
3177
3178 if (!bio_integrity_prep(bio))
3179 goto queue_exit;
3180
3181 blk_mq_bio_issue_init(q, bio);
3182 if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
3183 goto queue_exit;
3184
3185 if (bio_needs_zone_write_plugging(bio)) {
3186 if (blk_zone_plug_bio(bio, nr_segs))
3187 goto queue_exit;
3188 }
3189
3190 new_request:
3191 if (rq) {
3192 blk_mq_use_cached_rq(rq, plug, bio);
3193 } else {
3194 rq = blk_mq_get_new_requests(q, plug, bio);
3195 if (unlikely(!rq)) {
3196 if (bio->bi_opf & REQ_NOWAIT)
3197 bio_wouldblock_error(bio);
3198 goto queue_exit;
3199 }
3200 }
3201
3202 trace_block_getrq(bio);
3203
3204 rq_qos_track(q, rq, bio);
3205
3206 blk_mq_bio_to_request(rq, bio, nr_segs);
3207
3208 ret = blk_crypto_rq_get_keyslot(rq);
3209 if (ret != BLK_STS_OK) {
3210 bio->bi_status = ret;
3211 bio_endio(bio);
3212 blk_mq_free_request(rq);
3213 return;
3214 }
3215
3216 if (bio_zone_write_plugging(bio))
3217 blk_zone_write_plug_init_request(rq);
3218
3219 if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
3220 return;
3221
3222 if (plug) {
3223 blk_add_rq_to_plug(plug, rq);
3224 return;
3225 }
3226
3227 hctx = rq->mq_hctx;
3228 if ((rq->rq_flags & RQF_USE_SCHED) ||
3229 (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) {
3230 blk_mq_insert_request(rq, 0);
3231 blk_mq_run_hw_queue(hctx, true);
3232 } else {
3233 blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq));
3234 }
3235 return;
3236
3237 queue_exit:
3238 /*
3239 * Don't drop the queue reference if we were trying to use a cached
3240 * request and thus didn't acquire one.
3241 */
3242 if (!rq)
3243 blk_queue_exit(q);
3244 }
3245
3246 #ifdef CONFIG_BLK_MQ_STACKING
3247 /**
3248 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
3249 * @rq: the request being queued
3250 */
3251 blk_status_t blk_insert_cloned_request(struct request *rq)
3252 {
3253 struct request_queue *q = rq->q;
3254 unsigned int max_sectors = blk_queue_get_max_sectors(rq);
3255 unsigned int max_segments = blk_rq_get_max_segments(rq);
3256 blk_status_t ret;
3257
3258 if (blk_rq_sectors(rq) > max_sectors) {
3259 /*
3260 * SCSI device does not have a good way to return if
3261 * Write Same/Zero is actually supported. If a device rejects
3262 * a non-read/write command (discard, write same,etc.) the
3263 * low-level device driver will set the relevant queue limit to
3264 * 0 to prevent blk-lib from issuing more of the offending
3265 * operations. Commands queued prior to the queue limit being
3266 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
3267 * errors being propagated to upper layers.
3268 */
3269 if (max_sectors == 0)
3270 return BLK_STS_NOTSUPP;
3271
3272 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
3273 __func__, blk_rq_sectors(rq), max_sectors);
3274 return BLK_STS_IOERR;
3275 }
3276
3277 /*
3278 * The queue settings related to segment counting may differ from the
3279 * original queue.
3280 */
3281 rq->nr_phys_segments = blk_recalc_rq_segments(rq);
3282 if (rq->nr_phys_segments > max_segments) {
3283 printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n",
3284 __func__, rq->nr_phys_segments, max_segments);
3285 return BLK_STS_IOERR;
3286 }
3287
3288 /*
3289 * Integrity segment counting depends on the same queue limits
3290 * (virt_boundary_mask, seg_boundary_mask, max_segment_size) that
3291 * vary across stacked queues, so recompute against the bottom
3292 * queue just like nr_phys_segments above.
3293 */
3294 if (blk_integrity_rq(rq) && rq->bio) {
3295 unsigned short max_int_segs = queue_max_integrity_segments(q);
3296
3297 rq->nr_integrity_segments =
3298 blk_rq_count_integrity_sg(rq->q, rq->bio);
3299 if (rq->nr_integrity_segments > max_int_segs) {
3300 printk(KERN_ERR "%s: over max integrity segments limit. (%u > %u)\n",
3301 __func__, rq->nr_integrity_segments,
3302 max_int_segs);
3303 return BLK_STS_IOERR;
3304 }
3305 }
3306
3307 if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
3308 return BLK_STS_IOERR;
3309
3310 ret = blk_crypto_rq_get_keyslot(rq);
3311 if (ret != BLK_STS_OK)
3312 return ret;
3313
3314 blk_account_io_start(rq);
3315
3316 /*
3317 * Since we have a scheduler attached on the top device,
3318 * bypass a potential scheduler on the bottom device for
3319 * insert.
3320 */
3321 blk_mq_run_dispatch_ops(q,
3322 ret = blk_mq_request_issue_directly(rq, true));
3323 if (ret)
3324 blk_account_io_done(rq, blk_time_get_ns());
3325 return ret;
3326 }
3327 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
3328
3329 /**
3330 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
3331 * @rq: the clone request to be cleaned up
3332 *
3333 * Description:
3334 * Free all bios in @rq for a cloned request.
3335 */
3336 void blk_rq_unprep_clone(struct request *rq)
3337 {
3338 struct bio *bio;
3339
3340 while ((bio = rq->bio) != NULL) {
3341 rq->bio = bio->bi_next;
3342
3343 bio_put(bio);
3344 }
3345 }
3346 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3347
3348 /**
3349 * blk_rq_prep_clone - Helper function to setup clone request
3350 * @rq: the request to be setup
3351 * @rq_src: original request to be cloned
3352 * @bs: bio_set that bios for clone are allocated from
3353 * @gfp_mask: memory allocation mask for bio
3354 * @bio_ctr: setup function to be called for each clone bio.
3355 * Returns %0 for success, non %0 for failure.
3356 * @data: private data to be passed to @bio_ctr
3357 *
3358 * Description:
3359 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3360 * Also, pages which the original bios are pointing to are not copied
3361 * and the cloned bios just point same pages.
3362 * So cloned bios must be completed before original bios, which means
3363 * the caller must complete @rq before @rq_src.
3364 */
3365 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
3366 struct bio_set *bs, gfp_t gfp_mask,
3367 int (*bio_ctr)(struct bio *, struct bio *, void *),
3368 void *data)
3369 {
3370 struct bio *bio_src;
3371
3372 if (!bs)
3373 bs = &fs_bio_set;
3374
3375 __rq_for_each_bio(bio_src, rq_src) {
3376 struct bio *bio = bio_alloc_clone(rq->q->disk->part0, bio_src,
3377 gfp_mask, bs);
3378 if (!bio)
3379 goto free_and_out;
3380
3381 if (bio_ctr && bio_ctr(bio, bio_src, data)) {
3382 bio_put(bio);
3383 goto free_and_out;
3384 }
3385
3386 if (rq->bio) {
3387 rq->biotail->bi_next = bio;
3388 rq->biotail = bio;
3389 } else {
3390 rq->bio = rq->biotail = bio;
3391 }
3392 }
3393
3394 /* Copy attributes of the original request to the clone request. */
3395 rq->__sector = blk_rq_pos(rq_src);
3396 rq->__data_len = blk_rq_bytes(rq_src);
3397 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
3398 rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
3399 rq->special_vec = rq_src->special_vec;
3400 }
3401 rq->nr_phys_segments = rq_src->nr_phys_segments;
3402 rq->nr_integrity_segments = rq_src->nr_integrity_segments;
3403
3404 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
3405 goto free_and_out;
3406
3407 return 0;
3408
3409 free_and_out:
3410 blk_rq_unprep_clone(rq);
3411
3412 return -ENOMEM;
3413 }
3414 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3415 #endif /* CONFIG_BLK_MQ_STACKING */
3416
3417 /*
3418 * Steal bios from a request and add them to a bio list.
3419 * The request must not have been partially completed before.
3420 */
3421 void blk_steal_bios(struct bio_list *list, struct request *rq)
3422 {
3423 if (rq->bio) {
3424 if (list->tail)
3425 list->tail->bi_next = rq->bio;
3426 else
3427 list->head = rq->bio;
3428 list->tail = rq->biotail;
3429
3430 rq->bio = NULL;
3431 rq->biotail = NULL;
3432 }
3433
3434 rq->__data_len = 0;
3435 }
3436 EXPORT_SYMBOL_GPL(blk_steal_bios);
3437
3438 static size_t order_to_size(unsigned int order)
3439 {
3440 return (size_t)PAGE_SIZE << order;
3441 }
3442
3443 /* called before freeing request pool in @tags */
3444 static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
3445 struct blk_mq_tags *tags)
3446 {
3447 struct page *page;
3448
3449 /*
3450 * There is no need to clear mapping if driver tags is not initialized
3451 * or the mapping belongs to the driver tags.
3452 */
3453 if (!drv_tags || drv_tags == tags)
3454 return;
3455
3456 list_for_each_entry(page, &tags->page_list, lru) {
3457 unsigned long start = (unsigned long)page_address(page);
3458 unsigned long end = start + order_to_size(page->private);
3459 int i;
3460
3461 for (i = 0; i < drv_tags->nr_tags; i++) {
3462 struct request *rq = drv_tags->rqs[i];
3463 unsigned long rq_addr = (unsigned long)rq;
3464
3465 if (rq_addr >= start && rq_addr < end) {
3466 WARN_ON_ONCE(req_ref_read(rq) != 0);
3467 cmpxchg(&drv_tags->rqs[i], rq, NULL);
3468 }
3469 }
3470 }
3471 }
3472
3473 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
3474 unsigned int hctx_idx)
3475 {
3476 struct blk_mq_tags *drv_tags;
3477
3478 if (list_empty(&tags->page_list))
3479 return;
3480
3481 if (blk_mq_is_shared_tags(set->flags))
3482 drv_tags = set->shared_tags;
3483 else
3484 drv_tags = set->tags[hctx_idx];
3485
3486 if (tags->static_rqs && set->ops->exit_request) {
3487 int i;
3488
3489 for (i = 0; i < tags->nr_tags; i++) {
3490 struct request *rq = tags->static_rqs[i];
3491
3492 if (!rq)
3493 continue;
3494 set->ops->exit_request(set, rq, hctx_idx);
3495 tags->static_rqs[i] = NULL;
3496 }
3497 }
3498
3499 blk_mq_clear_rq_mapping(drv_tags, tags);
3500 /*
3501 * Free request pages in SRCU callback, which is called from
3502 * blk_mq_free_tags().
3503 */
3504 }
3505
3506 void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags)
3507 {
3508 kfree(tags->rqs);
3509 tags->rqs = NULL;
3510 kfree(tags->static_rqs);
3511 tags->static_rqs = NULL;
3512
3513 blk_mq_free_tags(set, tags);
3514 }
3515
3516 static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
3517 unsigned int hctx_idx)
3518 {
3519 int i;
3520
3521 for (i = 0; i < set->nr_maps; i++) {
3522 unsigned int start = set->map[i].queue_offset;
3523 unsigned int end = start + set->map[i].nr_queues;
3524
3525 if (hctx_idx >= start && hctx_idx < end)
3526 break;
3527 }
3528
3529 if (i >= set->nr_maps)
3530 i = HCTX_TYPE_DEFAULT;
3531
3532 return i;
3533 }
3534
3535 static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set,
3536 unsigned int hctx_idx)
3537 {
3538 enum hctx_type type = hctx_idx_to_type(set, hctx_idx);
3539
3540 return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx);
3541 }
3542
3543 static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
3544 unsigned int hctx_idx,
3545 unsigned int nr_tags,
3546 unsigned int reserved_tags)
3547 {
3548 int node = blk_mq_get_hctx_node(set, hctx_idx);
3549 struct blk_mq_tags *tags;
3550
3551 if (node == NUMA_NO_NODE)
3552 node = set->numa_node;
3553
3554 tags = blk_mq_init_tags(nr_tags, reserved_tags, set->flags, node);
3555 if (!tags)
3556 return NULL;
3557
3558 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3559 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3560 node);
3561 if (!tags->rqs)
3562 goto err_free_tags;
3563
3564 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3565 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3566 node);
3567 if (!tags->static_rqs)
3568 goto err_free_rqs;
3569
3570 return tags;
3571
3572 err_free_rqs:
3573 kfree(tags->rqs);
3574 err_free_tags:
3575 blk_mq_free_tags(set, tags);
3576 return NULL;
3577 }
3578
3579 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
3580 unsigned int hctx_idx, int node)
3581 {
3582 int ret;
3583
3584 if (set->ops->init_request) {
3585 ret = set->ops->init_request(set, rq, hctx_idx, node);
3586 if (ret)
3587 return ret;
3588 }
3589
3590 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
3591 return 0;
3592 }
3593
3594 static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
3595 struct blk_mq_tags *tags,
3596 unsigned int hctx_idx, unsigned int depth)
3597 {
3598 unsigned int i, j, entries_per_page, max_order = 4;
3599 int node = blk_mq_get_hctx_node(set, hctx_idx);
3600 size_t rq_size, left;
3601
3602 if (node == NUMA_NO_NODE)
3603 node = set->numa_node;
3604
3605 /*
3606 * rq_size is the size of the request plus driver payload, rounded
3607 * to the cacheline size
3608 */
3609 rq_size = round_up(sizeof(struct request) + set->cmd_size,
3610 cache_line_size());
3611 left = rq_size * depth;
3612
3613 for (i = 0; i < depth; ) {
3614 int this_order = max_order;
3615 struct page *page;
3616 int to_do;
3617 void *p;
3618
3619 while (this_order && left < order_to_size(this_order - 1))
3620 this_order--;
3621
3622 do {
3623 page = alloc_pages_node(node,
3624 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
3625 this_order);
3626 if (page)
3627 break;
3628 if (!this_order--)
3629 break;
3630 if (order_to_size(this_order) < rq_size)
3631 break;
3632 } while (1);
3633
3634 if (!page)
3635 goto fail;
3636
3637 page->private = this_order;
3638 list_add_tail(&page->lru, &tags->page_list);
3639
3640 p = page_address(page);
3641 /*
3642 * Allow kmemleak to scan these pages as they contain pointers
3643 * to additional allocations like via ops->init_request().
3644 */
3645 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
3646 entries_per_page = order_to_size(this_order) / rq_size;
3647 to_do = min(entries_per_page, depth - i);
3648 left -= to_do * rq_size;
3649 for (j = 0; j < to_do; j++) {
3650 struct request *rq = p;
3651
3652 tags->static_rqs[i] = rq;
3653 if (blk_mq_init_request(set, rq, hctx_idx, node)) {
3654 tags->static_rqs[i] = NULL;
3655 goto fail;
3656 }
3657
3658 p += rq_size;
3659 i++;
3660 }
3661 }
3662 return 0;
3663
3664 fail:
3665 blk_mq_free_rqs(set, tags, hctx_idx);
3666 return -ENOMEM;
3667 }
3668
3669 struct rq_iter_data {
3670 struct blk_mq_hw_ctx *hctx;
3671 bool has_rq;
3672 };
3673
3674 static bool blk_mq_has_request(struct request *rq, void *data)
3675 {
3676 struct rq_iter_data *iter_data = data;
3677
3678 if (rq->mq_hctx != iter_data->hctx)
3679 return true;
3680 iter_data->has_rq = true;
3681 return false;
3682 }
3683
3684 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
3685 {
3686 struct blk_mq_tags *tags = hctx->sched_tags ?
3687 hctx->sched_tags : hctx->tags;
3688 struct rq_iter_data data = {
3689 .hctx = hctx,
3690 };
3691 int srcu_idx;
3692
3693 srcu_idx = srcu_read_lock(&hctx->queue->tag_set->tags_srcu);
3694 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
3695 srcu_read_unlock(&hctx->queue->tag_set->tags_srcu, srcu_idx);
3696
3697 return data.has_rq;
3698 }
3699
3700 static bool blk_mq_hctx_has_online_cpu(struct blk_mq_hw_ctx *hctx,
3701 unsigned int this_cpu)
3702 {
3703 enum hctx_type type = hctx->type;
3704 int cpu;
3705
3706 /*
3707 * hctx->cpumask has to rule out isolated CPUs, but userspace still
3708 * might submit IOs on these isolated CPUs, so use the queue map to
3709 * check if all CPUs mapped to this hctx are offline
3710 */
3711 for_each_online_cpu(cpu) {
3712 struct blk_mq_hw_ctx *h = blk_mq_map_queue_type(hctx->queue,
3713 type, cpu);
3714
3715 if (h != hctx)
3716 continue;
3717
3718 /* this hctx has at least one online CPU */
3719 if (this_cpu != cpu)
3720 return true;
3721 }
3722
3723 return false;
3724 }
3725
3726 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
3727 {
3728 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3729 struct blk_mq_hw_ctx, cpuhp_online);
3730 int ret = 0;
3731
3732 if (!hctx->nr_ctx || blk_mq_hctx_has_online_cpu(hctx, cpu))
3733 return 0;
3734
3735 /*
3736 * Prevent new request from being allocated on the current hctx.
3737 *
3738 * The smp_mb__after_atomic() Pairs with the implied barrier in
3739 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is
3740 * seen once we return from the tag allocator.
3741 */
3742 set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3743 smp_mb__after_atomic();
3744
3745 /*
3746 * Try to grab a reference to the queue and wait for any outstanding
3747 * requests. If we could not grab a reference the queue has been
3748 * frozen and there are no requests.
3749 */
3750 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
3751 while (blk_mq_hctx_has_requests(hctx)) {
3752 /*
3753 * The wakeup capable IRQ handler of block device is
3754 * not called during suspend. Skip the loop by checking
3755 * pm_wakeup_pending to prevent the deadlock and improve
3756 * suspend latency.
3757 */
3758 if (pm_wakeup_pending()) {
3759 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3760 ret = -EBUSY;
3761 break;
3762 }
3763 msleep(5);
3764 }
3765 percpu_ref_put(&hctx->queue->q_usage_counter);
3766 }
3767
3768 return ret;
3769 }
3770
3771 /*
3772 * Check if one CPU is mapped to the specified hctx
3773 *
3774 * Isolated CPUs have been ruled out from hctx->cpumask, which is supposed
3775 * to be used for scheduling kworker only. For other usage, please call this
3776 * helper for checking if one CPU belongs to the specified hctx
3777 */
3778 static bool blk_mq_cpu_mapped_to_hctx(unsigned int cpu,
3779 const struct blk_mq_hw_ctx *hctx)
3780 {
3781 struct blk_mq_hw_ctx *mapped_hctx = blk_mq_map_queue_type(hctx->queue,
3782 hctx->type, cpu);
3783
3784 return mapped_hctx == hctx;
3785 }
3786
3787 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
3788 {
3789 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3790 struct blk_mq_hw_ctx, cpuhp_online);
3791
3792 if (blk_mq_cpu_mapped_to_hctx(cpu, hctx))
3793 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3794 return 0;
3795 }
3796
3797 /*
3798 * 'cpu' is going away. splice any existing rq_list entries from this
3799 * software queue to the hw queue dispatch list, and ensure that it
3800 * gets run.
3801 */
3802 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
3803 {
3804 struct blk_mq_hw_ctx *hctx;
3805 struct blk_mq_ctx *ctx;
3806 LIST_HEAD(tmp);
3807 enum hctx_type type;
3808
3809 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
3810 if (!blk_mq_cpu_mapped_to_hctx(cpu, hctx))
3811 return 0;
3812
3813 ctx = __blk_mq_get_ctx(hctx->queue, cpu);
3814 type = hctx->type;
3815
3816 spin_lock(&ctx->lock);
3817 if (!list_empty(&ctx->rq_lists[type])) {
3818 list_splice_init(&ctx->rq_lists[type], &tmp);
3819 blk_mq_hctx_clear_pending(hctx, ctx);
3820 }
3821 spin_unlock(&ctx->lock);
3822
3823 if (list_empty(&tmp))
3824 return 0;
3825
3826 spin_lock(&hctx->lock);
3827 list_splice_tail_init(&tmp, &hctx->dispatch);
3828 spin_unlock(&hctx->lock);
3829
3830 blk_mq_run_hw_queue(hctx, true);
3831 return 0;
3832 }
3833
3834 static void __blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3835 {
3836 lockdep_assert_held(&blk_mq_cpuhp_lock);
3837
3838 if (!(hctx->flags & BLK_MQ_F_STACKING) &&
3839 !hlist_unhashed(&hctx->cpuhp_online)) {
3840 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3841 &hctx->cpuhp_online);
3842 INIT_HLIST_NODE(&hctx->cpuhp_online);
3843 }
3844
3845 if (!hlist_unhashed(&hctx->cpuhp_dead)) {
3846 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3847 &hctx->cpuhp_dead);
3848 INIT_HLIST_NODE(&hctx->cpuhp_dead);
3849 }
3850 }
3851
3852 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3853 {
3854 mutex_lock(&blk_mq_cpuhp_lock);
3855 __blk_mq_remove_cpuhp(hctx);
3856 mutex_unlock(&blk_mq_cpuhp_lock);
3857 }
3858
3859 static void __blk_mq_add_cpuhp(struct blk_mq_hw_ctx *hctx)
3860 {
3861 lockdep_assert_held(&blk_mq_cpuhp_lock);
3862
3863 if (!(hctx->flags & BLK_MQ_F_STACKING) &&
3864 hlist_unhashed(&hctx->cpuhp_online))
3865 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3866 &hctx->cpuhp_online);
3867
3868 if (hlist_unhashed(&hctx->cpuhp_dead))
3869 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3870 &hctx->cpuhp_dead);
3871 }
3872
3873 static void __blk_mq_remove_cpuhp_list(struct list_head *head)
3874 {
3875 struct blk_mq_hw_ctx *hctx;
3876
3877 lockdep_assert_held(&blk_mq_cpuhp_lock);
3878
3879 list_for_each_entry(hctx, head, hctx_list)
3880 __blk_mq_remove_cpuhp(hctx);
3881 }
3882
3883 /*
3884 * Unregister cpuhp callbacks from exited hw queues
3885 *
3886 * Safe to call if this `request_queue` is live
3887 */
3888 static void blk_mq_remove_hw_queues_cpuhp(struct request_queue *q)
3889 {
3890 LIST_HEAD(hctx_list);
3891
3892 spin_lock(&q->unused_hctx_lock);
3893 list_splice_init(&q->unused_hctx_list, &hctx_list);
3894 spin_unlock(&q->unused_hctx_lock);
3895
3896 mutex_lock(&blk_mq_cpuhp_lock);
3897 __blk_mq_remove_cpuhp_list(&hctx_list);
3898 mutex_unlock(&blk_mq_cpuhp_lock);
3899
3900 spin_lock(&q->unused_hctx_lock);
3901 list_splice(&hctx_list, &q->unused_hctx_list);
3902 spin_unlock(&q->unused_hctx_lock);
3903 }
3904
3905 /*
3906 * Register cpuhp callbacks from all hw queues
3907 *
3908 * Safe to call if this `request_queue` is live
3909 */
3910 static void blk_mq_add_hw_queues_cpuhp(struct request_queue *q)
3911 {
3912 struct blk_mq_hw_ctx *hctx;
3913 unsigned long i;
3914
3915 mutex_lock(&blk_mq_cpuhp_lock);
3916 queue_for_each_hw_ctx(q, hctx, i)
3917 __blk_mq_add_cpuhp(hctx);
3918 mutex_unlock(&blk_mq_cpuhp_lock);
3919 }
3920
3921 /*
3922 * Before freeing hw queue, clearing the flush request reference in
3923 * tags->rqs[] for avoiding potential UAF.
3924 */
3925 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
3926 unsigned int queue_depth, struct request *flush_rq)
3927 {
3928 int i;
3929
3930 /* The hw queue may not be mapped yet */
3931 if (!tags)
3932 return;
3933
3934 WARN_ON_ONCE(req_ref_read(flush_rq) != 0);
3935
3936 for (i = 0; i < queue_depth; i++)
3937 cmpxchg(&tags->rqs[i], flush_rq, NULL);
3938 }
3939
3940 static void blk_free_flush_queue_callback(struct rcu_head *head)
3941 {
3942 struct blk_flush_queue *fq =
3943 container_of(head, struct blk_flush_queue, rcu_head);
3944
3945 blk_free_flush_queue(fq);
3946 }
3947
3948 /* hctx->ctxs will be freed in queue's release handler */
3949 static void blk_mq_exit_hctx(struct request_queue *q,
3950 struct blk_mq_tag_set *set,
3951 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
3952 {
3953 struct request *flush_rq = hctx->fq->flush_rq;
3954
3955 if (blk_mq_hw_queue_mapped(hctx))
3956 blk_mq_tag_idle(hctx);
3957
3958 if (blk_queue_init_done(q))
3959 blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
3960 set->queue_depth, flush_rq);
3961 if (set->ops->exit_request)
3962 set->ops->exit_request(set, flush_rq, hctx_idx);
3963
3964 if (set->ops->exit_hctx)
3965 set->ops->exit_hctx(hctx, hctx_idx);
3966
3967 call_srcu(&set->tags_srcu, &hctx->fq->rcu_head,
3968 blk_free_flush_queue_callback);
3969 hctx->fq = NULL;
3970
3971 xa_erase(&q->hctx_table, hctx_idx);
3972
3973 spin_lock(&q->unused_hctx_lock);
3974 list_add(&hctx->hctx_list, &q->unused_hctx_list);
3975 spin_unlock(&q->unused_hctx_lock);
3976 }
3977
3978 static void blk_mq_exit_hw_queues(struct request_queue *q,
3979 struct blk_mq_tag_set *set, int nr_queue)
3980 {
3981 struct blk_mq_hw_ctx *hctx;
3982 unsigned long i;
3983
3984 queue_for_each_hw_ctx(q, hctx, i) {
3985 if (i == nr_queue)
3986 break;
3987 blk_mq_remove_cpuhp(hctx);
3988 blk_mq_exit_hctx(q, set, hctx, i);
3989 }
3990 }
3991
3992 static int blk_mq_init_hctx(struct request_queue *q,
3993 struct blk_mq_tag_set *set,
3994 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
3995 {
3996 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
3997
3998 hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
3999 if (!hctx->fq)
4000 goto fail;
4001
4002 hctx->queue_num = hctx_idx;
4003
4004 hctx->tags = set->tags[hctx_idx];
4005
4006 if (set->ops->init_hctx &&
4007 set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
4008 goto fail_free_fq;
4009
4010 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
4011 hctx->numa_node))
4012 goto exit_hctx;
4013
4014 if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL))
4015 goto exit_flush_rq;
4016
4017 return 0;
4018
4019 exit_flush_rq:
4020 if (set->ops->exit_request)
4021 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
4022 exit_hctx:
4023 if (set->ops->exit_hctx)
4024 set->ops->exit_hctx(hctx, hctx_idx);
4025 fail_free_fq:
4026 blk_free_flush_queue(hctx->fq);
4027 hctx->fq = NULL;
4028 fail:
4029 return -1;
4030 }
4031
4032 static struct blk_mq_hw_ctx *
4033 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
4034 int node)
4035 {
4036 struct blk_mq_hw_ctx *hctx;
4037 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
4038
4039 hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);
4040 if (!hctx)
4041 goto fail_alloc_hctx;
4042
4043 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
4044 goto free_hctx;
4045
4046 atomic_set(&hctx->nr_active, 0);
4047 if (node == NUMA_NO_NODE)
4048 node = set->numa_node;
4049 hctx->numa_node = node;
4050
4051 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
4052 spin_lock_init(&hctx->lock);
4053 INIT_LIST_HEAD(&hctx->dispatch);
4054 INIT_HLIST_NODE(&hctx->cpuhp_dead);
4055 INIT_HLIST_NODE(&hctx->cpuhp_online);
4056 hctx->queue = q;
4057 hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
4058
4059 INIT_LIST_HEAD(&hctx->hctx_list);
4060
4061 /*
4062 * Allocate space for all possible cpus to avoid allocation at
4063 * runtime
4064 */
4065 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
4066 gfp, node);
4067 if (!hctx->ctxs)
4068 goto free_cpumask;
4069
4070 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
4071 gfp, node, false, false))
4072 goto free_ctxs;
4073 hctx->nr_ctx = 0;
4074
4075 spin_lock_init(&hctx->dispatch_wait_lock);
4076 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
4077 INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
4078
4079 blk_mq_hctx_kobj_init(hctx);
4080
4081 return hctx;
4082
4083 free_ctxs:
4084 kfree(hctx->ctxs);
4085 free_cpumask:
4086 free_cpumask_var(hctx->cpumask);
4087 free_hctx:
4088 kfree(hctx);
4089 fail_alloc_hctx:
4090 return NULL;
4091 }
4092
4093 static void blk_mq_init_cpu_queues(struct request_queue *q,
4094 unsigned int nr_hw_queues)
4095 {
4096 struct blk_mq_tag_set *set = q->tag_set;
4097 unsigned int i, j;
4098
4099 for_each_possible_cpu(i) {
4100 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
4101 struct blk_mq_hw_ctx *hctx;
4102 int k;
4103
4104 __ctx->cpu = i;
4105 spin_lock_init(&__ctx->lock);
4106 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
4107 INIT_LIST_HEAD(&__ctx->rq_lists[k]);
4108
4109 __ctx->queue = q;
4110
4111 /*
4112 * Set local node, IFF we have more than one hw queue. If
4113 * not, we remain on the home node of the device
4114 */
4115 for (j = 0; j < set->nr_maps; j++) {
4116 hctx = blk_mq_map_queue_type(q, j, i);
4117 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
4118 hctx->numa_node = cpu_to_node(i);
4119 }
4120 }
4121 }
4122
4123 struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
4124 unsigned int hctx_idx,
4125 unsigned int depth)
4126 {
4127 struct blk_mq_tags *tags;
4128 int ret;
4129
4130 tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
4131 if (!tags)
4132 return NULL;
4133
4134 ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
4135 if (ret) {
4136 blk_mq_free_rq_map(set, tags);
4137 return NULL;
4138 }
4139
4140 return tags;
4141 }
4142
4143 static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
4144 int hctx_idx)
4145 {
4146 if (blk_mq_is_shared_tags(set->flags)) {
4147 set->tags[hctx_idx] = set->shared_tags;
4148
4149 return true;
4150 }
4151
4152 set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
4153 set->queue_depth);
4154
4155 return set->tags[hctx_idx];
4156 }
4157
4158 void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
4159 struct blk_mq_tags *tags,
4160 unsigned int hctx_idx)
4161 {
4162 if (tags) {
4163 blk_mq_free_rqs(set, tags, hctx_idx);
4164 blk_mq_free_rq_map(set, tags);
4165 }
4166 }
4167
4168 static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
4169 unsigned int hctx_idx)
4170 {
4171 if (!blk_mq_is_shared_tags(set->flags))
4172 blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx);
4173
4174 set->tags[hctx_idx] = NULL;
4175 }
4176
4177 static void blk_mq_map_swqueue(struct request_queue *q)
4178 {
4179 unsigned int j, hctx_idx;
4180 unsigned long i;
4181 struct blk_mq_hw_ctx *hctx;
4182 struct blk_mq_ctx *ctx;
4183 struct blk_mq_tag_set *set = q->tag_set;
4184
4185 queue_for_each_hw_ctx(q, hctx, i) {
4186 cpumask_clear(hctx->cpumask);
4187 hctx->nr_ctx = 0;
4188 hctx->dispatch_from = NULL;
4189 }
4190
4191 /*
4192 * Map software to hardware queues.
4193 *
4194 * If the cpu isn't present, the cpu is mapped to first hctx.
4195 */
4196 for_each_possible_cpu(i) {
4197
4198 ctx = per_cpu_ptr(q->queue_ctx, i);
4199 for (j = 0; j < set->nr_maps; j++) {
4200 if (!set->map[j].nr_queues) {
4201 ctx->hctxs[j] = blk_mq_map_queue_type(q,
4202 HCTX_TYPE_DEFAULT, i);
4203 continue;
4204 }
4205 hctx_idx = set->map[j].mq_map[i];
4206 /* unmapped hw queue can be remapped after CPU topo changed */
4207 if (!set->tags[hctx_idx] &&
4208 !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
4209 /*
4210 * If tags initialization fail for some hctx,
4211 * that hctx won't be brought online. In this
4212 * case, remap the current ctx to hctx[0] which
4213 * is guaranteed to always have tags allocated
4214 */
4215 set->map[j].mq_map[i] = 0;
4216 }
4217
4218 hctx = blk_mq_map_queue_type(q, j, i);
4219 ctx->hctxs[j] = hctx;
4220 /*
4221 * If the CPU is already set in the mask, then we've
4222 * mapped this one already. This can happen if
4223 * devices share queues across queue maps.
4224 */
4225 if (cpumask_test_cpu(i, hctx->cpumask))
4226 continue;
4227
4228 cpumask_set_cpu(i, hctx->cpumask);
4229 hctx->type = j;
4230 ctx->index_hw[hctx->type] = hctx->nr_ctx;
4231 hctx->ctxs[hctx->nr_ctx++] = ctx;
4232
4233 /*
4234 * If the nr_ctx type overflows, we have exceeded the
4235 * amount of sw queues we can support.
4236 */
4237 BUG_ON(!hctx->nr_ctx);
4238 }
4239
4240 for (; j < HCTX_MAX_TYPES; j++)
4241 ctx->hctxs[j] = blk_mq_map_queue_type(q,
4242 HCTX_TYPE_DEFAULT, i);
4243 }
4244
4245 queue_for_each_hw_ctx(q, hctx, i) {
4246 int cpu;
4247
4248 /*
4249 * If no software queues are mapped to this hardware queue,
4250 * disable it and free the request entries.
4251 */
4252 if (!hctx->nr_ctx) {
4253 /* Never unmap queue 0. We need it as a
4254 * fallback in case of a new remap fails
4255 * allocation
4256 */
4257 if (i)
4258 __blk_mq_free_map_and_rqs(set, i);
4259
4260 hctx->tags = NULL;
4261 continue;
4262 }
4263
4264 hctx->tags = set->tags[i];
4265 WARN_ON(!hctx->tags);
4266
4267 /*
4268 * Set the map size to the number of mapped software queues.
4269 * This is more accurate and more efficient than looping
4270 * over all possibly mapped software queues.
4271 */
4272 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
4273
4274 /*
4275 * Rule out isolated CPUs from hctx->cpumask to avoid
4276 * running block kworker on isolated CPUs
4277 */
4278 for_each_cpu(cpu, hctx->cpumask) {
4279 if (cpu_is_isolated(cpu))
4280 cpumask_clear_cpu(cpu, hctx->cpumask);
4281 }
4282
4283 /*
4284 * Initialize batch roundrobin counts
4285 */
4286 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
4287 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
4288 }
4289 }
4290
4291 /*
4292 * Caller needs to ensure that we're either frozen/quiesced, or that
4293 * the queue isn't live yet.
4294 */
4295 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
4296 {
4297 struct blk_mq_hw_ctx *hctx;
4298 unsigned long i;
4299
4300 queue_for_each_hw_ctx(q, hctx, i) {
4301 if (shared) {
4302 hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
4303 } else {
4304 blk_mq_tag_idle(hctx);
4305 hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4306 }
4307 }
4308 }
4309
4310 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
4311 bool shared)
4312 {
4313 struct request_queue *q;
4314 unsigned int memflags;
4315
4316 lockdep_assert_held(&set->tag_list_lock);
4317
4318 list_for_each_entry(q, &set->tag_list, tag_set_list) {
4319 memflags = blk_mq_freeze_queue(q);
4320 queue_set_hctx_shared(q, shared);
4321 blk_mq_unfreeze_queue(q, memflags);
4322 }
4323 }
4324
4325 static void blk_mq_del_queue_tag_set(struct request_queue *q)
4326 {
4327 struct blk_mq_tag_set *set = q->tag_set;
4328
4329 mutex_lock(&set->tag_list_lock);
4330 list_del_rcu(&q->tag_set_list);
4331 if (list_is_singular(&set->tag_list)) {
4332 /* just transitioned to unshared */
4333 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4334 /* update existing queue */
4335 blk_mq_update_tag_set_shared(set, false);
4336 }
4337 mutex_unlock(&set->tag_list_lock);
4338 }
4339
4340 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
4341 struct request_queue *q)
4342 {
4343 mutex_lock(&set->tag_list_lock);
4344
4345 /*
4346 * Check to see if we're transitioning to shared (from 1 to 2 queues).
4347 */
4348 if (!list_empty(&set->tag_list) &&
4349 !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
4350 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
4351 /* update existing queue */
4352 blk_mq_update_tag_set_shared(set, true);
4353 }
4354 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
4355 queue_set_hctx_shared(q, true);
4356 list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
4357
4358 mutex_unlock(&set->tag_list_lock);
4359 }
4360
4361 /* All allocations will be freed in release handler of q->mq_kobj */
4362 static int blk_mq_alloc_ctxs(struct request_queue *q)
4363 {
4364 struct blk_mq_ctxs *ctxs;
4365 int cpu;
4366
4367 ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
4368 if (!ctxs)
4369 return -ENOMEM;
4370
4371 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
4372 if (!ctxs->queue_ctx)
4373 goto fail;
4374
4375 for_each_possible_cpu(cpu) {
4376 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
4377 ctx->ctxs = ctxs;
4378 }
4379
4380 q->mq_kobj = &ctxs->kobj;
4381 q->queue_ctx = ctxs->queue_ctx;
4382
4383 return 0;
4384 fail:
4385 kfree(ctxs);
4386 return -ENOMEM;
4387 }
4388
4389 /*
4390 * It is the actual release handler for mq, but we do it from
4391 * request queue's release handler for avoiding use-after-free
4392 * and headache because q->mq_kobj shouldn't have been introduced,
4393 * but we can't group ctx/kctx kobj without it.
4394 */
4395 void blk_mq_release(struct request_queue *q)
4396 {
4397 struct blk_mq_hw_ctx *hctx, *next;
4398 unsigned long i;
4399
4400 queue_for_each_hw_ctx(q, hctx, i)
4401 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
4402
4403 /* all hctx are in .unused_hctx_list now */
4404 list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
4405 list_del_init(&hctx->hctx_list);
4406 kobject_put(&hctx->kobj);
4407 }
4408
4409 xa_destroy(&q->hctx_table);
4410
4411 /*
4412 * release .mq_kobj and sw queue's kobject now because
4413 * both share lifetime with request queue.
4414 */
4415 blk_mq_sysfs_deinit(q);
4416 }
4417
4418 struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set,
4419 struct queue_limits *lim, void *queuedata)
4420 {
4421 struct queue_limits default_lim = { };
4422 struct request_queue *q;
4423 int ret;
4424
4425 if (!lim)
4426 lim = &default_lim;
4427 lim->features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT;
4428 if (set->nr_maps > HCTX_TYPE_POLL)
4429 lim->features |= BLK_FEAT_POLL;
4430
4431 q = blk_alloc_queue(lim, set->numa_node);
4432 if (IS_ERR(q))
4433 return q;
4434 q->queuedata = queuedata;
4435 ret = blk_mq_init_allocated_queue(set, q);
4436 if (ret) {
4437 blk_put_queue(q);
4438 return ERR_PTR(ret);
4439 }
4440 return q;
4441 }
4442 EXPORT_SYMBOL(blk_mq_alloc_queue);
4443
4444 /**
4445 * blk_mq_destroy_queue - shutdown a request queue
4446 * @q: request queue to shutdown
4447 *
4448 * This shuts down a request queue allocated by blk_mq_alloc_queue(). All future
4449 * requests will be failed with -ENODEV. The caller is responsible for dropping
4450 * the reference from blk_mq_alloc_queue() by calling blk_put_queue().
4451 *
4452 * Context: can sleep
4453 */
4454 void blk_mq_destroy_queue(struct request_queue *q)
4455 {
4456 WARN_ON_ONCE(!queue_is_mq(q));
4457 WARN_ON_ONCE(blk_queue_registered(q));
4458
4459 might_sleep();
4460
4461 blk_queue_flag_set(QUEUE_FLAG_DYING, q);
4462 blk_queue_start_drain(q);
4463 blk_mq_freeze_queue_wait(q);
4464
4465 blk_sync_queue(q);
4466 blk_mq_cancel_work_sync(q);
4467 blk_mq_exit_queue(q);
4468 }
4469 EXPORT_SYMBOL(blk_mq_destroy_queue);
4470
4471 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set,
4472 struct queue_limits *lim, void *queuedata,
4473 struct lock_class_key *lkclass)
4474 {
4475 struct request_queue *q;
4476 struct gendisk *disk;
4477
4478 q = blk_mq_alloc_queue(set, lim, queuedata);
4479 if (IS_ERR(q))
4480 return ERR_CAST(q);
4481
4482 disk = __alloc_disk_node(q, set->numa_node, lkclass);
4483 if (!disk) {
4484 blk_mq_destroy_queue(q);
4485 blk_put_queue(q);
4486 return ERR_PTR(-ENOMEM);
4487 }
4488 set_bit(GD_OWNS_QUEUE, &disk->state);
4489 return disk;
4490 }
4491 EXPORT_SYMBOL(__blk_mq_alloc_disk);
4492
4493 struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
4494 struct lock_class_key *lkclass)
4495 {
4496 struct gendisk *disk;
4497
4498 if (!blk_get_queue(q))
4499 return NULL;
4500 disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
4501 if (!disk)
4502 blk_put_queue(q);
4503 return disk;
4504 }
4505 EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
4506
4507 /*
4508 * Only hctx removed from cpuhp list can be reused
4509 */
4510 static bool blk_mq_hctx_is_reusable(struct blk_mq_hw_ctx *hctx)
4511 {
4512 return hlist_unhashed(&hctx->cpuhp_online) &&
4513 hlist_unhashed(&hctx->cpuhp_dead);
4514 }
4515
4516 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
4517 struct blk_mq_tag_set *set, struct request_queue *q,
4518 int hctx_idx, int node)
4519 {
4520 struct blk_mq_hw_ctx *hctx = NULL, *tmp;
4521
4522 /* reuse dead hctx first */
4523 spin_lock(&q->unused_hctx_lock);
4524 list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
4525 if (tmp->numa_node == node && blk_mq_hctx_is_reusable(tmp)) {
4526 hctx = tmp;
4527 break;
4528 }
4529 }
4530 if (hctx)
4531 list_del_init(&hctx->hctx_list);
4532 spin_unlock(&q->unused_hctx_lock);
4533
4534 if (!hctx)
4535 hctx = blk_mq_alloc_hctx(q, set, node);
4536 if (!hctx)
4537 goto fail;
4538
4539 if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
4540 goto free_hctx;
4541
4542 return hctx;
4543
4544 free_hctx:
4545 kobject_put(&hctx->kobj);
4546 fail:
4547 return NULL;
4548 }
4549
4550 static void __blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4551 struct request_queue *q)
4552 {
4553 struct blk_mq_hw_ctx *hctx;
4554 unsigned long i, j;
4555
4556 for (i = 0; i < set->nr_hw_queues; i++) {
4557 int old_node;
4558 int node = blk_mq_get_hctx_node(set, i);
4559 struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i);
4560
4561 if (old_hctx) {
4562 old_node = old_hctx->numa_node;
4563 blk_mq_exit_hctx(q, set, old_hctx, i);
4564 }
4565
4566 if (!blk_mq_alloc_and_init_hctx(set, q, i, node)) {
4567 if (!old_hctx)
4568 break;
4569 pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n",
4570 node, old_node);
4571 hctx = blk_mq_alloc_and_init_hctx(set, q, i, old_node);
4572 WARN_ON_ONCE(!hctx);
4573 }
4574 }
4575 /*
4576 * Increasing nr_hw_queues fails. Free the newly allocated
4577 * hctxs and keep the previous q->nr_hw_queues.
4578 */
4579 if (i != set->nr_hw_queues) {
4580 j = q->nr_hw_queues;
4581 } else {
4582 j = i;
4583 q->nr_hw_queues = set->nr_hw_queues;
4584 }
4585
4586 xa_for_each_start(&q->hctx_table, j, hctx, j)
4587 blk_mq_exit_hctx(q, set, hctx, j);
4588 }
4589
4590 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4591 struct request_queue *q)
4592 {
4593 __blk_mq_realloc_hw_ctxs(set, q);
4594
4595 /* unregister cpuhp callbacks for exited hctxs */
4596 blk_mq_remove_hw_queues_cpuhp(q);
4597
4598 /* register cpuhp for new initialized hctxs */
4599 blk_mq_add_hw_queues_cpuhp(q);
4600 }
4601
4602 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
4603 struct request_queue *q)
4604 {
4605 /* mark the queue as mq asap */
4606 q->mq_ops = set->ops;
4607
4608 /*
4609 * ->tag_set has to be setup before initialize hctx, which cpuphp
4610 * handler needs it for checking queue mapping
4611 */
4612 q->tag_set = set;
4613
4614 if (blk_mq_alloc_ctxs(q))
4615 goto err_exit;
4616
4617 /* init q->mq_kobj and sw queues' kobjects */
4618 blk_mq_sysfs_init(q);
4619
4620 INIT_LIST_HEAD(&q->unused_hctx_list);
4621 spin_lock_init(&q->unused_hctx_lock);
4622
4623 xa_init(&q->hctx_table);
4624
4625 blk_mq_realloc_hw_ctxs(set, q);
4626 if (!q->nr_hw_queues)
4627 goto err_hctxs;
4628
4629 INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
4630 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
4631
4632 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
4633
4634 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
4635 INIT_LIST_HEAD(&q->flush_list);
4636 INIT_LIST_HEAD(&q->requeue_list);
4637 spin_lock_init(&q->requeue_lock);
4638
4639 q->nr_requests = set->queue_depth;
4640
4641 blk_mq_init_cpu_queues(q, set->nr_hw_queues);
4642 blk_mq_map_swqueue(q);
4643 blk_mq_add_queue_tag_set(set, q);
4644 return 0;
4645
4646 err_hctxs:
4647 blk_mq_release(q);
4648 err_exit:
4649 q->mq_ops = NULL;
4650 return -ENOMEM;
4651 }
4652 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
4653
4654 /* tags can _not_ be used after returning from blk_mq_exit_queue */
4655 void blk_mq_exit_queue(struct request_queue *q)
4656 {
4657 struct blk_mq_tag_set *set = q->tag_set;
4658
4659 /* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
4660 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
4661 /* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
4662 blk_mq_del_queue_tag_set(q);
4663 }
4664
4665 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
4666 {
4667 int i;
4668
4669 if (blk_mq_is_shared_tags(set->flags)) {
4670 set->shared_tags = blk_mq_alloc_map_and_rqs(set,
4671 BLK_MQ_NO_HCTX_IDX,
4672 set->queue_depth);
4673 if (!set->shared_tags)
4674 return -ENOMEM;
4675 }
4676
4677 for (i = 0; i < set->nr_hw_queues; i++) {
4678 if (!__blk_mq_alloc_map_and_rqs(set, i))
4679 goto out_unwind;
4680 cond_resched();
4681 }
4682
4683 return 0;
4684
4685 out_unwind:
4686 while (--i >= 0)
4687 __blk_mq_free_map_and_rqs(set, i);
4688
4689 if (blk_mq_is_shared_tags(set->flags)) {
4690 blk_mq_free_map_and_rqs(set, set->shared_tags,
4691 BLK_MQ_NO_HCTX_IDX);
4692 }
4693
4694 return -ENOMEM;
4695 }
4696
4697 /*
4698 * Allocate the request maps associated with this tag_set. Note that this
4699 * may reduce the depth asked for, if memory is tight. set->queue_depth
4700 * will be updated to reflect the allocated depth.
4701 */
4702 static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
4703 {
4704 unsigned int depth;
4705 int err;
4706
4707 depth = set->queue_depth;
4708 do {
4709 err = __blk_mq_alloc_rq_maps(set);
4710 if (!err)
4711 break;
4712
4713 set->queue_depth >>= 1;
4714 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
4715 err = -ENOMEM;
4716 break;
4717 }
4718 } while (set->queue_depth);
4719
4720 if (!set->queue_depth || err) {
4721 pr_err("blk-mq: failed to allocate request map\n");
4722 return -ENOMEM;
4723 }
4724
4725 if (depth != set->queue_depth)
4726 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
4727 depth, set->queue_depth);
4728
4729 return 0;
4730 }
4731
4732 static void blk_mq_update_queue_map(struct blk_mq_tag_set *set)
4733 {
4734 /*
4735 * blk_mq_map_queues() and multiple .map_queues() implementations
4736 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
4737 * number of hardware queues.
4738 */
4739 if (set->nr_maps == 1)
4740 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
4741
4742 if (set->ops->map_queues) {
4743 int i;
4744
4745 /*
4746 * transport .map_queues is usually done in the following
4747 * way:
4748 *
4749 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
4750 * mask = get_cpu_mask(queue)
4751 * for_each_cpu(cpu, mask)
4752 * set->map[x].mq_map[cpu] = queue;
4753 * }
4754 *
4755 * When we need to remap, the table has to be cleared for
4756 * killing stale mapping since one CPU may not be mapped
4757 * to any hw queue.
4758 */
4759 for (i = 0; i < set->nr_maps; i++)
4760 blk_mq_clear_mq_map(&set->map[i]);
4761
4762 set->ops->map_queues(set);
4763 } else {
4764 BUG_ON(set->nr_maps > 1);
4765 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
4766 }
4767 }
4768
4769 static struct blk_mq_tags **blk_mq_prealloc_tag_set_tags(
4770 struct blk_mq_tag_set *set,
4771 int new_nr_hw_queues)
4772 {
4773 struct blk_mq_tags **new_tags;
4774 int i;
4775
4776 if (set->nr_hw_queues >= new_nr_hw_queues)
4777 return NULL;
4778
4779 new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
4780 GFP_KERNEL, set->numa_node);
4781 if (!new_tags)
4782 return ERR_PTR(-ENOMEM);
4783
4784 if (set->tags)
4785 memcpy(new_tags, set->tags, set->nr_hw_queues *
4786 sizeof(*set->tags));
4787
4788 for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) {
4789 if (blk_mq_is_shared_tags(set->flags)) {
4790 new_tags[i] = set->shared_tags;
4791 } else {
4792 new_tags[i] = blk_mq_alloc_map_and_rqs(set, i,
4793 set->queue_depth);
4794 if (!new_tags[i])
4795 goto out_unwind;
4796 }
4797 cond_resched();
4798 }
4799
4800 return new_tags;
4801 out_unwind:
4802 while (--i >= set->nr_hw_queues) {
4803 if (!blk_mq_is_shared_tags(set->flags))
4804 blk_mq_free_map_and_rqs(set, new_tags[i], i);
4805 }
4806 kfree(new_tags);
4807 return ERR_PTR(-ENOMEM);
4808 }
4809
4810 /*
4811 * Alloc a tag set to be associated with one or more request queues.
4812 * May fail with EINVAL for various error conditions. May adjust the
4813 * requested depth down, if it's too large. In that case, the set
4814 * value will be stored in set->queue_depth.
4815 */
4816 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
4817 {
4818 int i, ret;
4819
4820 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
4821
4822 if (!set->nr_hw_queues)
4823 return -EINVAL;
4824 if (!set->queue_depth)
4825 return -EINVAL;
4826 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
4827 return -EINVAL;
4828
4829 if (!set->ops->queue_rq)
4830 return -EINVAL;
4831
4832 if (!set->ops->get_budget ^ !set->ops->put_budget)
4833 return -EINVAL;
4834
4835 if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
4836 pr_info("blk-mq: reduced tag depth to %u\n",
4837 BLK_MQ_MAX_DEPTH);
4838 set->queue_depth = BLK_MQ_MAX_DEPTH;
4839 }
4840
4841 if (!set->nr_maps)
4842 set->nr_maps = 1;
4843 else if (set->nr_maps > HCTX_MAX_TYPES)
4844 return -EINVAL;
4845
4846 /*
4847 * If a crashdump is active, then we are potentially in a very
4848 * memory constrained environment. Limit us to 64 tags to prevent
4849 * using too much memory.
4850 */
4851 if (is_kdump_kernel())
4852 set->queue_depth = min(64U, set->queue_depth);
4853
4854 /*
4855 * There is no use for more h/w queues than cpus if we just have
4856 * a single map
4857 */
4858 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
4859 set->nr_hw_queues = nr_cpu_ids;
4860
4861 if (set->flags & BLK_MQ_F_BLOCKING) {
4862 set->srcu = kmalloc(sizeof(*set->srcu), GFP_KERNEL);
4863 if (!set->srcu)
4864 return -ENOMEM;
4865 ret = init_srcu_struct(set->srcu);
4866 if (ret)
4867 goto out_free_srcu;
4868 }
4869 ret = init_srcu_struct(&set->tags_srcu);
4870 if (ret)
4871 goto out_cleanup_srcu;
4872
4873 init_rwsem(&set->update_nr_hwq_lock);
4874
4875 ret = -ENOMEM;
4876 set->tags = kcalloc_node(set->nr_hw_queues,
4877 sizeof(struct blk_mq_tags *), GFP_KERNEL,
4878 set->numa_node);
4879 if (!set->tags)
4880 goto out_cleanup_tags_srcu;
4881
4882 for (i = 0; i < set->nr_maps; i++) {
4883 set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
4884 sizeof(set->map[i].mq_map[0]),
4885 GFP_KERNEL, set->numa_node);
4886 if (!set->map[i].mq_map)
4887 goto out_free_mq_map;
4888 set->map[i].nr_queues = set->nr_hw_queues;
4889 }
4890
4891 blk_mq_update_queue_map(set);
4892
4893 ret = blk_mq_alloc_set_map_and_rqs(set);
4894 if (ret)
4895 goto out_free_mq_map;
4896
4897 mutex_init(&set->tag_list_lock);
4898 INIT_LIST_HEAD(&set->tag_list);
4899
4900 return 0;
4901
4902 out_free_mq_map:
4903 for (i = 0; i < set->nr_maps; i++) {
4904 kfree(set->map[i].mq_map);
4905 set->map[i].mq_map = NULL;
4906 }
4907 kfree(set->tags);
4908 set->tags = NULL;
4909 out_cleanup_tags_srcu:
4910 cleanup_srcu_struct(&set->tags_srcu);
4911 out_cleanup_srcu:
4912 if (set->flags & BLK_MQ_F_BLOCKING)
4913 cleanup_srcu_struct(set->srcu);
4914 out_free_srcu:
4915 if (set->flags & BLK_MQ_F_BLOCKING)
4916 kfree(set->srcu);
4917 return ret;
4918 }
4919 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
4920
4921 /* allocate and initialize a tagset for a simple single-queue device */
4922 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
4923 const struct blk_mq_ops *ops, unsigned int queue_depth,
4924 unsigned int set_flags)
4925 {
4926 memset(set, 0, sizeof(*set));
4927 set->ops = ops;
4928 set->nr_hw_queues = 1;
4929 set->nr_maps = 1;
4930 set->queue_depth = queue_depth;
4931 set->numa_node = NUMA_NO_NODE;
4932 set->flags = set_flags;
4933 return blk_mq_alloc_tag_set(set);
4934 }
4935 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
4936
4937 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
4938 {
4939 int i, j;
4940
4941 for (i = 0; i < set->nr_hw_queues; i++)
4942 __blk_mq_free_map_and_rqs(set, i);
4943
4944 if (blk_mq_is_shared_tags(set->flags)) {
4945 blk_mq_free_map_and_rqs(set, set->shared_tags,
4946 BLK_MQ_NO_HCTX_IDX);
4947 }
4948
4949 for (j = 0; j < set->nr_maps; j++) {
4950 kfree(set->map[j].mq_map);
4951 set->map[j].mq_map = NULL;
4952 }
4953
4954 kfree(set->tags);
4955 set->tags = NULL;
4956
4957 srcu_barrier(&set->tags_srcu);
4958 cleanup_srcu_struct(&set->tags_srcu);
4959 if (set->flags & BLK_MQ_F_BLOCKING) {
4960 cleanup_srcu_struct(set->srcu);
4961 kfree(set->srcu);
4962 }
4963 }
4964 EXPORT_SYMBOL(blk_mq_free_tag_set);
4965
4966 struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q,
4967 struct elevator_tags *et,
4968 unsigned int nr)
4969 {
4970 struct blk_mq_tag_set *set = q->tag_set;
4971 struct elevator_tags *old_et = NULL;
4972 struct blk_mq_hw_ctx *hctx;
4973 unsigned long i;
4974
4975 blk_mq_quiesce_queue(q);
4976
4977 if (blk_mq_is_shared_tags(set->flags)) {
4978 /*
4979 * Shared tags, for sched tags, we allocate max initially hence
4980 * tags can't grow, see blk_mq_alloc_sched_tags().
4981 */
4982 if (q->elevator)
4983 blk_mq_tag_update_sched_shared_tags(q, nr);
4984 else
4985 blk_mq_tag_resize_shared_tags(set, nr);
4986 } else if (!q->elevator) {
4987 /*
4988 * Non-shared hardware tags, nr is already checked from
4989 * queue_requests_store() and tags can't grow.
4990 */
4991 queue_for_each_hw_ctx(q, hctx, i) {
4992 if (!hctx->tags)
4993 continue;
4994 sbitmap_queue_resize(&hctx->tags->bitmap_tags,
4995 nr - hctx->tags->nr_reserved_tags);
4996 }
4997 } else if (nr <= q->elevator->et->nr_requests) {
4998 /* Non-shared sched tags, and tags don't grow. */
4999 queue_for_each_hw_ctx(q, hctx, i) {
5000 if (!hctx->sched_tags)
5001 continue;
5002 sbitmap_queue_resize(&hctx->sched_tags->bitmap_tags,
5003 nr - hctx->sched_tags->nr_reserved_tags);
5004 }
5005 } else {
5006 /* Non-shared sched tags, and tags grow */
5007 queue_for_each_hw_ctx(q, hctx, i)
5008 hctx->sched_tags = et->tags[i];
5009 old_et = q->elevator->et;
5010 q->elevator->et = et;
5011 }
5012
5013 q->nr_requests = nr;
5014 if (q->elevator && q->elevator->type->ops.depth_updated)
5015 q->elevator->type->ops.depth_updated(q);
5016
5017 blk_mq_unquiesce_queue(q);
5018 return old_et;
5019 }
5020
5021 /*
5022 * Switch back to the elevator type stored in the xarray.
5023 */
5024 static void blk_mq_elv_switch_back(struct request_queue *q,
5025 struct xarray *elv_tbl)
5026 {
5027 struct elv_change_ctx *ctx = xa_load(elv_tbl, q->id);
5028
5029 if (WARN_ON_ONCE(!ctx))
5030 return;
5031
5032 /* The elv_update_nr_hw_queues unfreezes the queue. */
5033 elv_update_nr_hw_queues(q, ctx);
5034
5035 /* Drop the reference acquired in blk_mq_elv_switch_none. */
5036 if (ctx->type)
5037 elevator_put(ctx->type);
5038 }
5039
5040 /*
5041 * Stores elevator name and type in ctx and set current elevator to none.
5042 */
5043 static int blk_mq_elv_switch_none(struct request_queue *q,
5044 struct xarray *elv_tbl)
5045 {
5046 struct elv_change_ctx *ctx;
5047
5048 lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock);
5049
5050 /*
5051 * Accessing q->elevator without holding q->elevator_lock is safe here
5052 * because we're called from nr_hw_queue update which is protected by
5053 * set->update_nr_hwq_lock in the writer context. So, scheduler update/
5054 * switch code (which acquires the same lock in the reader context)
5055 * can't run concurrently.
5056 */
5057 if (q->elevator) {
5058 ctx = xa_load(elv_tbl, q->id);
5059 if (WARN_ON_ONCE(!ctx))
5060 return -ENOENT;
5061
5062 ctx->name = q->elevator->type->elevator_name;
5063
5064 /*
5065 * Before we switch elevator to 'none', take a reference to
5066 * the elevator module so that while nr_hw_queue update is
5067 * running, no one can remove elevator module. We'd put the
5068 * reference to elevator module later when we switch back
5069 * elevator.
5070 */
5071 __elevator_get(q->elevator->type);
5072
5073 /*
5074 * Store elevator type so that we can release the reference
5075 * taken above later.
5076 */
5077 ctx->type = q->elevator->type;
5078 elevator_set_none(q);
5079 }
5080 return 0;
5081 }
5082
5083 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
5084 int nr_hw_queues)
5085 {
5086 struct request_queue *q;
5087 int prev_nr_hw_queues = set->nr_hw_queues;
5088 unsigned int memflags;
5089 int i;
5090 struct xarray elv_tbl;
5091 struct blk_mq_tags **new_tags;
5092 bool queues_frozen = false;
5093
5094 lockdep_assert_held(&set->tag_list_lock);
5095
5096 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
5097 nr_hw_queues = nr_cpu_ids;
5098 if (nr_hw_queues < 1)
5099 return;
5100 if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
5101 return;
5102
5103 memflags = memalloc_noio_save();
5104
5105 xa_init(&elv_tbl);
5106 if (blk_mq_alloc_sched_ctx_batch(&elv_tbl, set) < 0)
5107 goto out_free_ctx;
5108
5109 if (blk_mq_alloc_sched_res_batch(&elv_tbl, set, nr_hw_queues) < 0)
5110 goto out_free_ctx;
5111
5112 list_for_each_entry(q, &set->tag_list, tag_set_list) {
5113 blk_mq_debugfs_unregister_hctxs(q);
5114 blk_mq_sysfs_unregister_hctxs(q);
5115 }
5116
5117 /*
5118 * Switch IO scheduler to 'none', cleaning up the data associated
5119 * with the previous scheduler. We will switch back once we are done
5120 * updating the new sw to hw queue mappings.
5121 */
5122 list_for_each_entry(q, &set->tag_list, tag_set_list)
5123 if (blk_mq_elv_switch_none(q, &elv_tbl))
5124 goto switch_back;
5125
5126 new_tags = blk_mq_prealloc_tag_set_tags(set, nr_hw_queues);
5127 if (IS_ERR(new_tags))
5128 goto switch_back;
5129
5130 list_for_each_entry(q, &set->tag_list, tag_set_list)
5131 blk_mq_freeze_queue_nomemsave(q);
5132 queues_frozen = true;
5133 if (new_tags) {
5134 kfree(set->tags);
5135 set->tags = new_tags;
5136 }
5137 set->nr_hw_queues = nr_hw_queues;
5138
5139 fallback:
5140 blk_mq_update_queue_map(set);
5141 list_for_each_entry(q, &set->tag_list, tag_set_list) {
5142 __blk_mq_realloc_hw_ctxs(set, q);
5143
5144 if (q->nr_hw_queues != set->nr_hw_queues) {
5145 int i = prev_nr_hw_queues;
5146
5147 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
5148 nr_hw_queues, prev_nr_hw_queues);
5149 for (; i < set->nr_hw_queues; i++)
5150 __blk_mq_free_map_and_rqs(set, i);
5151
5152 set->nr_hw_queues = prev_nr_hw_queues;
5153 goto fallback;
5154 }
5155 blk_mq_map_swqueue(q);
5156 }
5157 switch_back:
5158 /* The blk_mq_elv_switch_back unfreezes queue for us. */
5159 list_for_each_entry(q, &set->tag_list, tag_set_list) {
5160 /* switch_back expects queue to be frozen */
5161 if (!queues_frozen)
5162 blk_mq_freeze_queue_nomemsave(q);
5163 blk_mq_elv_switch_back(q, &elv_tbl);
5164 }
5165
5166 list_for_each_entry(q, &set->tag_list, tag_set_list) {
5167 blk_mq_sysfs_register_hctxs(q);
5168 blk_mq_debugfs_register_hctxs(q);
5169
5170 blk_mq_remove_hw_queues_cpuhp(q);
5171 blk_mq_add_hw_queues_cpuhp(q);
5172 }
5173
5174 out_free_ctx:
5175 blk_mq_free_sched_ctx_batch(&elv_tbl);
5176 xa_destroy(&elv_tbl);
5177 memalloc_noio_restore(memflags);
5178
5179 /* Free the excess tags when nr_hw_queues shrink. */
5180 for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++)
5181 __blk_mq_free_map_and_rqs(set, i);
5182 }
5183
5184 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
5185 {
5186 down_write(&set->update_nr_hwq_lock);
5187 mutex_lock(&set->tag_list_lock);
5188 __blk_mq_update_nr_hw_queues(set, nr_hw_queues);
5189 mutex_unlock(&set->tag_list_lock);
5190 up_write(&set->update_nr_hwq_lock);
5191 }
5192 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
5193
5194 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
5195 struct io_comp_batch *iob, unsigned int flags)
5196 {
5197 long state = get_current_state();
5198 int ret;
5199
5200 do {
5201 ret = q->mq_ops->poll(hctx, iob);
5202 if (ret > 0) {
5203 __set_current_state(TASK_RUNNING);
5204 return ret;
5205 }
5206
5207 if (signal_pending_state(state, current))
5208 __set_current_state(TASK_RUNNING);
5209 if (task_is_running(current))
5210 return 1;
5211
5212 if (ret < 0 || (flags & BLK_POLL_ONESHOT))
5213 break;
5214 cpu_relax();
5215 } while (!need_resched());
5216
5217 __set_current_state(TASK_RUNNING);
5218 return 0;
5219 }
5220
5221 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
5222 struct io_comp_batch *iob, unsigned int flags)
5223 {
5224 if (!blk_mq_can_poll(q))
5225 return 0;
5226 return blk_hctx_poll(q, xa_load(&q->hctx_table, cookie), iob, flags);
5227 }
5228
5229 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
5230 unsigned int poll_flags)
5231 {
5232 struct request_queue *q = rq->q;
5233 int ret;
5234
5235 if (!blk_rq_is_poll(rq))
5236 return 0;
5237 if (!percpu_ref_tryget(&q->q_usage_counter))
5238 return 0;
5239
5240 ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags);
5241 blk_queue_exit(q);
5242
5243 return ret;
5244 }
5245 EXPORT_SYMBOL_GPL(blk_rq_poll);
5246
5247 unsigned int blk_mq_rq_cpu(struct request *rq)
5248 {
5249 return rq->mq_ctx->cpu;
5250 }
5251 EXPORT_SYMBOL(blk_mq_rq_cpu);
5252
5253 void blk_mq_cancel_work_sync(struct request_queue *q)
5254 {
5255 struct blk_mq_hw_ctx *hctx;
5256 unsigned long i;
5257
5258 cancel_delayed_work_sync(&q->requeue_work);
5259
5260 queue_for_each_hw_ctx(q, hctx, i)
5261 cancel_delayed_work_sync(&hctx->run_work);
5262 }
5263
5264 static int __init blk_mq_init(void)
5265 {
5266 int i;
5267
5268 for_each_possible_cpu(i)
5269 init_llist_head(&per_cpu(blk_cpu_done, i));
5270 for_each_possible_cpu(i)
5271 INIT_CSD(&per_cpu(blk_cpu_csd, i),
5272 __blk_mq_complete_request_remote, NULL);
5273 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
5274
5275 cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
5276 "block/softirq:dead", NULL,
5277 blk_softirq_cpu_dead);
5278 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
5279 blk_mq_hctx_notify_dead);
5280 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
5281 blk_mq_hctx_notify_online,
5282 blk_mq_hctx_notify_offline);
5283 return 0;
5284 }
5285 subsys_initcall(blk_mq_init);