diff --git a/block/blk-mq.c b/block/blk-mq.c index 6fe8cc378209..faf8188133fa 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -71,6 +71,7 @@ static int blk_mq_poll_stats_bkt(const struct request *rq) static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) { return !list_empty_careful(&hctx->dispatch) || + !list_empty_careful(&hctx->queue->requeue_list) || sbitmap_any_bit_set(&hctx->ctx_map) || blk_mq_sched_has_work(hctx); } @@ -771,33 +772,36 @@ void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) } EXPORT_SYMBOL(blk_mq_requeue_request); -static void blk_mq_requeue_work(struct work_struct *work) +static bool blk_mq_has_sqsched(struct request_queue *q); + +static void blk_mq_process_requeue_list(struct blk_mq_hw_ctx *hctx) { - struct request_queue *q = - container_of(work, struct request_queue, requeue_work.work); - LIST_HEAD(rq_list); + struct request_queue *q = hctx->queue; struct request *rq, *next; + LIST_HEAD(at_head); + LIST_HEAD(at_tail); + + if (list_empty_careful(&q->requeue_list)) + return; spin_lock_irq(&q->requeue_lock); - list_splice_init(&q->requeue_list, &rq_list); + list_for_each_entry_safe(rq, next, &q->requeue_list, queuelist) { + if (!blk_mq_has_sqsched(q) && rq->mq_hctx != hctx) + continue; + if (rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP)) { + rq->rq_flags &= ~RQF_SOFTBARRIER; + list_move(&rq->queuelist, &at_head); + } else { + list_move(&rq->queuelist, &at_tail); + } + } spin_unlock_irq(&q->requeue_lock); - list_for_each_entry_safe(rq, next, &rq_list, queuelist) { - if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP))) - continue; - - rq->rq_flags &= ~RQF_SOFTBARRIER; - list_del_init(&rq->queuelist); + list_for_each_entry_safe(rq, next, &at_head, queuelist) blk_mq_sched_insert_request(rq, /*at_head=*/true, false, false); - } - while (!list_empty(&rq_list)) { - rq = list_entry(rq_list.next, struct request, queuelist); - list_del_init(&rq->queuelist); + list_for_each_entry_safe(rq, next, &at_tail, queuelist) blk_mq_sched_insert_request(rq, false, false, false); - } - - blk_mq_run_hw_queues(q, false); } void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, @@ -827,15 +831,14 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, void blk_mq_kick_requeue_list(struct request_queue *q) { - kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); + blk_mq_run_hw_queues(q, /*async=*/in_atomic()); } EXPORT_SYMBOL(blk_mq_kick_requeue_list); void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs) { - kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, - msecs_to_jiffies(msecs)); + blk_mq_delay_run_hw_queues(q, msecs); } EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); @@ -1498,6 +1501,8 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); + blk_mq_process_requeue_list(hctx); + hctx_lock(hctx, &srcu_idx); blk_mq_sched_dispatch_requests(hctx); hctx_unlock(hctx, srcu_idx); @@ -1686,7 +1691,7 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async) * scheduler. */ if (!sq_hctx || sq_hctx == hctx || - !list_empty_careful(&hctx->dispatch)) + blk_mq_hctx_has_pending(hctx)) blk_mq_run_hw_queue(hctx, async); } } @@ -1714,7 +1719,7 @@ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) * scheduler. */ if (!sq_hctx || sq_hctx == hctx || - !list_empty_careful(&hctx->dispatch)) + blk_mq_hctx_has_pending(hctx)) blk_mq_delay_run_hw_queue(hctx, msecs); } } @@ -3295,7 +3300,6 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, set->map[HCTX_TYPE_POLL].nr_queues) blk_queue_flag_set(QUEUE_FLAG_POLL, q); - INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); INIT_LIST_HEAD(&q->requeue_list); spin_lock_init(&q->requeue_lock); @@ -4016,8 +4020,6 @@ void blk_mq_cancel_work_sync(struct request_queue *q) struct blk_mq_hw_ctx *hctx; int i; - cancel_delayed_work_sync(&q->requeue_work); - queue_for_each_hw_ctx(q, hctx, i) cancel_delayed_work_sync(&hctx->run_work); }