From 0c5273e138e6a62ef64d63b2b0e030d673d6a443 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 21 Jul 2023 10:27:30 -0700 Subject: [PATCH] BACKPORT: FROMGIT: block: Improve performance for BLK_MQ_F_BLOCKING drivers blk_mq_run_queue() runs the queue asynchronously if BLK_MQ_F_BLOCKING has been set. This is suboptimal since running the queue asynchronously is slower than running the queue synchronously. This patch modifies blk_mq_run_queue() as follows if BLK_MQ_F_BLOCKING has been set: - Run the queue synchronously if it is allowed to sleep. - Run the queue asynchronously if it is not allowed to sleep. Additionally, blk_mq_run_hw_queue(hctx, false) calls are modified into blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING) if the caller may be invoked from atomic context. The following caller chains have been reviewed: blk_mq_run_hw_queue(hctx, false) blk_mq_get_tag() /* may sleep, hence the functions it calls may also sleep */ blk_execute_rq() /* may sleep */ blk_mq_run_hw_queues(q, async=false) blk_freeze_queue_start() /* may sleep */ blk_mq_requeue_work() /* may sleep */ scsi_kick_queue() scsi_requeue_run_queue() /* may sleep */ scsi_run_host_queues() scsi_ioctl_reset() /* may sleep */ blk_mq_insert_requests(hctx, ctx, list, run_queue_async=false) blk_mq_dispatch_plug_list(plug, from_sched=false) blk_mq_flush_plug_list(plug, from_schedule=false) __blk_flush_plug(plug, from_schedule=false) blk_add_rq_to_plug() blk_mq_submit_bio() /* may sleep if REQ_NOWAIT has not been set */ blk_mq_plug_issue_direct() blk_mq_flush_plug_list() /* see above */ blk_mq_dispatch_plug_list(plug, from_sched=false) blk_mq_flush_plug_list() /* see above */ blk_mq_try_issue_directly() blk_mq_submit_bio() /* may sleep if REQ_NOWAIT has not been set */ blk_mq_try_issue_list_directly(hctx, list) blk_mq_insert_requests() /* see above */ Cc: Christoph Hellwig Cc: Ming Lei Change-Id: I597e0fa587057ac22e52f73351cbfa1de2c72a6b Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20230721172731.955724-4-bvanassche@acm.org Signed-off-by: Jens Axboe Bug: 291379528 (cherry picked from commit 65a558f66c308251e256317957b75d1e643c33c3 git://git.kernel.dk/linux-block for-next) Signed-off-by: Bart Van Assche --- block/blk-exec.c | 5 ++++- block/blk-mq.c | 13 ++++++------- drivers/scsi/scsi_lib.c | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/block/blk-exec.c b/block/blk-exec.c index d6cd501c0d34..f58ffcad8280 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -59,7 +59,10 @@ void blk_execute_rq_nowait(struct gendisk *bd_disk, struct request *rq, * don't check dying flag for MQ because the request won't * be reused after dying flag is set */ - blk_mq_sched_insert_request(rq, at_head, true, false); + blk_mq_sched_insert_request(rq, at_head, true, + rq->mq_hctx->flags & BLK_MQ_F_BLOCKING && + rq->cmd_flags & REQ_NOWAIT); + } EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); diff --git a/block/blk-mq.c b/block/blk-mq.c index 474ec2fdceb8..9da923d49cb2 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1561,11 +1561,9 @@ static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, if (unlikely(blk_mq_hctx_stopped(hctx))) return; - if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { - if (cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) { - __blk_mq_run_hw_queue(hctx); - return; - } + if (!async && cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) { + __blk_mq_run_hw_queue(hctx); + return; } kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, @@ -1768,7 +1766,7 @@ void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) { clear_bit(BLK_MQ_S_STOPPED, &hctx->state); - blk_mq_run_hw_queue(hctx, false); + blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING); } EXPORT_SYMBOL(blk_mq_start_hw_queue); @@ -1798,7 +1796,8 @@ void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) int i; queue_for_each_hw_ctx(q, hctx, i) - blk_mq_start_stopped_hw_queue(hctx, async); + blk_mq_start_stopped_hw_queue(hctx, async || + (hctx->flags & BLK_MQ_F_BLOCKING)); } EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 77f379649f64..8d7bb439855f 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -328,7 +328,8 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev) * but in most cases, we will be first. Ideally, each LU on the * target would get some limited time or requests on the target. */ - blk_mq_run_hw_queues(current_sdev->request_queue, false); + blk_mq_run_hw_queues(current_sdev->request_queue, + shost->queuecommand_may_block); spin_lock_irqsave(shost->host_lock, flags); if (starget->starget_sdev_user)