shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-03-27 04:50:25 +09:00
Code Issues Packages Projects Releases Wiki Activity

ANDROID: block: Preserve the order of requeued requests

Browse Source 
If a queue is run before all requeued requests have been sent to the I/O
scheduler, the I/O scheduler may dispatch the wrong request. Fix this by
making __blk_mq_run_hw_queue() process the requeue_list instead of
blk_mq_requeue_work().

Bug: 275581839
Change-Id: Id72c0787607f90857ae8e75998d532b96b46b8c2
Signed-off-by: Bart Van Assche <bvanassche@google.com>
This commit is contained in:
Bart Van Assche
2023-04-05 15:16:37 -07:00
committed by Treehugger Robot
parent a08e170d90
commit 5d90f7cbaf
1 changed files with 28 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
block/blk-mq.c
View File

@@ -71,6 +71,7 @@ static int blk_mq_poll_stats_bkt(const struct request *rq)
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
{
return !list_empty_careful(&hctx->dispatch) ||
!list_empty_careful(&hctx->queue->requeue_list) ||
sbitmap_any_bit_set(&hctx->ctx_map) ||
blk_mq_sched_has_work(hctx);
}
@@ -771,33 +772,36 @@ void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
}
EXPORT_SYMBOL(blk_mq_requeue_request);
static void blk_mq_requeue_work(struct work_struct *work)
static bool blk_mq_has_sqsched(struct request_queue *q);
static void blk_mq_process_requeue_list(struct blk_mq_hw_ctx *hctx)
{
struct request_queue *q =
container_of(work, struct request_queue, requeue_work.work);
LIST_HEAD(rq_list);
struct request_queue *q = hctx->queue;
struct request *rq, *next;
LIST_HEAD(at_head);
LIST_HEAD(at_tail);
if (list_empty_careful(&q->requeue_list))
return;
spin_lock_irq(&q->requeue_lock);
list_splice_init(&q->requeue_list, &rq_list);
list_for_each_entry_safe(rq, next, &q->requeue_list, queuelist) {
if (!blk_mq_has_sqsched(q) && rq->mq_hctx != hctx)
continue;
if (rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP)) {
rq->rq_flags &= ~RQF_SOFTBARRIER;
list_move(&rq->queuelist, &at_head);
} else {
list_move(&rq->queuelist, &at_tail);
}
}
spin_unlock_irq(&q->requeue_lock);
list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP)))
continue;
rq->rq_flags &= ~RQF_SOFTBARRIER;
list_del_init(&rq->queuelist);
list_for_each_entry_safe(rq, next, &at_head, queuelist)
blk_mq_sched_insert_request(rq, /*at_head=*/true, false, false);
}
while (!list_empty(&rq_list)) {
rq = list_entry(rq_list.next, struct request, queuelist);
list_del_init(&rq->queuelist);
list_for_each_entry_safe(rq, next, &at_tail, queuelist)
blk_mq_sched_insert_request(rq, false, false, false);
}
blk_mq_run_hw_queues(q, false);
}
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
@@ -827,15 +831,14 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
void blk_mq_kick_requeue_list(struct request_queue *q)
{
kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
blk_mq_run_hw_queues(q, /*async=*/in_atomic());
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
unsigned long msecs)
{
kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
msecs_to_jiffies(msecs));
blk_mq_delay_run_hw_queues(q, msecs);
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
@@ -1498,6 +1501,8 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
blk_mq_process_requeue_list(hctx);
hctx_lock(hctx, &srcu_idx);
blk_mq_sched_dispatch_requests(hctx);
hctx_unlock(hctx, srcu_idx);
@@ -1686,7 +1691,7 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async)
* scheduler.
*/
if (!sq_hctx || sq_hctx == hctx ||
!list_empty_careful(&hctx->dispatch))
blk_mq_hctx_has_pending(hctx))
blk_mq_run_hw_queue(hctx, async);
}
}
@@ -1714,7 +1719,7 @@ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
* scheduler.
*/
if (!sq_hctx || sq_hctx == hctx ||
!list_empty_careful(&hctx->dispatch))
blk_mq_hctx_has_pending(hctx))
blk_mq_delay_run_hw_queue(hctx, msecs);
}
}
@@ -3295,7 +3300,6 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
set->map[HCTX_TYPE_POLL].nr_queues)
blk_queue_flag_set(QUEUE_FLAG_POLL, q);
INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
INIT_LIST_HEAD(&q->requeue_list);
spin_lock_init(&q->requeue_lock);
@@ -4016,8 +4020,6 @@ void blk_mq_cancel_work_sync(struct request_queue *q)
struct blk_mq_hw_ctx *hctx;
int i;
cancel_delayed_work_sync(&q->requeue_work);
queue_for_each_hw_ctx(q, hctx, i)
cancel_delayed_work_sync(&hctx->run_work);
}