diff --git a/lib/sbitmap.c b/lib/sbitmap.c index c515072eca29..443accfe685d 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -555,12 +555,12 @@ void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq, } EXPORT_SYMBOL_GPL(sbitmap_queue_min_shallow_depth); -static void __sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr) +static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq) { int i, wake_index; if (!atomic_read(&sbq->ws_active)) - return; + return NULL; wake_index = atomic_read(&sbq->wake_index); for (i = 0; i < SBQ_WAIT_QUEUES; i++) { @@ -574,22 +574,20 @@ static void __sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr) */ wake_index = sbq_index_inc(wake_index); - /* - * It is sufficient to wake up at least one waiter to - * guarantee forward progress. - */ - if (waitqueue_active(&ws->wait) && - wake_up_nr(&ws->wait, nr)) - break; + if (waitqueue_active(&ws->wait)) { + if (wake_index != atomic_read(&sbq->wake_index)) + atomic_set(&sbq->wake_index, wake_index); + return ws; + } } - if (wake_index != atomic_read(&sbq->wake_index)) - atomic_set(&sbq->wake_index, wake_index); + return NULL; } void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr) { unsigned int wake_batch = READ_ONCE(sbq->wake_batch); + struct sbq_wait_state *ws = NULL; unsigned int wakeups; if (!atomic_read(&sbq->ws_active)) @@ -601,10 +599,16 @@ void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr) do { if (atomic_read(&sbq->completion_cnt) - wakeups < wake_batch) return; + + if (!ws) { + ws = sbq_wake_ptr(sbq); + if (!ws) + return; + } } while (!atomic_try_cmpxchg(&sbq->wakeup_cnt, &wakeups, wakeups + wake_batch)); - __sbitmap_queue_wake_up(sbq, wake_batch); + wake_up_nr(&ws->wait, wake_batch); } EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up);