diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index d23f8ea63082..fb0fb8825574 100644
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -1482,6 +1482,8 @@ static void htb_qlen_notify(struct Qdisc *sch, unsigned long arg)
 {
 	struct htb_class *cl = (struct htb_class *)arg;
 
+	if (!cl->prio_activity)
+		return;
 	htb_deactivate(qdisc_priv(sch), cl);
 }