diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index c217c0e87018..ad6358e9556b 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -180,10 +180,14 @@ static int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
 		int	tmp;
 
 		/* Skip out endpoints */
-		if (!dep || !dep->direction ||
-		    !(dep->flags & DWC3_EP_ENABLED))
+		if (!dep || !dep->direction)
 			continue;
 
+		if (!(dep->flags & DWC3_EP_ENABLED)) {
+			fifo_number++;
+			continue;
+		}
+
 		if (usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
 			mult = 3;
 		} else if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {