diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index a58e776145b9..0ca339cc2f96 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -892,7 +892,8 @@ static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
-	if (usb_endpoint_xfer_isoc(ep->desc))
+	if (usb_endpoint_xfer_isoc(ep->desc) &&
+	    usb_endpoint_maxp(ep->desc) >= 1024)
 		dwc3_gadget_resize_tx_fifos(dwc);
 
 	return ret;