diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 2f4d79d53a33..5b362eda94b1 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -1042,6 +1042,7 @@ struct dwc3_scratchpad_array {
  * 	3	- Reserved
  * @dis_metastability_quirk: set to disable metastability quirk.
  * @needs_fifo_resize: set if we want to resize TXFIFO.
+ * @fifo_resize_status: true if the TXFIFOs have been resized.
  * @drd_connected: true when usb connected to a host or a device(drd mode),
  *			false otherwise.
  * @en_runtime: true when need runtime PM management. For example, RK3399 need
@@ -1237,6 +1238,7 @@ struct dwc3 {
 
 	unsigned		dis_metastability_quirk:1;
 	unsigned		needs_fifo_resize:1;
+	unsigned		fifo_resize_status:1;
 	unsigned		drd_connected:1;
 	unsigned		en_runtime:1;
 
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 1f7be980e44b..a58e776145b9 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -164,7 +164,7 @@ static int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
 	int		mdwidth;
 	u8		num, fifo_number;
 
-	if (!dwc->needs_fifo_resize)
+	if (!dwc->needs_fifo_resize || dwc->fifo_resize_status)
 		return 0;
 
 	mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
@@ -223,6 +223,8 @@ static int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
 		fifo_number++;
 	}
 
+	dwc->fifo_resize_status = true;
+
 	return 0;
 }
 
@@ -890,7 +892,8 @@ static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
-	dwc3_gadget_resize_tx_fifos(dwc);
+	if (usb_endpoint_xfer_isoc(ep->desc))
+		dwc3_gadget_resize_tx_fifos(dwc);
 
 	return ret;
 }
@@ -2192,6 +2195,7 @@ static int dwc3_gadget_stop(struct usb_gadget *g)
 	__dwc3_gadget_stop(dwc);
 
 out1:
+	dwc->fifo_resize_status	= false;
 	dwc->gadget_driver	= NULL;
 	spin_unlock_irqrestore(&dwc->lock, flags);