diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c
index 97a6bd8bac90..592d59c94222 100644
--- a/drivers/usb/gadget/function/f_accessory.c
+++ b/drivers/usb/gadget/function/f_accessory.c
@@ -824,13 +824,13 @@ static int acc_release(struct inode *ip, struct file *fp)
 	if (!dev)
 		return -ENOENT;
 
-	WARN_ON(!atomic_xchg(&dev->open_excl, 0));
 	/* indicate that we are disconnected
 	 * still could be online so don't touch online flag
 	 */
 	dev->disconnected = 1;
 
 	fp->private_data = NULL;
+	WARN_ON(!atomic_xchg(&dev->open_excl, 0));
 	put_acc_dev(dev);
 	return 0;
 }