diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c
index 21ae29dd5075..9ede3e7a8e48 100644
--- a/drivers/usb/gadget/function/f_accessory.c
+++ b/drivers/usb/gadget/function/f_accessory.c
@@ -302,6 +302,12 @@ static void __put_acc_dev(struct kref *kref)
 	struct acc_dev_ref *ref = container_of(kref, struct acc_dev_ref, kref);
 	struct acc_dev *dev = ref->acc_dev;
 
+	/* Cancel any async work */
+	cancel_delayed_work_sync(&dev->start_work);
+	cancel_work_sync(&dev->getprotocol_work);
+	cancel_work_sync(&dev->sendstring_work);
+	cancel_work_sync(&dev->hid_work);
+
 	ref->acc_dev = NULL;
 	kfree(dev);
 }