diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index 267546a00987..56a45cd8505e 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -278,7 +278,7 @@ static void sync_file_free(struct kref *kref)
 	struct sync_file *sync_file = container_of(kref, struct sync_file,
 						     kref);
 
-	if (test_bit(POLL_ENABLED, &sync_file->flags))
+	if (test_bit(POLL_ENABLED, &sync_file->fence->flags))
 		fence_remove_callback(sync_file->fence, &sync_file->cb);
 	fence_put(sync_file->fence);
 	kfree(sync_file);
@@ -298,8 +298,7 @@ static unsigned int sync_file_poll(struct file *file, poll_table *wait)
 
 	poll_wait(file, &sync_file->wq, wait);
 
-	if (list_empty(&sync_file->cb.node) &&
-	    !test_and_set_bit(POLL_ENABLED, &sync_file->flags)) {
+	if (!test_and_set_bit(POLL_ENABLED, &sync_file->fence->flags)) {
 		if (fence_add_callback(sync_file->fence, &sync_file->cb,
 					   fence_check_cb_func) < 0)
 			wake_up_all(&sync_file->wq);
diff --git a/include/linux/sync_file.h b/include/linux/sync_file.h
index 35ec6c45e190..aa17ccfc2f57 100644
--- a/include/linux/sync_file.h
+++ b/include/linux/sync_file.h
@@ -40,13 +40,12 @@ struct sync_file {
 #endif
 
 	wait_queue_head_t	wq;
-	unsigned long		flags;
 
 	struct fence		*fence;
 	struct fence_cb cb;
 };
 
-#define POLL_ENABLED 0
+#define POLL_ENABLED FENCE_FLAG_USER_BITS
 
 struct sync_file *sync_file_create(struct fence *fence);
 struct fence *sync_file_get_fence(int fd);