diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index ad74b2e8da11..3ee66b7a095a 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -241,11 +241,7 @@ struct bpf_map {
 	 */
 	atomic64_t refcnt ____cacheline_aligned;
 	atomic64_t usercnt;
-	/* rcu is used before freeing and work is only used during freeing */
-	union {
-		struct work_struct work;
-		struct rcu_head rcu;
-	};
+	struct work_struct work;
 	struct mutex freeze_mutex;
 	atomic64_t writecnt;
 	/* 'Ownership' of program-containing map is claimed by the first program
@@ -261,7 +257,6 @@ struct bpf_map {
 	} owner;
 	bool bypass_spec_v1;
 	bool frozen; /* write-once; write-protected by freeze_mutex */
-	bool free_after_mult_rcu_gp;
 	s64 __percpu *elem_count;
 };
 
diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index 141f3332038c..47ecc4818c93 100644
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -117,15 +117,10 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map,
 
 void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
 {
-	struct bpf_map *inner_map = ptr;
-
-	/* The inner map may still be used by both non-sleepable and sleepable
-	 * bpf program, so free it after one RCU grace period and one tasks
-	 * trace RCU grace period.
+	/* ptr->ops->map_free() has to go through one
+	 * rcu grace period by itself.
 	 */
-	if (need_defer)
-		WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true);
-	bpf_map_put(inner_map);
+	bpf_map_put(ptr);
 }
 
 u32 bpf_map_fd_sys_lookup_elem(void *ptr)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 8598799d1fc8..97b72d165b06 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -630,28 +630,6 @@ static void bpf_map_put_uref(struct bpf_map *map)
 	}
 }
 
-static void bpf_map_free_in_work(struct bpf_map *map)
-{
-	INIT_WORK(&map->work, bpf_map_free_deferred);
-	/* Avoid spawning kworkers, since they all might contend
-	 * for the same mutex like slab_mutex.
-	 */
-	queue_work(system_unbound_wq, &map->work);
-}
-
-static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
-{
-	bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
-}
-
-static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
-{
-	if (rcu_trace_implies_rcu_gp())
-		bpf_map_free_rcu_gp(rcu);
-	else
-		call_rcu(rcu, bpf_map_free_rcu_gp);
-}
-
 /* decrement map refcnt and schedule it for freeing via workqueue
  * (unrelying map implementation ops->map_free() might sleep)
  */
@@ -661,11 +639,11 @@ static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 		/* bpf_map_free_id() must be called first */
 		bpf_map_free_id(map, do_idr_lock);
 		btf_put(map->btf);
-
-		if (READ_ONCE(map->free_after_mult_rcu_gp))
-			call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
-		else
-			bpf_map_free_in_work(map);
+		INIT_WORK(&map->work, bpf_map_free_deferred);
+		/* Avoid spawning kworkers, since they all might contend
+		 * for the same mutex like slab_mutex.
+		 */
+		queue_work(system_unbound_wq, &map->work);
 	}
 }