diff --git a/fs/namei.c b/fs/namei.c
index 5e1c2ab2ae70..4248647f1ab2 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -187,7 +187,7 @@ getname_flags(const char __user *filename, int flags, int *empty)
 		}
 	}
 
-	atomic_set(&result->refcnt, 1);
+	result->refcnt = 1;
 	/* The empty path is special. */
 	if (unlikely(!len)) {
 		if (empty)
@@ -248,7 +248,7 @@ getname_kernel(const char * filename)
 	memcpy((char *)result->name, filename, len);
 	result->uptr = NULL;
 	result->aname = NULL;
-	atomic_set(&result->refcnt, 1);
+	result->refcnt = 1;
 	audit_getname(result);
 
 	return result;
@@ -259,10 +259,9 @@ void putname(struct filename *name)
 	if (IS_ERR(name))
 		return;
 
-	if (WARN_ON_ONCE(!atomic_read(&name->refcnt)))
-		return;
+	BUG_ON(name->refcnt <= 0);
 
-	if (!atomic_dec_and_test(&name->refcnt))
+	if (--name->refcnt > 0)
 		return;
 
 	if (name->name != name->iname) {
diff --git a/include/linux/fs.h b/include/linux/fs.h
index ba93b8caacf8..565829e270fa 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -2745,7 +2745,7 @@ struct audit_names;
 struct filename {
 	const char		*name;	/* pointer to actual string */
 	const __user char	*uptr;	/* original userland pointer */
-	atomic_t		refcnt;
+	int			refcnt;
 	struct audit_names	*aname;
 	const char		iname[];
 };
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index c5f41fc75d54..a2240f54fc22 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -2208,7 +2208,7 @@ __audit_reusename(const __user char *uptr)
 		if (!n->name)
 			continue;
 		if (n->name->uptr == uptr) {
-			atomic_inc(&n->name->refcnt);
+			n->name->refcnt++;
 			return n->name;
 		}
 	}
@@ -2237,7 +2237,7 @@ void __audit_getname(struct filename *name)
 	n->name = name;
 	n->name_len = AUDIT_NAME_FULL;
 	name->aname = n;
-	atomic_inc(&name->refcnt);
+	name->refcnt++;
 }
 
 static inline int audit_copy_fcaps(struct audit_names *name,
@@ -2369,7 +2369,7 @@ out_alloc:
 		return;
 	if (name) {
 		n->name = name;
-		atomic_inc(&name->refcnt);
+		name->refcnt++;
 	}
 
 out:
@@ -2496,7 +2496,7 @@ void __audit_inode_child(struct inode *parent,
 		if (found_parent) {
 			found_child->name = found_parent->name;
 			found_child->name_len = AUDIT_NAME_FULL;
-			atomic_inc(&found_child->name->refcnt);
+			found_child->name->refcnt++;
 		}
 	}