shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-10 12:57:06 +09:00
Code Issues Packages Projects Releases Wiki Activity

Revert "arm64: Avoid redundant type conversions in xchg() and cmpxchg()"

Browse Source 
This reverts commit 73152d61e4.

The LL/SC atomics are exported to modules and so we cannot break the
ABI by changing their names. Since the original commit just removes some
redundant masking operations, there is no functional impact in reverting
it to maintain the ABI.

Cc: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: Ibebe6a63ead13ca566e0e0ff7e58eb325119f620
Signed-off-by: Will Deacon <willdeacon@google.com>
This commit is contained in:
Will Deacon
2021-03-08 10:36:37 +00:00
parent 4266b7dc92
commit 6164fcac06
3 changed files with 108 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
arch/arm64/include/asm/atomic_ll_sc.h
View File

@@ -248,54 +248,53 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
}
__LL_SC_EXPORT(atomic64_dec_if_positive);
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \
__LL_SC_INLINE u##sz \
__LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
unsigned long old, \
u##sz new)) \
#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl, constraint) \
__LL_SC_INLINE unsigned long \
__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
unsigned long old, \
unsigned long new)) \
{ \
unsigned long tmp; \
u##sz oldval; \
unsigned long tmp, oldval; \
\
asm volatile( \
" prfm pstl1strm, %[v]\n" \
"1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \
"1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \
" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
" cbnz %" #w "[tmp], 2f\n" \
" st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \
" st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
" cbnz %w[tmp], 1b\n" \
" " #mb "\n" \
"2:" \
: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
[v] "+Q" (*(u##sz *)ptr) \
: [old] #constraint "r" (old), [new] "r" (new) \
[v] "+Q" (*(unsigned long *)ptr) \
: [old] #constraint "r" (old), [new] "r" (new) \
: cl); \
\
return oldval; \
} \
__LL_SC_EXPORT(__cmpxchg_case_##name##sz);
__LL_SC_EXPORT(__cmpxchg_case_##name);
/*
* Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
* handle the 'K' constraint for the value 4294967295 - thus we use no
* constraint for 32 bit operations.
*/
__CMPXCHG_CASE(w, b, , 8, , , , , )
__CMPXCHG_CASE(w, h, , 16, , , , , )
__CMPXCHG_CASE(w, , , 32, , , , , )
__CMPXCHG_CASE( , , , 64, , , , , L)
__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", )
__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", )
__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", )
__CMPXCHG_CASE( , , acq_, 64, , a, , "memory", L)
__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory", )
__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", )
__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", )
__CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L)
__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", )
__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", )
__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", )
__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L)
__CMPXCHG_CASE(w, b, 1, , , , , )
__CMPXCHG_CASE(w, h, 2, , , , , )
__CMPXCHG_CASE(w, , 4, , , , , )
__CMPXCHG_CASE( , , 8, , , , , L)
__CMPXCHG_CASE(w, b, acq_1, , a, , "memory", )
__CMPXCHG_CASE(w, h, acq_2, , a, , "memory", )
__CMPXCHG_CASE(w, , acq_4, , a, , "memory", )
__CMPXCHG_CASE( , , acq_8, , a, , "memory", L)
__CMPXCHG_CASE(w, b, rel_1, , , l, "memory", )
__CMPXCHG_CASE(w, h, rel_2, , , l, "memory", )
__CMPXCHG_CASE(w, , rel_4, , , l, "memory", )
__CMPXCHG_CASE( , , rel_8, , , l, "memory", L)
__CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory", )
__CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory", )
__CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory", )
__CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory", L)
#undef __CMPXCHG_CASE

46
arch/arm64/include/asm/atomic_lse.h
View File

@@ -480,24 +480,24 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \
static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr, \
unsigned long old, \
u##sz new) \
#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
unsigned long old, \
unsigned long new) \
{ \
register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
register unsigned long x1 asm ("x1") = old; \
register u##sz x2 asm ("x2") = new; \
register unsigned long x2 asm ("x2") = new; \
\
asm volatile( \
__LSE_PREAMBLE \
ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
__LL_SC_CMPXCHG(name##sz) \
__LL_SC_CMPXCHG(name) \
__nops(2), \
/* LSE atomics */ \
" mov " #w "30, %" #w "[old]\n" \
" cas" #mb #sfx "\t" #w "30, %" #w "[new], %[v]\n" \
" cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
" mov %" #w "[ret], " #w "30") \
: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
: [old] "r" (x1), [new] "r" (x2) \
@@ -506,22 +506,22 @@ static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr, \
return x0; \
}
__CMPXCHG_CASE(w, b, , 8, )
__CMPXCHG_CASE(w, h, , 16, )
__CMPXCHG_CASE(w, , , 32, )
__CMPXCHG_CASE(x, , , 64, )
__CMPXCHG_CASE(w, b, acq_, 8, a, "memory")
__CMPXCHG_CASE(w, h, acq_, 16, a, "memory")
__CMPXCHG_CASE(w, , acq_, 32, a, "memory")
__CMPXCHG_CASE(x, , acq_, 64, a, "memory")
__CMPXCHG_CASE(w, b, rel_, 8, l, "memory")
__CMPXCHG_CASE(w, h, rel_, 16, l, "memory")
__CMPXCHG_CASE(w, , rel_, 32, l, "memory")
__CMPXCHG_CASE(x, , rel_, 64, l, "memory")
__CMPXCHG_CASE(w, b, mb_, 8, al, "memory")
__CMPXCHG_CASE(w, h, mb_, 16, al, "memory")
__CMPXCHG_CASE(w, , mb_, 32, al, "memory")
__CMPXCHG_CASE(x, , mb_, 64, al, "memory")
__CMPXCHG_CASE(w, b, 1, )
__CMPXCHG_CASE(w, h, 2, )
__CMPXCHG_CASE(w, , 4, )
__CMPXCHG_CASE(x, , 8, )
__CMPXCHG_CASE(w, b, acq_1, a, "memory")
__CMPXCHG_CASE(w, h, acq_2, a, "memory")
__CMPXCHG_CASE(w, , acq_4, a, "memory")
__CMPXCHG_CASE(x, , acq_8, a, "memory")
__CMPXCHG_CASE(w, b, rel_1, l, "memory")
__CMPXCHG_CASE(w, h, rel_2, l, "memory")
__CMPXCHG_CASE(w, , rel_4, l, "memory")
__CMPXCHG_CASE(x, , rel_8, l, "memory")
__CMPXCHG_CASE(w, b, mb_1, al, "memory")
__CMPXCHG_CASE(w, h, mb_2, al, "memory")
__CMPXCHG_CASE(w, , mb_4, al, "memory")
__CMPXCHG_CASE(x, , mb_8, al, "memory")
#undef __LL_SC_CMPXCHG
#undef __CMPXCHG_CASE

116
arch/arm64/include/asm/cmpxchg.h
View File

@@ -30,46 +30,46 @@
* barrier case is generated as release+dmb for the former and
* acquire+release for the latter.
*/
#define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl) \
static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \
{ \
u##sz ret; \
unsigned long tmp; \
\
asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
" prfm pstl1strm, %2\n" \
"1: ld" #acq "xr" #sfx "\t%" #w "0, %2\n" \
" st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n" \
" cbnz %w1, 1b\n" \
" " #mb, \
/* LSE atomics */ \
" swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n" \
__nops(3) \
" " #nop_lse) \
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr) \
: "r" (x) \
: cl); \
\
return ret; \
#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \
static inline unsigned long __xchg_case_##name(unsigned long x, \
volatile void *ptr) \
{ \
unsigned long ret, tmp; \
\
asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
" prfm pstl1strm, %2\n" \
"1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \
" st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \
" cbnz %w1, 1b\n" \
" " #mb, \
/* LSE atomics */ \
" swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \
__nops(3) \
" " #nop_lse) \
: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned long *)ptr) \
: "r" (x) \
: cl); \
\
return ret; \
}
__XCHG_CASE(w, b, , 8, , , , , , )
__XCHG_CASE(w, h, , 16, , , , , , )
__XCHG_CASE(w, , , 32, , , , , , )
__XCHG_CASE( , , , 64, , , , , , )
__XCHG_CASE(w, b, acq_, 8, , , a, a, , "memory")
__XCHG_CASE(w, h, acq_, 16, , , a, a, , "memory")
__XCHG_CASE(w, , acq_, 32, , , a, a, , "memory")
__XCHG_CASE( , , acq_, 64, , , a, a, , "memory")
__XCHG_CASE(w, b, rel_, 8, , , , , l, "memory")
__XCHG_CASE(w, h, rel_, 16, , , , , l, "memory")
__XCHG_CASE(w, , rel_, 32, , , , , l, "memory")
__XCHG_CASE( , , rel_, 64, , , , , l, "memory")
__XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory")
__XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, b, 1, , , , , , )
__XCHG_CASE(w, h, 2, , , , , , )
__XCHG_CASE(w, , 4, , , , , , )
__XCHG_CASE( , , 8, , , , , , )
__XCHG_CASE(w, b, acq_1, , , a, a, , "memory")
__XCHG_CASE(w, h, acq_2, , , a, a, , "memory")
__XCHG_CASE(w, , acq_4, , , a, a, , "memory")
__XCHG_CASE( , , acq_8, , , a, a, , "memory")
__XCHG_CASE(w, b, rel_1, , , , , l, "memory")
__XCHG_CASE(w, h, rel_2, , , , , l, "memory")
__XCHG_CASE(w, , rel_4, , , , , l, "memory")
__XCHG_CASE( , , rel_8, , , , , l, "memory")
__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory")
__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory")
#undef __XCHG_CASE
@@ -80,13 +80,13 @@ static __always_inline unsigned long __xchg##sfx(unsigned long x, \
{ \
switch (size) { \
case 1: \
return __xchg_case##sfx##_8(x, ptr); \
return __xchg_case##sfx##_1(x, ptr); \
case 2: \
return __xchg_case##sfx##_16(x, ptr); \
return __xchg_case##sfx##_2(x, ptr); \
case 4: \
return __xchg_case##sfx##_32(x, ptr); \
return __xchg_case##sfx##_4(x, ptr); \
case 8: \
return __xchg_case##sfx##_64(x, ptr); \
return __xchg_case##sfx##_8(x, ptr); \
default: \
BUILD_BUG(); \
} \
@@ -123,13 +123,13 @@ static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
{ \
switch (size) { \
case 1: \
return __cmpxchg_case##sfx##_8(ptr, (u8)old, new); \
return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \
case 2: \
return __cmpxchg_case##sfx##_16(ptr, (u16)old, new); \
return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \
case 4: \
return __cmpxchg_case##sfx##_32(ptr, old, new); \
return __cmpxchg_case##sfx##_4(ptr, old, new); \
case 8: \
return __cmpxchg_case##sfx##_64(ptr, old, new); \
return __cmpxchg_case##sfx##_8(ptr, old, new); \
default: \
BUILD_BUG(); \
} \
@@ -197,16 +197,16 @@ __CMPXCHG_GEN(_mb)
__ret; \
})
#define __CMPWAIT_CASE(w, sfx, sz) \
static inline void __cmpwait_case_##sz(volatile void *ptr, \
unsigned long val) \
#define __CMPWAIT_CASE(w, sz, name) \
static inline void __cmpwait_case_##name(volatile void *ptr, \
unsigned long val) \
{ \
unsigned long tmp; \
\
asm volatile( \
" sevl\n" \
" wfe\n" \
" ldxr" #sfx "\t%" #w "[tmp], %[v]\n" \
" ldxr" #sz "\t%" #w "[tmp], %[v]\n" \
" eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
" cbnz %" #w "[tmp], 1f\n" \
" wfe\n" \
@@ -215,10 +215,10 @@ static inline void __cmpwait_case_##sz(volatile void *ptr, \
: [val] "r" (val)); \
}
__CMPWAIT_CASE(w, b, 8);
__CMPWAIT_CASE(w, h, 16);
__CMPWAIT_CASE(w, , 32);
__CMPWAIT_CASE( , , 64);
__CMPWAIT_CASE(w, b, 1);
__CMPWAIT_CASE(w, h, 2);
__CMPWAIT_CASE(w, , 4);
__CMPWAIT_CASE( , , 8);
#undef __CMPWAIT_CASE
@@ -229,13 +229,13 @@ static __always_inline void __cmpwait##sfx(volatile void *ptr, \
{ \
switch (size) { \
case 1: \
return __cmpwait_case##sfx##_8(ptr, (u8)val); \
return __cmpwait_case##sfx##_1(ptr, (u8)val); \
case 2: \
return __cmpwait_case##sfx##_16(ptr, (u16)val); \
return __cmpwait_case##sfx##_2(ptr, (u16)val); \
case 4: \
return __cmpwait_case##sfx##_32(ptr, val); \
return __cmpwait_case##sfx##_4(ptr, val); \
case 8: \
return __cmpwait_case##sfx##_64(ptr, val); \
return __cmpwait_case##sfx##_8(ptr, val); \
default: \
BUILD_BUG(); \
} \