Commit 7aab7aa4 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

x86/atomics: Always inline arch_atomic64*() 



As already done for regular arch_atomic*(), always inline arch_atomic64*().

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20230126151323.585115019@infradead.org
parent 3017ba4b

arch/x86/include/asm/atomic64_32.h

+22 −22
Original line number Diff line number Diff line
@@ -71,7 +71,7 @@ ATOMIC64_DECL(add_unless); 
 * the old value.

 */



static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)

static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)

{

	return arch_cmpxchg64(&v->counter, o, n);

}
@@ -85,7 +85,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) 
 * Atomically xchgs the value of @v to @n and returns

 * the old value.

 */

static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)

static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)

{

	s64 o;

	unsigned high = (unsigned)(n >> 32);
@@ -104,7 +104,7 @@ static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) 
 *

 * Atomically sets the value of @v to @n.

 */

static inline void arch_atomic64_set(atomic64_t *v, s64 i)

static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)

{

	unsigned high = (unsigned)(i >> 32);

	unsigned low = (unsigned)i;
@@ -119,7 +119,7 @@ static inline void arch_atomic64_set(atomic64_t *v, s64 i) 
 *

 * Atomically reads the value of @v and returns it.

 */

static inline s64 arch_atomic64_read(const atomic64_t *v)

static __always_inline s64 arch_atomic64_read(const atomic64_t *v)

{

	s64 r;

	alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
@@ -133,7 +133,7 @@ static inline s64 arch_atomic64_read(const atomic64_t *v) 
 *

 * Atomically adds @i to @v and returns @i + *@v

 */

static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)

{

	alternative_atomic64(add_return,

			     ASM_OUTPUT2("+A" (i), "+c" (v)),
@@ -145,7 +145,7 @@ static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) 
/*

 * Other variants with different arithmetic operators:

 */

static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)

{

	alternative_atomic64(sub_return,

			     ASM_OUTPUT2("+A" (i), "+c" (v)),
@@ -154,7 +154,7 @@ static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) 
}

#define arch_atomic64_sub_return arch_atomic64_sub_return



static inline s64 arch_atomic64_inc_return(atomic64_t *v)

static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v)

{

	s64 a;

	alternative_atomic64(inc_return, "=&A" (a),
@@ -163,7 +163,7 @@ static inline s64 arch_atomic64_inc_return(atomic64_t *v) 
}

#define arch_atomic64_inc_return arch_atomic64_inc_return



static inline s64 arch_atomic64_dec_return(atomic64_t *v)

static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v)

{

	s64 a;

	alternative_atomic64(dec_return, "=&A" (a),
@@ -179,7 +179,7 @@ static inline s64 arch_atomic64_dec_return(atomic64_t *v) 
 *

 * Atomically adds @i to @v.

 */

static inline s64 arch_atomic64_add(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_add(s64 i, atomic64_t *v)

{

	__alternative_atomic64(add, add_return,

			       ASM_OUTPUT2("+A" (i), "+c" (v)),
@@ -194,7 +194,7 @@ static inline s64 arch_atomic64_add(s64 i, atomic64_t *v) 
 *

 * Atomically subtracts @i from @v.

 */

static inline s64 arch_atomic64_sub(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_sub(s64 i, atomic64_t *v)

{

	__alternative_atomic64(sub, sub_return,

			       ASM_OUTPUT2("+A" (i), "+c" (v)),
@@ -208,7 +208,7 @@ static inline s64 arch_atomic64_sub(s64 i, atomic64_t *v) 
 *

 * Atomically increments @v by 1.

 */

static inline void arch_atomic64_inc(atomic64_t *v)

static __always_inline void arch_atomic64_inc(atomic64_t *v)

{

	__alternative_atomic64(inc, inc_return, /* no output */,

			       "S" (v) : "memory", "eax", "ecx", "edx");
@@ -221,7 +221,7 @@ static inline void arch_atomic64_inc(atomic64_t *v) 
 *

 * Atomically decrements @v by 1.

 */

static inline void arch_atomic64_dec(atomic64_t *v)

static __always_inline void arch_atomic64_dec(atomic64_t *v)

{

	__alternative_atomic64(dec, dec_return, /* no output */,

			       "S" (v) : "memory", "eax", "ecx", "edx");
@@ -237,7 +237,7 @@ static inline void arch_atomic64_dec(atomic64_t *v) 
 * Atomically adds @a to @v, so long as it was not @u.

 * Returns non-zero if the add was done, zero otherwise.

 */

static inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)

static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)

{

	unsigned low = (unsigned)u;

	unsigned high = (unsigned)(u >> 32);
@@ -248,7 +248,7 @@ static inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 
}

#define arch_atomic64_add_unless arch_atomic64_add_unless



static inline int arch_atomic64_inc_not_zero(atomic64_t *v)

static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v)

{

	int r;

	alternative_atomic64(inc_not_zero, "=&a" (r),
@@ -257,7 +257,7 @@ static inline int arch_atomic64_inc_not_zero(atomic64_t *v) 
}

#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero



static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)

static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)

{

	s64 r;

	alternative_atomic64(dec_if_positive, "=&A" (r),
@@ -269,7 +269,7 @@ static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 
#undef alternative_atomic64

#undef __alternative_atomic64



static inline void arch_atomic64_and(s64 i, atomic64_t *v)

static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)

{

	s64 old, c = 0;
@@ -277,7 +277,7 @@ static inline void arch_atomic64_and(s64 i, atomic64_t *v) 
		c = old;

}



static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)

{

	s64 old, c = 0;
@@ -288,7 +288,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) 
}

#define arch_atomic64_fetch_and arch_atomic64_fetch_and



static inline void arch_atomic64_or(s64 i, atomic64_t *v)

static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)

{

	s64 old, c = 0;
@@ -296,7 +296,7 @@ static inline void arch_atomic64_or(s64 i, atomic64_t *v) 
		c = old;

}



static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)

{

	s64 old, c = 0;
@@ -307,7 +307,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) 
}

#define arch_atomic64_fetch_or arch_atomic64_fetch_or



static inline void arch_atomic64_xor(s64 i, atomic64_t *v)

static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)

{

	s64 old, c = 0;
@@ -315,7 +315,7 @@ static inline void arch_atomic64_xor(s64 i, atomic64_t *v) 
		c = old;

}



static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)

{

	s64 old, c = 0;
@@ -326,7 +326,7 @@ static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) 
}

#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor



static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)

{

	s64 old, c = 0;

arch/x86/include/asm/atomic64_64.h

+18 −18
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ 
 * Atomically reads the value of @v.

 * Doesn't imply a read memory barrier.

 */

static inline s64 arch_atomic64_read(const atomic64_t *v)

static __always_inline s64 arch_atomic64_read(const atomic64_t *v)

{

	return __READ_ONCE((v)->counter);

}
@@ -29,7 +29,7 @@ static inline s64 arch_atomic64_read(const atomic64_t *v) 
 *

 * Atomically sets the value of @v to @i.

 */

static inline void arch_atomic64_set(atomic64_t *v, s64 i)

static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)

{

	__WRITE_ONCE(v->counter, i);

}
@@ -55,7 +55,7 @@ static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v) 
 *

 * Atomically subtracts @i from @v.

 */

static inline void arch_atomic64_sub(s64 i, atomic64_t *v)

static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)

{

	asm volatile(LOCK_PREFIX "subq %1,%0"

		     : "=m" (v->counter)
@@ -71,7 +71,7 @@ static inline void arch_atomic64_sub(s64 i, atomic64_t *v) 
 * true if the result is zero, or false for all

 * other cases.

 */

static inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)

static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)

{

	return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);

}
@@ -113,7 +113,7 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v) 
 * returns true if the result is 0, or false for all other

 * cases.

 */

static inline bool arch_atomic64_dec_and_test(atomic64_t *v)

static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v)

{

	return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);

}
@@ -127,7 +127,7 @@ static inline bool arch_atomic64_dec_and_test(atomic64_t *v) 
 * and returns true if the result is zero, or false for all

 * other cases.

 */

static inline bool arch_atomic64_inc_and_test(atomic64_t *v)

static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v)

{

	return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);

}
@@ -142,7 +142,7 @@ static inline bool arch_atomic64_inc_and_test(atomic64_t *v) 
 * if the result is negative, or false when

 * result is greater than or equal to zero.

 */

static inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)

static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)

{

	return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);

}
@@ -161,25 +161,25 @@ static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) 
}

#define arch_atomic64_add_return arch_atomic64_add_return



static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)

{

	return arch_atomic64_add_return(-i, v);

}

#define arch_atomic64_sub_return arch_atomic64_sub_return



static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)

{

	return xadd(&v->counter, i);

}

#define arch_atomic64_fetch_add arch_atomic64_fetch_add



static inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)

{

	return xadd(&v->counter, -i);

}

#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub



static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)

static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)

{

	return arch_cmpxchg(&v->counter, old, new);

}
@@ -191,13 +191,13 @@ static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s 
}

#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg



static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)

static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)

{

	return arch_xchg(&v->counter, new);

}

#define arch_atomic64_xchg arch_atomic64_xchg



static inline void arch_atomic64_and(s64 i, atomic64_t *v)

static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)

{

	asm volatile(LOCK_PREFIX "andq %1,%0"

			: "+m" (v->counter)
@@ -205,7 +205,7 @@ static inline void arch_atomic64_and(s64 i, atomic64_t *v) 
			: "memory");

}



static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)

{

	s64 val = arch_atomic64_read(v);
@@ -215,7 +215,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) 
}

#define arch_atomic64_fetch_and arch_atomic64_fetch_and



static inline void arch_atomic64_or(s64 i, atomic64_t *v)

static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)

{

	asm volatile(LOCK_PREFIX "orq %1,%0"

			: "+m" (v->counter)
@@ -223,7 +223,7 @@ static inline void arch_atomic64_or(s64 i, atomic64_t *v) 
			: "memory");

}



static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)

{

	s64 val = arch_atomic64_read(v);
@@ -233,7 +233,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) 
}

#define arch_atomic64_fetch_or arch_atomic64_fetch_or



static inline void arch_atomic64_xor(s64 i, atomic64_t *v)

static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)

{

	asm volatile(LOCK_PREFIX "xorq %1,%0"

			: "+m" (v->counter)
@@ -241,7 +241,7 @@ static inline void arch_atomic64_xor(s64 i, atomic64_t *v) 
			: "memory");

}



static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)

static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)

{

	s64 val = arch_atomic64_read(v);