Commit 5c361425 authored by Marco Elver's avatar Marco Elver Committed by Ingo Molnar
Browse files

kcsan: Prefer __always_inline for fast-path 



Prefer __always_inline for fast-path functions that are called outside
of user_access_save, to avoid generating UACCESS warnings when
optimizing for size (CC_OPTIMIZE_FOR_SIZE). It will also avoid future
surprises with compiler versions that change the inlining heuristic even
when optimizing for performance.

Reported-by: default avatarRandy Dunlap <rdunlap@infradead.org>
Acked-by: Randy Dunlap <rdunlap@infradead.org> # build-tested
Signed-off-by: default avatarMarco Elver <elver@google.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Link: http://lkml.kernel.org/r/58708908-84a0-0a81-a836-ad97e33dbb62@infradead.org
parent df10846f

kernel/kcsan/atomic.h

+1 −1
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ 
 * than cast to volatile. Eventually, we hope to be able to remove this

 * function.

 */

static inline bool kcsan_is_atomic(const volatile void *ptr)

static __always_inline bool kcsan_is_atomic(const volatile void *ptr)

{

	/* only jiffies for now */

	return ptr == &jiffies;

kernel/kcsan/core.c

+9 −9
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1]; 
 */

static DEFINE_PER_CPU(long, kcsan_skip);



static inline atomic_long_t *find_watchpoint(unsigned long addr,

static __always_inline atomic_long_t *find_watchpoint(unsigned long addr,

						      size_t size,

						      bool expect_write,

						      long *encoded_watchpoint)
@@ -146,7 +146,7 @@ insert_watchpoint(unsigned long addr, size_t size, bool is_write) 
 *	2. the thread that set up the watchpoint already removed it;

 *	3. the watchpoint was removed and then re-used.

 */

static inline bool

static __always_inline bool

try_consume_watchpoint(atomic_long_t *watchpoint, long encoded_watchpoint)

{

	return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint, CONSUMED_WATCHPOINT);
@@ -160,7 +160,7 @@ static inline bool remove_watchpoint(atomic_long_t *watchpoint) 
	return atomic_long_xchg_relaxed(watchpoint, INVALID_WATCHPOINT) != CONSUMED_WATCHPOINT;

}



static inline struct kcsan_ctx *get_ctx(void)

static __always_inline struct kcsan_ctx *get_ctx(void)

{

	/*

	 * In interrupts, use raw_cpu_ptr to avoid unnecessary checks, that would
@@ -169,7 +169,7 @@ static inline struct kcsan_ctx *get_ctx(void) 
	return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);

}



static inline bool is_atomic(const volatile void *ptr)

static __always_inline bool is_atomic(const volatile void *ptr)

{

	struct kcsan_ctx *ctx = get_ctx();
@@ -193,7 +193,7 @@ static inline bool is_atomic(const volatile void *ptr) 
	return kcsan_is_atomic(ptr);

}



static inline bool should_watch(const volatile void *ptr, int type)

static __always_inline bool should_watch(const volatile void *ptr, int type)

{

	/*

	 * Never set up watchpoints when memory operations are atomic.
@@ -226,7 +226,7 @@ static inline void reset_kcsan_skip(void) 
	this_cpu_write(kcsan_skip, skip_count);

}



static inline bool kcsan_is_enabled(void)

static __always_inline bool kcsan_is_enabled(void)

{

	return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;

}

kernel/kcsan/encoding.h

+7 −7
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ encode_watchpoint(unsigned long addr, size_t size, bool is_write) 
		      (addr & WATCHPOINT_ADDR_MASK));

}



static inline bool decode_watchpoint(long watchpoint,

static __always_inline bool decode_watchpoint(long watchpoint,

					      unsigned long *addr_masked,

					      size_t *size,

					      bool *is_write)
@@ -78,12 +78,12 @@ static inline bool decode_watchpoint(long watchpoint, 
/*

 * Return watchpoint slot for an address.

 */

static inline int watchpoint_slot(unsigned long addr)

static __always_inline int watchpoint_slot(unsigned long addr)

{

	return (addr / PAGE_SIZE) % CONFIG_KCSAN_NUM_WATCHPOINTS;

}



static inline bool matching_access(unsigned long addr1, size_t size1,

static __always_inline bool matching_access(unsigned long addr1, size_t size1,

					    unsigned long addr2, size_t size2)

{

	unsigned long end_range1 = addr1 + size1 - 1;