locking/lockdep: Handle statically initialized PER_CPU locks properly (383776fa) · Commits · jan.koester / Linux

include/linux/module.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -493,6 +493,7 @@ static inline int module_is_live(struct module *mod)
		struct module *__module_text_address(unsigned long addr);
		struct module *__module_address(unsigned long addr);
		bool is_module_address(unsigned long addr);
		bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr);
		bool is_module_percpu_address(unsigned long addr);
		bool is_module_text_address(unsigned long addr);

		@@ -660,6 +661,11 @@ static inline bool is_module_percpu_address(unsigned long addr)
		return false;
		}

		static inline bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
		{
		return false;
		}

		static inline bool is_module_text_address(unsigned long addr)
		{
		return false;

include/linux/percpu.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -110,6 +110,7 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
		#endif

		extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
		extern bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr);
		extern bool is_kernel_percpu_address(unsigned long addr);

		#if !defined(CONFIG_SMP) \|\| !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)

kernel/locking/lockdep.c

+23 −10

Original line number	Diff line number	Diff line
		@@ -660,6 +660,7 @@ look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)
		struct lockdep_subclass_key *key;
		struct hlist_head *hash_head;
		struct lock_class *class;
		bool is_static = false;

		if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
		debug_locks_off();
		@@ -673,10 +674,23 @@ look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)

		/*
		* Static locks do not have their class-keys yet - for them the key
		* is the lock object itself:
		*/
		if (unlikely(!lock->key))
		* is the lock object itself. If the lock is in the per cpu area,
		* the canonical address of the lock (per cpu offset removed) is
		* used.
		*/
		if (unlikely(!lock->key)) {
		unsigned long can_addr, addr = (unsigned long)lock;

		if (__is_kernel_percpu_address(addr, &can_addr))
		lock->key = (void *)can_addr;
		else if (__is_module_percpu_address(addr, &can_addr))
		lock->key = (void *)can_addr;
		else if (static_obj(lock))
		lock->key = (void *)lock;
		else
		return ERR_PTR(-EINVAL);
		is_static = true;
		}

		/*
		* NOTE: the class-key must be unique. For dynamic locks, a static
		@@ -708,7 +722,7 @@ look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)
		}
		}

		return NULL;
		return is_static \|\| static_obj(lock->key) ? NULL : ERR_PTR(-EINVAL);
		}

		/*
		@@ -726,19 +740,18 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
		DEBUG_LOCKS_WARN_ON(!irqs_disabled());

		class = look_up_lock_class(lock, subclass);
		if (likely(class))
		if (likely(!IS_ERR_OR_NULL(class)))
		goto out_set_class_cache;

		/*
		* Debug-check: all keys must be persistent!
		*/
		if (!static_obj(lock->key)) {
		if (IS_ERR(class)) {
		debug_locks_off();
		printk("INFO: trying to register non-static key.\n");
		printk("the code is fine but needs lockdep annotation.\n");
		printk("turning off the locking correctness validator.\n");
		dump_stack();

		return NULL;
		}

		@@ -3419,7 +3432,7 @@ static int match_held_lock(struct held_lock hlock, struct lockdep_map lock)
		* Clearly if the lock hasn't been acquired _ever_, we're not
		* holding it either, so report failure.
		*/
		if (!class)
		if (IS_ERR_OR_NULL(class))
		return 0;

		/*
		@@ -4225,7 +4238,7 @@ void lockdep_reset_lock(struct lockdep_map *lock)
		* If the class exists we look it up and zap it:
		*/
		class = look_up_lock_class(lock, j);
		if (class)
		if (!IS_ERR_OR_NULL(class))
		zap_class(class);
		}
		/*

kernel/module.c

+24 −12

Original line number	Diff line number	Diff line
		@@ -665,16 +665,7 @@ static void percpu_modcopy(struct module *mod,
		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
		}

		/**
		* is_module_percpu_address - test whether address is from module static percpu
		* @addr: address to test
		*
		* Test whether @addr belongs to module static percpu area.
		*
		* RETURNS:
		* %true if @addr is from module static percpu area
		*/
		bool is_module_percpu_address(unsigned long addr)
		bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
		{
		struct module *mod;
		unsigned int cpu;
		@@ -688,9 +679,11 @@ bool is_module_percpu_address(unsigned long addr)
		continue;
		for_each_possible_cpu(cpu) {
		void *start = per_cpu_ptr(mod->percpu, cpu);
		void va = (void )addr;

		if ((void *)addr >= start &&
		(void *)addr < start + mod->percpu_size) {
		if (va >= start && va < start + mod->percpu_size) {
		if (can_addr)
		*can_addr = (unsigned long) (va - start);
		preempt_enable();
		return true;
		}
		@@ -701,6 +694,20 @@ bool is_module_percpu_address(unsigned long addr)
		return false;
		}

		/**
		* is_module_percpu_address - test whether address is from module static percpu
		* @addr: address to test
		*
		* Test whether @addr belongs to module static percpu area.
		*
		* RETURNS:
		* %true if @addr is from module static percpu area
		*/
		bool is_module_percpu_address(unsigned long addr)
		{
		return __is_module_percpu_address(addr, NULL);
		}

		#else /* ... !CONFIG_SMP */

		static inline void __percpu mod_percpu(struct module mod)
		@@ -732,6 +739,11 @@ bool is_module_percpu_address(unsigned long addr)
		return false;
		}

		bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
		{
		return false;
		}

		#endif /* CONFIG_SMP */

		#define MODINFO_ATTR(field) \

mm/percpu.c

+23 −14

Original line number	Diff line number	Diff line
		@@ -1281,18 +1281,7 @@ void free_percpu(void __percpu *ptr)
		}
		EXPORT_SYMBOL_GPL(free_percpu);

		/**
		* is_kernel_percpu_address - test whether address is from static percpu area
		* @addr: address to test
		*
		* Test whether @addr belongs to in-kernel static percpu area. Module
		* static percpu areas are not considered. For those, use
		* is_module_percpu_address().
		*
		* RETURNS:
		* %true if @addr is from in-kernel static percpu area, %false otherwise.
		*/
		bool is_kernel_percpu_address(unsigned long addr)
		bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr)
		{
		#ifdef CONFIG_SMP
		const size_t static_size = __per_cpu_end - __per_cpu_start;
		@@ -1301,15 +1290,35 @@ bool is_kernel_percpu_address(unsigned long addr)

		for_each_possible_cpu(cpu) {
		void *start = per_cpu_ptr(base, cpu);
		void va = (void )addr;

		if ((void )addr >= start && (void )addr < start + static_size)
		if (va >= start && va < start + static_size) {
		if (can_addr)
		*can_addr = (unsigned long) (va - start);
		return true;
		}
		}
		#endif
		/* on UP, can't distinguish from other static vars, always false */
		return false;
		}

		/**
		* is_kernel_percpu_address - test whether address is from static percpu area
		* @addr: address to test
		*
		* Test whether @addr belongs to in-kernel static percpu area. Module
		* static percpu areas are not considered. For those, use
		* is_module_percpu_address().
		*
		* RETURNS:
		* %true if @addr is from in-kernel static percpu area, %false otherwise.
		*/
		bool is_kernel_percpu_address(unsigned long addr)
		{
		return __is_kernel_percpu_address(addr, NULL);
		}

		/**
		* per_cpu_ptr_to_phys - convert translated percpu address to physical address
		* @addr: the address to be converted to physical address