Merge branch 'slab/next' into for-linus (bfb91fb6) · Commits · jan.koester / Linux

include/linux/slub_def.h

+4 −4

Original line number	Original line	Diff line number	Diff line
	@@ -37,9 +37,7 @@ enum stat_item {

	struct kmem_cache_cpu {		struct kmem_cache_cpu {
	void *freelist; / Pointer to next available object */		void *freelist; / Pointer to next available object */
	#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long tid; /* Globally unique transaction id */		unsigned long tid; /* Globally unique transaction id */
	#endif
	struct page page; / The slab from which we are allocating */		struct page page; / The slab from which we are allocating */
	int node; /* The node of the page (or -1 for debug) */		int node; /* The node of the page (or -1 for debug) */
	#ifdef CONFIG_SLUB_STATS		#ifdef CONFIG_SLUB_STATS
	@@ -179,7 +177,8 @@ static __always_inline int kmalloc_index(size_t size)
	if (size <= 4 * 1024) return 12;		if (size <= 4 * 1024) return 12;
	/*		/*
	* The following is only needed to support architectures with a larger page		* The following is only needed to support architectures with a larger page
	* size than 4k.		* size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page
			* size we would have to go up to 128k.
	*/		*/
	if (size <= 8 * 1024) return 13;		if (size <= 8 * 1024) return 13;
	if (size <= 16 * 1024) return 14;		if (size <= 16 * 1024) return 14;
	@@ -190,7 +189,8 @@ static __always_inline int kmalloc_index(size_t size)
	if (size <= 512 * 1024) return 19;		if (size <= 512 * 1024) return 19;
	if (size <= 1024 * 1024) return 20;		if (size <= 1024 * 1024) return 20;
	if (size <= 2 * 1024 * 1024) return 21;		if (size <= 2 * 1024 * 1024) return 21;
	return -1;		BUG();
			return -1; /* Will never be reached */

	/*		/*
	* What we really wanted to do and cannot do because of compiler issues is:		* What we really wanted to do and cannot do because of compiler issues is:

mm/slub.c

+65 −100

Original line number	Original line	Diff line number	Diff line
	@@ -261,6 +261,18 @@ static inline void get_freepointer(struct kmem_cache s, void *object)
	return (void *)(object + s->offset);		return (void *)(object + s->offset);
	}		}

			static inline void get_freepointer_safe(struct kmem_cache s, void *object)
			{
			void *p;

			#ifdef CONFIG_DEBUG_PAGEALLOC
			probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
			#else
			p = get_freepointer(s, object);
			#endif
			return p;
			}

	static inline void set_freepointer(struct kmem_cache s, void object, void *fp)		static inline void set_freepointer(struct kmem_cache s, void object, void *fp)
	{		{
	(void *)(object + s->offset) = fp;		(void *)(object + s->offset) = fp;
	@@ -271,10 +283,6 @@ static inline void set_freepointer(struct kmem_cache s, void object, void *fp)
	for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\		for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
	__p += (__s)->size)		__p += (__s)->size)

	/* Scan freelist */
	#define for_each_free_object(__p, __s, __free) \
	for (__p = (__free); __p; __p = get_freepointer((__s), __p))

	/* Determine object index from a given position */		/* Determine object index from a given position */
	static inline int slab_index(void p, struct kmem_cache s, void *addr)		static inline int slab_index(void p, struct kmem_cache s, void *addr)
	{		{
	@@ -331,6 +339,21 @@ static inline int oo_objects(struct kmem_cache_order_objects x)
	}		}

	#ifdef CONFIG_SLUB_DEBUG		#ifdef CONFIG_SLUB_DEBUG
			/*
			* Determine a map of object in use on a page.
			*
			* Slab lock or node listlock must be held to guarantee that the page does
			* not vanish from under us.
			*/
			static void get_map(struct kmem_cache s, struct page page, unsigned long *map)
			{
			void *p;
			void *addr = page_address(page);

			for (p = page->freelist; p; p = get_freepointer(s, p))
			set_bit(slab_index(p, s, addr), map);
			}

	/*		/*
	* Debug settings:		* Debug settings:
	*/		*/
	@@ -1487,7 +1510,7 @@ static struct page get_partial(struct kmem_cache s, gfp_t flags, int node)
	int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;		int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;

	page = get_partial_node(get_node(s, searchnode));		page = get_partial_node(get_node(s, searchnode));
	if (page \|\| node != -1)		if (page \|\| node != NUMA_NO_NODE)
	return page;		return page;

	return get_any_partial(s, flags);		return get_any_partial(s, flags);
	@@ -1540,7 +1563,6 @@ static void unfreeze_slab(struct kmem_cache s, struct page page, int tail)
	}		}
	}		}

	#ifdef CONFIG_CMPXCHG_LOCAL
	#ifdef CONFIG_PREEMPT		#ifdef CONFIG_PREEMPT
	/*		/*
	* Calculate the next globally unique transaction for disambiguiation		* Calculate the next globally unique transaction for disambiguiation
	@@ -1600,17 +1622,12 @@ static inline void note_cmpxchg_failure(const char *n,
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);		stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
	}		}

	#endif

	void init_kmem_cache_cpus(struct kmem_cache *s)		void init_kmem_cache_cpus(struct kmem_cache *s)
	{		{
	#ifdef CONFIG_CMPXCHG_LOCAL
	int cpu;		int cpu;

	for_each_possible_cpu(cpu)		for_each_possible_cpu(cpu)
	per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);		per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
	#endif

	}		}
	/*		/*
	* Remove the cpu slab		* Remove the cpu slab
	@@ -1643,9 +1660,7 @@ static void deactivate_slab(struct kmem_cache s, struct kmem_cache_cpu c)
	page->inuse--;		page->inuse--;
	}		}
	c->page = NULL;		c->page = NULL;
	#ifdef CONFIG_CMPXCHG_LOCAL
	c->tid = next_tid(c->tid);		c->tid = next_tid(c->tid);
	#endif
	unfreeze_slab(s, page, tail);		unfreeze_slab(s, page, tail);
	}		}

	@@ -1779,8 +1794,7 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
	unsigned long addr, struct kmem_cache_cpu *c)		unsigned long addr, struct kmem_cache_cpu *c)
	{		{
	void **object;		void **object;
	struct page *new;		struct page *page;
	#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long flags;		unsigned long flags;

	local_irq_save(flags);		local_irq_save(flags);
	@@ -1791,38 +1805,36 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
	* pointer.		* pointer.
	*/		*/
	c = this_cpu_ptr(s->cpu_slab);		c = this_cpu_ptr(s->cpu_slab);
	#endif
	#endif		#endif

	/* We handle __GFP_ZERO in the caller */		/* We handle __GFP_ZERO in the caller */
	gfpflags &= ~__GFP_ZERO;		gfpflags &= ~__GFP_ZERO;

	if (!c->page)		page = c->page;
			if (!page)
	goto new_slab;		goto new_slab;

	slab_lock(c->page);		slab_lock(page);
	if (unlikely(!node_match(c, node)))		if (unlikely(!node_match(c, node)))
	goto another_slab;		goto another_slab;

	stat(s, ALLOC_REFILL);		stat(s, ALLOC_REFILL);

	load_freelist:		load_freelist:
	object = c->page->freelist;		object = page->freelist;
	if (unlikely(!object))		if (unlikely(!object))
	goto another_slab;		goto another_slab;
	if (kmem_cache_debug(s))		if (kmem_cache_debug(s))
	goto debug;		goto debug;

	c->freelist = get_freepointer(s, object);		c->freelist = get_freepointer(s, object);
	c->page->inuse = c->page->objects;		page->inuse = page->objects;
	c->page->freelist = NULL;		page->freelist = NULL;
	c->node = page_to_nid(c->page);
	unlock_out:		unlock_out:
	slab_unlock(c->page);		slab_unlock(page);
	#ifdef CONFIG_CMPXCHG_LOCAL
	c->tid = next_tid(c->tid);		c->tid = next_tid(c->tid);
	local_irq_restore(flags);		local_irq_restore(flags);
	#endif
	stat(s, ALLOC_SLOWPATH);		stat(s, ALLOC_SLOWPATH);
	return object;		return object;

	@@ -1830,10 +1842,11 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
	deactivate_slab(s, c);		deactivate_slab(s, c);

	new_slab:		new_slab:
	new = get_partial(s, gfpflags, node);		page = get_partial(s, gfpflags, node);
	if (new) {		if (page) {
	c->page = new;
	stat(s, ALLOC_FROM_PARTIAL);		stat(s, ALLOC_FROM_PARTIAL);
			c->node = page_to_nid(page);
			c->page = page;
	goto load_freelist;		goto load_freelist;
	}		}

	@@ -1841,33 +1854,35 @@ static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
	if (gfpflags & __GFP_WAIT)		if (gfpflags & __GFP_WAIT)
	local_irq_enable();		local_irq_enable();

	new = new_slab(s, gfpflags, node);		page = new_slab(s, gfpflags, node);

	if (gfpflags & __GFP_WAIT)		if (gfpflags & __GFP_WAIT)
	local_irq_disable();		local_irq_disable();

	if (new) {		if (page) {
	c = __this_cpu_ptr(s->cpu_slab);		c = __this_cpu_ptr(s->cpu_slab);
	stat(s, ALLOC_SLAB);		stat(s, ALLOC_SLAB);
	if (c->page)		if (c->page)
	flush_slab(s, c);		flush_slab(s, c);
	slab_lock(new);
	__SetPageSlubFrozen(new);		slab_lock(page);
	c->page = new;		__SetPageSlubFrozen(page);
			c->node = page_to_nid(page);
			c->page = page;
	goto load_freelist;		goto load_freelist;
	}		}
	if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())		if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
	slab_out_of_memory(s, gfpflags, node);		slab_out_of_memory(s, gfpflags, node);
	#ifdef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);		local_irq_restore(flags);
	#endif
	return NULL;		return NULL;
	debug:		debug:
	if (!alloc_debug_processing(s, c->page, object, addr))		if (!alloc_debug_processing(s, page, object, addr))
	goto another_slab;		goto another_slab;

	c->page->inuse++;		page->inuse++;
	c->page->freelist = get_freepointer(s, object);		page->freelist = get_freepointer(s, object);
			deactivate_slab(s, c);
			c->page = NULL;
	c->node = NUMA_NO_NODE;		c->node = NUMA_NO_NODE;
	goto unlock_out;		goto unlock_out;
	}		}
	@@ -1887,20 +1902,12 @@ static __always_inline void slab_alloc(struct kmem_cache s,
	{		{
	void **object;		void **object;
	struct kmem_cache_cpu *c;		struct kmem_cache_cpu *c;
	#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long tid;		unsigned long tid;
	#else
	unsigned long flags;
	#endif

	if (slab_pre_alloc_hook(s, gfpflags))		if (slab_pre_alloc_hook(s, gfpflags))
	return NULL;		return NULL;

	#ifndef CONFIG_CMPXCHG_LOCAL
	local_irq_save(flags);
	#else
	redo:		redo:
	#endif

	/*		/*
	* Must read kmem_cache cpu data via this cpu ptr. Preemption is		* Must read kmem_cache cpu data via this cpu ptr. Preemption is
	@@ -1910,7 +1917,6 @@ static __always_inline void slab_alloc(struct kmem_cache s,
	*/		*/
	c = __this_cpu_ptr(s->cpu_slab);		c = __this_cpu_ptr(s->cpu_slab);

	#ifdef CONFIG_CMPXCHG_LOCAL
	/*		/*
	* The transaction ids are globally unique per cpu and per operation on		* The transaction ids are globally unique per cpu and per operation on
	* a per cpu queue. Thus they can be guarantee that the cmpxchg_double		* a per cpu queue. Thus they can be guarantee that the cmpxchg_double
	@@ -1919,7 +1925,6 @@ static __always_inline void slab_alloc(struct kmem_cache s,
	*/		*/
	tid = c->tid;		tid = c->tid;
	barrier();		barrier();
	#endif

	object = c->freelist;		object = c->freelist;
	if (unlikely(!object \|\| !node_match(c, node)))		if (unlikely(!object \|\| !node_match(c, node)))
	@@ -1927,7 +1932,6 @@ static __always_inline void slab_alloc(struct kmem_cache s,
	object = __slab_alloc(s, gfpflags, node, addr, c);		object = __slab_alloc(s, gfpflags, node, addr, c);

	else {		else {
	#ifdef CONFIG_CMPXCHG_LOCAL
	/*		/*
	* The cmpxchg will only match if there was no additional		* The cmpxchg will only match if there was no additional
	* operation and if we are on the right processor.		* operation and if we are on the right processor.
	@@ -1943,21 +1947,14 @@ static __always_inline void slab_alloc(struct kmem_cache s,
	if (unlikely(!irqsafe_cpu_cmpxchg_double(		if (unlikely(!irqsafe_cpu_cmpxchg_double(
	s->cpu_slab->freelist, s->cpu_slab->tid,		s->cpu_slab->freelist, s->cpu_slab->tid,
	object, tid,		object, tid,
	get_freepointer(s, object), next_tid(tid)))) {		get_freepointer_safe(s, object), next_tid(tid)))) {

	note_cmpxchg_failure("slab_alloc", s, tid);		note_cmpxchg_failure("slab_alloc", s, tid);
	goto redo;		goto redo;
	}		}
	#else
	c->freelist = get_freepointer(s, object);
	#endif
	stat(s, ALLOC_FASTPATH);		stat(s, ALLOC_FASTPATH);
	}		}

	#ifndef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);
	#endif

	if (unlikely(gfpflags & __GFP_ZERO) && object)		if (unlikely(gfpflags & __GFP_ZERO) && object)
	memset(object, 0, s->objsize);		memset(object, 0, s->objsize);

	@@ -2034,18 +2031,15 @@ static void __slab_free(struct kmem_cache s, struct page page,
	{		{
	void *prior;		void *prior;
	void *object = (void )x;		void *object = (void )x;
	#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long flags;		unsigned long flags;

	local_irq_save(flags);		local_irq_save(flags);
	#endif
	slab_lock(page);		slab_lock(page);
	stat(s, FREE_SLOWPATH);		stat(s, FREE_SLOWPATH);

	if (kmem_cache_debug(s))		if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
	goto debug;		goto out_unlock;

	checks_ok:
	prior = page->freelist;		prior = page->freelist;
	set_freepointer(s, object, prior);		set_freepointer(s, object, prior);
	page->freelist = object;		page->freelist = object;
	@@ -2070,9 +2064,7 @@ static void __slab_free(struct kmem_cache s, struct page page,

	out_unlock:		out_unlock:
	slab_unlock(page);		slab_unlock(page);
	#ifdef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);		local_irq_restore(flags);
	#endif
	return;		return;

	slab_empty:		slab_empty:
	@@ -2084,17 +2076,9 @@ static void __slab_free(struct kmem_cache s, struct page page,
	stat(s, FREE_REMOVE_PARTIAL);		stat(s, FREE_REMOVE_PARTIAL);
	}		}
	slab_unlock(page);		slab_unlock(page);
	#ifdef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);		local_irq_restore(flags);
	#endif
	stat(s, FREE_SLAB);		stat(s, FREE_SLAB);
	discard_slab(s, page);		discard_slab(s, page);
	return;

	debug:
	if (!free_debug_processing(s, page, x, addr))
	goto out_unlock;
	goto checks_ok;
	}		}

	/*		/*
	@@ -2113,20 +2097,11 @@ static __always_inline void slab_free(struct kmem_cache *s,
	{		{
	void *object = (void )x;		void *object = (void )x;
	struct kmem_cache_cpu *c;		struct kmem_cache_cpu *c;
	#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long tid;		unsigned long tid;
	#else
	unsigned long flags;
	#endif

	slab_free_hook(s, x);		slab_free_hook(s, x);

	#ifndef CONFIG_CMPXCHG_LOCAL
	local_irq_save(flags);

	#else
	redo:		redo:
	#endif

	/*		/*
	* Determine the currently cpus per cpu slab.		* Determine the currently cpus per cpu slab.
	@@ -2136,15 +2111,12 @@ static __always_inline void slab_free(struct kmem_cache *s,
	*/		*/
	c = __this_cpu_ptr(s->cpu_slab);		c = __this_cpu_ptr(s->cpu_slab);

	#ifdef CONFIG_CMPXCHG_LOCAL
	tid = c->tid;		tid = c->tid;
	barrier();		barrier();
	#endif

	if (likely(page == c->page && c->node != NUMA_NO_NODE)) {		if (likely(page == c->page)) {
	set_freepointer(s, object, c->freelist);		set_freepointer(s, object, c->freelist);

	#ifdef CONFIG_CMPXCHG_LOCAL
	if (unlikely(!irqsafe_cpu_cmpxchg_double(		if (unlikely(!irqsafe_cpu_cmpxchg_double(
	s->cpu_slab->freelist, s->cpu_slab->tid,		s->cpu_slab->freelist, s->cpu_slab->tid,
	c->freelist, tid,		c->freelist, tid,
	@@ -2153,16 +2125,10 @@ static __always_inline void slab_free(struct kmem_cache *s,
	note_cmpxchg_failure("slab_free", s, tid);		note_cmpxchg_failure("slab_free", s, tid);
	goto redo;		goto redo;
	}		}
	#else
	c->freelist = object;
	#endif
	stat(s, FREE_FASTPATH);		stat(s, FREE_FASTPATH);
	} else		} else
	__slab_free(s, page, x, addr);		__slab_free(s, page, x, addr);

	#ifndef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);
	#endif
	}		}

	void kmem_cache_free(struct kmem_cache s, void x)		void kmem_cache_free(struct kmem_cache s, void x)
	@@ -2673,9 +2639,8 @@ static void list_slab_objects(struct kmem_cache s, struct page page,
	return;		return;
	slab_err(s, page, "%s", text);		slab_err(s, page, "%s", text);
	slab_lock(page);		slab_lock(page);
	for_each_free_object(p, s, page->freelist)
	set_bit(slab_index(p, s, addr), map);

			get_map(s, page, map);
	for_each_object(p, s, addr, page->objects) {		for_each_object(p, s, addr, page->objects) {

	if (!test_bit(slab_index(p, s, addr), map)) {		if (!test_bit(slab_index(p, s, addr), map)) {
	@@ -3203,7 +3168,7 @@ static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
	list_for_each_entry(p, &n->partial, lru)		list_for_each_entry(p, &n->partial, lru)
	p->slab = s;		p->slab = s;

	#ifdef CONFIG_SLAB_DEBUG		#ifdef CONFIG_SLUB_DEBUG
	list_for_each_entry(p, &n->full, lru)		list_for_each_entry(p, &n->full, lru)
	p->slab = s;		p->slab = s;
	#endif		#endif
	@@ -3610,8 +3575,9 @@ static int validate_slab(struct kmem_cache s, struct page page,
	/* Now we know that a valid freelist exists */		/* Now we know that a valid freelist exists */
	bitmap_zero(map, page->objects);		bitmap_zero(map, page->objects);

	for_each_free_object(p, s, page->freelist) {		get_map(s, page, map);
	set_bit(slab_index(p, s, addr), map);		for_each_object(p, s, addr, page->objects) {
			if (test_bit(slab_index(p, s, addr), map))
	if (!check_object(s, page, p, SLUB_RED_INACTIVE))		if (!check_object(s, page, p, SLUB_RED_INACTIVE))
	return 0;		return 0;
	}		}
	@@ -3821,8 +3787,7 @@ static void process_slab(struct loc_track t, struct kmem_cache s,
	void *p;		void *p;

	bitmap_zero(map, page->objects);		bitmap_zero(map, page->objects);
	for_each_free_object(p, s, page->freelist)		get_map(s, page, map);
	set_bit(slab_index(p, s, addr), map);

	for_each_object(p, s, addr, page->objects)		for_each_object(p, s, addr, page->objects)
	if (!test_bit(slab_index(p, s, addr), map))		if (!test_bit(slab_index(p, s, addr), map))