Newer
Older
* mm/percpu.c - percpu memory allocator
*
* Copyright (C) 2009 SUSE Linux Products GmbH
* Copyright (C) 2009 Tejun Heo <tj@kernel.org>
*
* This file is released under the GPLv2.
*
* This is percpu allocator which can handle both static and dynamic
* areas. Percpu areas are allocated in chunks. Each chunk is
* consisted of boot-time determined number of units and the first
* chunk is used for static percpu variables in the kernel image
* (special boot time alloc/init handling necessary as these areas
* need to be brought up before allocation services are running).
* Unit grows as necessary and all units grow or shrink in unison.
* When a chunk is filled up, another chunk is allocated.
*
* c0 c1 c2
* ------------------- ------------------- ------------
* | u0 | u1 | u2 | u3 | | u0 | u1 | u2 | u3 | | u0 | u1 | u
* ------------------- ...... ------------------- .... ------------
*
* Allocation is done in offset-size areas of single unit space. Ie,
* an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0,
* c1:u1, c1:u2 and c1:u3. On UMA, units corresponds directly to
* cpus. On NUMA, the mapping can be non-linear and even sparse.
* Percpu access can be done by configuring percpu base registers
* according to cpu to unit mapping and pcpu_unit_size.
* There are usually many small percpu allocations many of them being
* as small as 4 bytes. The allocator organizes chunks into lists
* according to free size and tries to allocate from the fullest one.
* Each chunk keeps the maximum contiguous area size hint which is
* guaranteed to be equal to or larger than the maximum contiguous
* area in the chunk. This helps the allocator not to iterate the
* chunk maps unnecessarily.
*
* Allocation state in each chunk is kept using an array of integers
* on chunk->map. A positive value in the map represents a free
* region and negative allocated. Allocation inside a chunk is done
* by scanning this map sequentially and serving the first matching
* entry. This is mostly copied from the percpu_modalloc() allocator.
* Chunks can be determined from the address using the index field
* in the page struct. The index field contains a pointer to the chunk.
*
* To use this allocator, arch code should do the followings.
*
* - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
* regular address to percpu pointer and back if they need to be
* different from the default
* - use pcpu_setup_first_chunk() during percpu area initialization to
* setup the first chunk containing the kernel static percpu area
*/
#include <linux/bitmap.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/percpu.h>
#include <linux/pfn.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
#include <asm/cacheflush.h>
#include <asm/sections.h>
#include <asm/io.h>
#define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */
#define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */
/* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
#ifndef __addr_to_pcpu_ptr
#define __addr_to_pcpu_ptr(addr) \
(void __percpu *)((unsigned long)(addr) - \
(unsigned long)pcpu_base_addr + \
(unsigned long)__per_cpu_start)
#endif
#ifndef __pcpu_ptr_to_addr
#define __pcpu_ptr_to_addr(ptr) \
(void __force *)((unsigned long)(ptr) + \
(unsigned long)pcpu_base_addr - \
(unsigned long)__per_cpu_start)
#else /* CONFIG_SMP */
/* on UP, it's always identity mapped */
#define __addr_to_pcpu_ptr(addr) (void __percpu *)(addr)
#define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr)
#endif /* CONFIG_SMP */
struct pcpu_chunk {
struct list_head list; /* linked to pcpu_slot lists */
int free_size; /* free bytes in the chunk */
int contig_hint; /* max contiguous size hint */
void *base_addr; /* base address of this chunk */
int map_used; /* # of map entries used before the sentry */
int map_alloc; /* # of map entries allocated */
int *map; /* allocation map */
bool immutable; /* no [de]population allowed */
unsigned long populated[]; /* populated bitmap */
static int pcpu_unit_pages __read_mostly;
static int pcpu_unit_size __read_mostly;
static int pcpu_nr_units __read_mostly;
static int pcpu_atom_size __read_mostly;
static int pcpu_nr_slots __read_mostly;
static size_t pcpu_chunk_struct_size __read_mostly;
/* cpus with the lowest and highest unit addresses */
static unsigned int pcpu_low_unit_cpu __read_mostly;
static unsigned int pcpu_high_unit_cpu __read_mostly;
/* the address of the first chunk which starts with the kernel static area */
void *pcpu_base_addr __read_mostly;
EXPORT_SYMBOL_GPL(pcpu_base_addr);
static const int *pcpu_unit_map __read_mostly; /* cpu -> unit */
const unsigned long *pcpu_unit_offsets __read_mostly; /* cpu -> unit offset */
/* group information, used for vm allocation */
static int pcpu_nr_groups __read_mostly;
static const unsigned long *pcpu_group_offsets __read_mostly;
static const size_t *pcpu_group_sizes __read_mostly;
/*
* The first chunk which always exists. Note that unlike other
* chunks, this one can be allocated and mapped in several different
* ways and thus often doesn't live in the vmalloc area.
*/
static struct pcpu_chunk *pcpu_first_chunk;
/*
* Optional reserved chunk. This chunk reserves part of the first
* chunk and serves it for reserved allocations. The amount of
* reserved offset is in pcpu_reserved_chunk_limit. When reserved
* area doesn't exist, the following variables contain NULL and 0
* respectively.
*/
Tejun Heo
committed
static struct pcpu_chunk *pcpu_reserved_chunk;
static int pcpu_reserved_chunk_limit;
* Free path accesses and alters only the index data structures and can be
* safely called from atomic context. When memory needs to be returned to
* the system, free path schedules reclaim_work.
static DEFINE_SPINLOCK(pcpu_lock); /* all internal data structures */
static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop */
static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
/* reclaim work to release fully free chunks, scheduled from free path */
static void pcpu_reclaim(struct work_struct *work);
static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
Tejun Heo
committed
static bool pcpu_addr_in_first_chunk(void *addr)
{
void *first_start = pcpu_first_chunk->base_addr;
return addr >= first_start && addr < first_start + pcpu_unit_size;
}
static bool pcpu_addr_in_reserved_chunk(void *addr)
{
void *first_start = pcpu_first_chunk->base_addr;
return addr >= first_start &&
addr < first_start + pcpu_reserved_chunk_limit;
}
static int __pcpu_size_to_slot(int size)
return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1);
}
static int pcpu_size_to_slot(int size)
{
if (size == pcpu_unit_size)
return pcpu_nr_slots - 1;
return __pcpu_size_to_slot(size);
}
static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
{
if (chunk->free_size < sizeof(int) || chunk->contig_hint < sizeof(int))
return 0;
return pcpu_size_to_slot(chunk->free_size);
}
/* set the pointer to a chunk in a page struct */
static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
{
page->index = (unsigned long)pcpu;
}
/* obtain pointer to a chunk from a page struct */
static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
{
return (struct pcpu_chunk *)page->index;
}
static int __maybe_unused pcpu_page_idx(unsigned int cpu, int page_idx)
return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
unsigned int cpu, int page_idx)
return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
static void __maybe_unused pcpu_next_unpop(struct pcpu_chunk *chunk,
int *rs, int *re, int end)
{
*rs = find_next_zero_bit(chunk->populated, end, *rs);
*re = find_next_bit(chunk->populated, end, *rs + 1);
}
static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk,
int *rs, int *re, int end)
{
*rs = find_next_bit(chunk->populated, end, *rs);
*re = find_next_zero_bit(chunk->populated, end, *rs + 1);
}
/*
* (Un)populated page region iterators. Iterate over (un)populated
* page regions between @start and @end in @chunk. @rs and @re should
* be integer variables and will be set to start and end page index of
* the current region.
*/
#define pcpu_for_each_unpop_region(chunk, rs, re, start, end) \
for ((rs) = (start), pcpu_next_unpop((chunk), &(rs), &(re), (end)); \
(rs) < (re); \
(rs) = (re) + 1, pcpu_next_unpop((chunk), &(rs), &(re), (end)))
#define pcpu_for_each_pop_region(chunk, rs, re, start, end) \
for ((rs) = (start), pcpu_next_pop((chunk), &(rs), &(re), (end)); \
(rs) < (re); \
(rs) = (re) + 1, pcpu_next_pop((chunk), &(rs), &(re), (end)))
* pcpu_mem_zalloc - allocate memory
* @size: bytes to allocate
* Allocate @size bytes. If @size is smaller than PAGE_SIZE,
* kzalloc() is used; otherwise, vzalloc() is used. The returned
* memory is always zeroed.
* CONTEXT:
* Does GFP_KERNEL allocation.
*
* Pointer to the allocated area on success, NULL on failure.
static void *pcpu_mem_zalloc(size_t size)
if (WARN_ON_ONCE(!slab_is_available()))
return NULL;
if (size <= PAGE_SIZE)
return kzalloc(size, GFP_KERNEL);
else
return vzalloc(size);
/**
* pcpu_mem_free - free memory
* @ptr: memory to free
* @size: size of the area
*
* Free @ptr. @ptr should have been allocated using pcpu_mem_zalloc().
*/
static void pcpu_mem_free(void *ptr, size_t size)
{
kfree(ptr);
vfree(ptr);
}
/**
* pcpu_chunk_relocate - put chunk in the appropriate chunk slot
* @chunk: chunk of interest
* @oslot: the previous slot it was on
*
* This function is called after an allocation or free changed @chunk.
* New slot according to the changed state is determined and @chunk is
Tejun Heo
committed
* moved to the slot. Note that the reserved chunk is never put on
* chunk slots.
*
* CONTEXT:
* pcpu_lock.
*/
static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
{
int nslot = pcpu_chunk_slot(chunk);
Tejun Heo
committed
if (chunk != pcpu_reserved_chunk && oslot != nslot) {
if (oslot < nslot)
list_move(&chunk->list, &pcpu_slot[nslot]);
else
list_move_tail(&chunk->list, &pcpu_slot[nslot]);
}
}
* pcpu_need_to_extend - determine whether chunk area map needs to be extended
* @chunk: chunk of interest
* Determine whether area map of @chunk needs to be extended to
* CONTEXT:
* pcpu_lock.
* New target map allocation length if extension is necessary, 0
* otherwise.
static int pcpu_need_to_extend(struct pcpu_chunk *chunk)
{
int new_alloc;
if (chunk->map_alloc >= chunk->map_used + 3)
return 0;
new_alloc = PCPU_DFL_MAP_ALLOC;
while (new_alloc < chunk->map_used + 3)
new_alloc *= 2;
return new_alloc;
}
/**
* pcpu_extend_area_map - extend area map of a chunk
* @chunk: chunk of interest
* @new_alloc: new target allocation length of the area map
*
* Extend area map of @chunk to have @new_alloc entries.
*
* CONTEXT:
* Does GFP_KERNEL allocation. Grabs and releases pcpu_lock.
*
* RETURNS:
* 0 on success, -errno on failure.
*/
static int pcpu_extend_area_map(struct pcpu_chunk *chunk, int new_alloc)
{
int *old = NULL, *new = NULL;
size_t old_size = 0, new_size = new_alloc * sizeof(new[0]);
unsigned long flags;
new = pcpu_mem_zalloc(new_size);
if (!new)
return -ENOMEM;
/* acquire pcpu_lock and switch to new area map */
spin_lock_irqsave(&pcpu_lock, flags);
if (new_alloc <= chunk->map_alloc)
goto out_unlock;
old_size = chunk->map_alloc * sizeof(chunk->map[0]);
old = chunk->map;
memcpy(new, old, old_size);
chunk->map_alloc = new_alloc;
chunk->map = new;
new = NULL;
out_unlock:
spin_unlock_irqrestore(&pcpu_lock, flags);
/*
* pcpu_mem_free() might end up calling vfree() which uses
* IRQ-unsafe lock and thus can't be called under pcpu_lock.
*/
pcpu_mem_free(old, old_size);
pcpu_mem_free(new, new_size);
return 0;
}
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
/**
* pcpu_fit_in_area - try to fit the requested allocation in a candidate area
* @chunk: chunk the candidate area belongs to
* @off: the offset to the start of the candidate area
* @this_size: the size of the candidate area
* @size: the size of the target allocation
* @align: the alignment of the target allocation
* @pop_only: only allocate from already populated region
*
* We're trying to allocate @size bytes aligned at @align. @chunk's area
* at @off sized @this_size is a candidate. This function determines
* whether the target allocation fits in the candidate area and returns the
* number of bytes to pad after @off. If the target area doesn't fit, -1
* is returned.
*
* If @pop_only is %true, this function only considers the already
* populated part of the candidate area.
*/
static int pcpu_fit_in_area(struct pcpu_chunk *chunk, int off, int this_size,
int size, int align, bool pop_only)
{
int cand_off = off;
while (true) {
int head = ALIGN(cand_off, align) - off;
int page_start, page_end, rs, re;
if (this_size < head + size)
return -1;
if (!pop_only)
return head;
/*
* If the first unpopulated page is beyond the end of the
* allocation, the whole allocation is populated;
* otherwise, retry from the end of the unpopulated area.
*/
page_start = PFN_DOWN(head + off);
page_end = PFN_UP(head + off + size);
rs = page_start;
pcpu_next_unpop(chunk, &rs, &re, PFN_UP(off + this_size));
if (rs >= page_end)
return head;
cand_off = re * PAGE_SIZE;
}
}
/**
* pcpu_alloc_area - allocate area from a pcpu_chunk
* @chunk: chunk of interest
* @pop_only: allocate only from the populated area
*
* Try to allocate @size bytes area aligned at @align from @chunk.
* Note that this function only allocates the offset. It doesn't
* populate or map the area.
*
* @chunk->map must have at least two free slots.
*
* CONTEXT:
* pcpu_lock.
*
* Allocated offset in @chunk on success, -1 if no matching area is
* found.
static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align,
bool pop_only)
{
int oslot = pcpu_chunk_slot(chunk);
int max_contig = 0;
int i, off;
for (i = chunk->first_free, p = chunk->map + i; i < chunk->map_used; i++, p++) {
int this_size;
off = *p;
if (off & 1)
continue;
this_size = (p[1] & ~1) - off;
head = pcpu_fit_in_area(chunk, off, this_size, size, align,
pop_only);
if (head < 0) {
if (!seen_free) {
chunk->first_free = i;
seen_free = true;
}
max_contig = max(this_size, max_contig);
continue;
}
/*
* If head is small or the previous block is free,
* merge'em. Note that 'small' is defined as smaller
* than sizeof(int), which is very small but isn't too
* uncommon for percpu allocations.
*/
if (head && (head < sizeof(int) || !(p[-1] & 1))) {
*p = off += head;
else
max_contig = max(*p - p[-1], max_contig);
head = 0;
}
/* if tail is small, just keep it around */
tail = this_size - head - size;
if (tail < sizeof(int)) {
size = this_size - head;
}
/* split if warranted */
if (head || tail) {
int nr_extra = !!head + !!tail;
/* insert new subblocks */
memmove(p + nr_extra + 1, p + 1,
sizeof(chunk->map[0]) * (chunk->map_used - i));
chunk->map_used += nr_extra;
if (!seen_free) {
chunk->first_free = i;
seen_free = true;
}
*++p = off += head;
++i;
max_contig = max(head, max_contig);
}
if (tail) {
max_contig = max(tail, max_contig);
if (!seen_free)
chunk->first_free = i + 1;
/* update hint and mark allocated */
if (i + 1 == chunk->map_used)
chunk->contig_hint = max_contig; /* fully scanned */
else
chunk->contig_hint = max(chunk->contig_hint,
max_contig);
chunk->free_size -= size;
*p |= 1;
pcpu_chunk_relocate(chunk, oslot);
return off;
}
chunk->contig_hint = max_contig; /* fully scanned */
pcpu_chunk_relocate(chunk, oslot);
/* tell the upper layer that this chunk has no matching area */
return -1;
}
/**
* pcpu_free_area - free area to a pcpu_chunk
* @chunk: chunk of interest
* @freeme: offset of area to free
*
* Free area starting from @freeme to @chunk. Note that this function
* only modifies the allocation map. It doesn't depopulate or unmap
* the area.
*
* CONTEXT:
* pcpu_lock.
*/
static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
{
int oslot = pcpu_chunk_slot(chunk);
int off = 0;
unsigned i, j;
int to_free = 0;
int *p;
freeme |= 1; /* we are searching for <given offset, in use> pair */
i = 0;
j = chunk->map_used;
while (i != j) {
unsigned k = (i + j) / 2;
off = chunk->map[k];
if (off < freeme)
i = k + 1;
else if (off > freeme)
j = k;
else
i = j = k;
}
if (i < chunk->first_free)
chunk->first_free = i;
p = chunk->map + i;
*p = off &= ~1;
chunk->free_size += (p[1] & ~1) - off;
/* merge with next? */
if (!(p[1] & 1))
to_free++;
if (i > 0 && !(p[-1] & 1)) {
to_free++;
if (to_free) {
chunk->map_used -= to_free;
memmove(p + 1, p + 1 + to_free,
(chunk->map_used - i) * sizeof(chunk->map[0]));
chunk->contig_hint = max(chunk->map[i + 1] - chunk->map[i] - 1, chunk->contig_hint);
pcpu_chunk_relocate(chunk, oslot);
}
static struct pcpu_chunk *pcpu_alloc_chunk(void)
{
struct pcpu_chunk *chunk;
chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size);
if (!chunk)
return NULL;
chunk->map = pcpu_mem_zalloc(PCPU_DFL_MAP_ALLOC *
sizeof(chunk->map[0]));
pcpu_mem_free(chunk, pcpu_chunk_struct_size);
return NULL;
}
chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
chunk->map[0] = 0;
chunk->map[1] = pcpu_unit_size | 1;
chunk->map_used = 1;
INIT_LIST_HEAD(&chunk->list);
chunk->free_size = pcpu_unit_size;
chunk->contig_hint = pcpu_unit_size;
return chunk;
}
static void pcpu_free_chunk(struct pcpu_chunk *chunk)
{
if (!chunk)
return;
pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
pcpu_mem_free(chunk, pcpu_chunk_struct_size);
/*
* Chunk management implementation.
*
* To allow different implementations, chunk alloc/free and
* [de]population are implemented in a separate file which is pulled
* into this file and compiled together. The following functions
* should be implemented.
*
* pcpu_populate_chunk - populate the specified range of a chunk
* pcpu_depopulate_chunk - depopulate the specified range of a chunk
* pcpu_create_chunk - create a new chunk
* pcpu_destroy_chunk - destroy a chunk, always preceded by full depop
* pcpu_addr_to_page - translate address to physical address
* pcpu_verify_alloc_info - check alloc_info is acceptable during init
static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size);
static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size);
static struct pcpu_chunk *pcpu_create_chunk(void);
static void pcpu_destroy_chunk(struct pcpu_chunk *chunk);
static struct page *pcpu_addr_to_page(void *addr);
static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
#ifdef CONFIG_NEED_PER_CPU_KM
#include "percpu-km.c"
#else
#include "percpu-vm.c"
/**
* pcpu_chunk_addr_search - determine chunk containing specified address
* @addr: address for which the chunk needs to be determined.
*
* RETURNS:
* The address of the found chunk.
*/
static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
{
/* is it in the first chunk? */
if (pcpu_addr_in_first_chunk(addr)) {
/* is it in the reserved area? */
if (pcpu_addr_in_reserved_chunk(addr))
return pcpu_reserved_chunk;
return pcpu_first_chunk;
}
/*
* The address is relative to unit0 which might be unused and
* thus unmapped. Offset the address to the unit space of the
* current processor before looking it up in the vmalloc
* space. Note that any possible cpu id can be used here, so
* there's no need to worry about preemption or cpu hotplug.
*/
addr += pcpu_unit_offsets[raw_smp_processor_id()];
return pcpu_get_page_chunk(pcpu_addr_to_page(addr));
Tejun Heo
committed
* pcpu_alloc - the percpu allocator
* @align: alignment of area (max PAGE_SIZE)
Tejun Heo
committed
* @reserved: allocate from the reserved chunk if available
* Allocate percpu area of @size bytes aligned at @align.
*
* CONTEXT:
* Does GFP_KERNEL allocation.
*
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
int page_start, page_end, rs, re;
unsigned long flags;
/*
* We want the lowest bit of offset available for in-use/free
* indicator, so force >= 16bit alignment and make size even.
*/
if (unlikely(align < 2))
align = 2;
size = ALIGN(size, 2);
if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
WARN(true, "illegal size (%zu) or align (%zu) for "
"percpu allocation\n", size, align);
return NULL;
}
spin_lock_irqsave(&pcpu_lock, flags);
Tejun Heo
committed
/* serve reserved allocations from the reserved chunk if available */
if (reserved && pcpu_reserved_chunk) {
chunk = pcpu_reserved_chunk;
if (size > chunk->contig_hint) {
err = "alloc from reserved chunk failed";
goto fail_unlock;
while ((new_alloc = pcpu_need_to_extend(chunk))) {
spin_unlock_irqrestore(&pcpu_lock, flags);
if (pcpu_extend_area_map(chunk, new_alloc) < 0) {
err = "failed to extend area map of reserved chunk";
}
spin_lock_irqsave(&pcpu_lock, flags);
}
off = pcpu_alloc_area(chunk, size, align, false);
Tejun Heo
committed
if (off >= 0)
goto area_found;
err = "alloc from reserved chunk failed";
goto fail_unlock;
Tejun Heo
committed
}
restart:
Tejun Heo
committed
/* search through normal chunks */
for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) {
list_for_each_entry(chunk, &pcpu_slot[slot], list) {
if (size > chunk->contig_hint)
continue;
new_alloc = pcpu_need_to_extend(chunk);
if (new_alloc) {
spin_unlock_irqrestore(&pcpu_lock, flags);
if (pcpu_extend_area_map(chunk,
new_alloc) < 0) {
err = "failed to extend area map";
}
spin_lock_irqsave(&pcpu_lock, flags);
/*
* pcpu_lock has been dropped, need to
* restart cpu_slot list walking.
*/
goto restart;
off = pcpu_alloc_area(chunk, size, align, false);
if (off >= 0)
goto area_found;
}
}
spin_unlock_irqrestore(&pcpu_lock, flags);
/*
* No space left. Create a new chunk. We don't want multiple
* tasks to create chunks simultaneously. Serialize and create iff
* there's still no empty chunk after grabbing the mutex.
*/
mutex_lock(&pcpu_alloc_mutex);
if (list_empty(&pcpu_slot[pcpu_nr_slots - 1])) {
chunk = pcpu_create_chunk();
if (!chunk) {
err = "failed to allocate new chunk";
goto fail;
}
spin_lock_irqsave(&pcpu_lock, flags);
pcpu_chunk_relocate(chunk, -1);
} else {
spin_lock_irqsave(&pcpu_lock, flags);
goto restart;
spin_unlock_irqrestore(&pcpu_lock, flags);
/* populate if not all pages are already there */
page_start = PFN_DOWN(off);
page_end = PFN_UP(off + size);
pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
WARN_ON(chunk->immutable);
ret = pcpu_populate_chunk(chunk, rs, re);
spin_lock_irqsave(&pcpu_lock, flags);
if (ret) {
mutex_unlock(&pcpu_alloc_mutex);
pcpu_free_area(chunk, off);
err = "failed to populate";
goto fail_unlock;
}
bitmap_set(chunk->populated, rs, re - rs);
mutex_unlock(&pcpu_alloc_mutex);
/* clear the areas and return address relative to base address */
for_each_possible_cpu(cpu)
memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
ptr = __addr_to_pcpu_ptr(chunk->base_addr + off);
kmemleak_alloc_percpu(ptr, size);
return ptr;
fail_unlock:
spin_unlock_irqrestore(&pcpu_lock, flags);
if (warn_limit) {
pr_warning("PERCPU: allocation failed, size=%zu align=%zu, "
"%s\n", size, align, err);
dump_stack();
if (!--warn_limit)
pr_info("PERCPU: limit reached, disable warning\n");
}
return NULL;
Tejun Heo
committed
/**
* __alloc_percpu - allocate dynamic percpu area
* @size: size of area to allocate in bytes
* @align: alignment of area (max PAGE_SIZE)
*
Tejun Heo
committed
* Allocate zero-filled percpu area of @size bytes aligned at @align.
* Might sleep. Might trigger writeouts.
Tejun Heo
committed
*
* CONTEXT:
* Does GFP_KERNEL allocation.
*
Tejun Heo
committed
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
void __percpu *__alloc_percpu(size_t size, size_t align)
Tejun Heo
committed
{
return pcpu_alloc(size, align, false);
}
EXPORT_SYMBOL_GPL(__alloc_percpu);
Tejun Heo
committed
/**
* __alloc_reserved_percpu - allocate reserved percpu area
* @size: size of area to allocate in bytes
* @align: alignment of area (max PAGE_SIZE)
*
Tejun Heo
committed
* Allocate zero-filled percpu area of @size bytes aligned at @align
* from reserved percpu area if arch has set it up; otherwise,
* allocation is served from the same dynamic area. Might sleep.
* Might trigger writeouts.
Tejun Heo
committed
*
* CONTEXT:
* Does GFP_KERNEL allocation.
*
Tejun Heo
committed
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
Tejun Heo
committed
{
return pcpu_alloc(size, align, true);
}
/**
* pcpu_reclaim - reclaim fully free chunks, workqueue function
* @work: unused
*
* Reclaim all fully free chunks except for the first one.
*
* CONTEXT:
* workqueue context.
*/
static void pcpu_reclaim(struct work_struct *work)
LIST_HEAD(todo);
struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1];
struct pcpu_chunk *chunk, *next;
mutex_lock(&pcpu_alloc_mutex);
spin_lock_irq(&pcpu_lock);
list_for_each_entry_safe(chunk, next, head, list) {
WARN_ON(chunk->immutable);
/* spare the first one */
if (chunk == list_first_entry(head, struct pcpu_chunk, list))
continue;
list_move(&chunk->list, &todo);
}
spin_unlock_irq(&pcpu_lock);
list_for_each_entry_safe(chunk, next, &todo, list) {
pcpu_for_each_pop_region(chunk, rs, re, 0, pcpu_unit_pages) {
pcpu_depopulate_chunk(chunk, rs, re);
bitmap_clear(chunk->populated, rs, re - rs);
}
pcpu_destroy_chunk(chunk);
}
/**
* free_percpu - free percpu area
* @ptr: pointer to area to free
*
* Free percpu area @ptr.
*
* CONTEXT:
* Can be called from atomic context.
void free_percpu(void __percpu *ptr)
unsigned long flags;
int off;
if (!ptr)
return;
kmemleak_free_percpu(ptr);
addr = __pcpu_ptr_to_addr(ptr);
spin_lock_irqsave(&pcpu_lock, flags);