Commit d31e23af authored by Ard Biesheuvel's avatar Ard Biesheuvel
Browse files

ARM: mm: make vmalloc_seq handling SMP safe 



Rework the vmalloc_seq handling so it can be used safely under SMP, as
we started using it to ensure that vmap'ed stacks are guaranteed to be
mapped by the active mm before switching to a task, and here we need to
ensure that changes to the page tables are visible to other CPUs when
they observe a change in the sequence count.

Since LPAE needs none of this, fold a check against it into the
vmalloc_seq counter check after breaking it out into a separate static
inline helper.

Given that vmap'ed stacks are now also supported on !SMP configurations,
let's drop the WARN() that could potentially now fire spuriously.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
parent aa0a20f5

arch/arm/include/asm/mmu.h

+1 −1
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ typedef struct { 
#else

	int		switch_pending;

#endif

	unsigned int	vmalloc_seq;

	atomic_t	vmalloc_seq;

	unsigned long	sigpage;

#ifdef CONFIG_VDSO

	unsigned long	vdso;

arch/arm/include/asm/mmu_context.h

+20 −2
Original line number Diff line number Diff line
@@ -23,6 +23,16 @@ 


void __check_vmalloc_seq(struct mm_struct *mm);



#ifdef CONFIG_MMU

static inline void check_vmalloc_seq(struct mm_struct *mm)

{

	if (!IS_ENABLED(CONFIG_ARM_LPAE) &&

	    unlikely(atomic_read(&mm->context.vmalloc_seq) !=

		     atomic_read(&init_mm.context.vmalloc_seq)))

		__check_vmalloc_seq(mm);

}

#endif



#ifdef CONFIG_CPU_HAS_ASID



void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk);
@@ -52,8 +62,7 @@ static inline void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm, 
static inline void check_and_switch_context(struct mm_struct *mm,

					    struct task_struct *tsk)

{

	if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))

		__check_vmalloc_seq(mm);

	check_vmalloc_seq(mm);



	if (irqs_disabled())

		/*
@@ -129,6 +138,15 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, 
#endif

}



#ifdef CONFIG_VMAP_STACK

static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)

{

	if (mm != &init_mm)

		check_vmalloc_seq(mm);

}

#define enter_lazy_tlb enter_lazy_tlb

#endif



#include <asm-generic/mmu_context.h>



#endif

arch/arm/include/asm/page.h

+1 −2
Original line number Diff line number Diff line
@@ -147,11 +147,10 @@ extern void copy_page(void *to, const void *from); 
#include <asm/pgtable-3level-types.h>

#else

#include <asm/pgtable-2level-types.h>

#endif



#ifdef CONFIG_VMAP_STACK

#define ARCH_PAGE_TABLE_SYNC_MASK	PGTBL_PMD_MODIFIED

#endif

#endif



#endif /* CONFIG_MMU */

arch/arm/kernel/traps.c

+7 −18
Original line number Diff line number Diff line
@@ -885,6 +885,7 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) 
	die("kernel stack overflow", regs, 0);

}



#ifndef CONFIG_ARM_LPAE

/*

 * Normally, we rely on the logic in do_translation_fault() to update stale PMD

 * entries covering the vmalloc space in a task's page tables when it first
@@ -895,26 +896,14 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) 
 * So we need to ensure that these PMD entries are up to date *before* the MM

 * switch. As we already have some logic in the MM switch path that takes care

 * of this, let's trigger it by bumping the counter every time the core vmalloc

 * code modifies a PMD entry in the vmalloc region.

 * code modifies a PMD entry in the vmalloc region. Use release semantics on

 * the store so that other CPUs observing the counter's new value are

 * guaranteed to see the updated page table entries as well.

 */

void arch_sync_kernel_mappings(unsigned long start, unsigned long end)

{

	if (start > VMALLOC_END || end < VMALLOC_START)

		return;



	/*

	 * This hooks into the core vmalloc code to receive notifications of

	 * any PMD level changes that have been made to the kernel page tables.

	 * This means it should only be triggered once for every MiB worth of

	 * vmalloc space, given that we don't support huge vmalloc/vmap on ARM,

	 * and that kernel PMD level table entries are rarely (if ever)

	 * updated.

	 *

	 * This means that the counter is going to max out at ~250 for the

	 * typical case. If it overflows, something entirely unexpected has

	 * occurred so let's throw a warning if that happens.

	 */

	WARN_ON(++init_mm.context.vmalloc_seq == UINT_MAX);

	if (start < VMALLOC_END && end > VMALLOC_START)

		atomic_inc_return_release(&init_mm.context.vmalloc_seq);

}



#endif

#endif

arch/arm/mm/context.c

+1 −2
Original line number Diff line number Diff line
@@ -240,8 +240,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) 
	unsigned int cpu = smp_processor_id();

	u64 asid;



	if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))

		__check_vmalloc_seq(mm);

	check_vmalloc_seq(mm);



	/*

	 * We cannot update the pgd and the ASID atomicly with classic