Commit c7eff738 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon
Browse files

arm64: kpti-ng: simplify page table traversal logic 



Simplify the KPTI G-to-nG asm helper code by:
- pulling the 'table bit' test into the get/put macros so we can combine
  them and incorporate the entire loop;
- moving the 'table bit' test after the update of bit #11 so we no
  longer need separate next_xxx and skip_xxx labels;
- redefining the pmd/pud register aliases and the next_pmd/next_pud
  labels instead of branching to them if the number of configured page
  table levels is less than 3 or 4, respectively.

No functional change intended, except for the fact that we now descend
into a next level table after setting bit #11 on its descriptor but this
should make no difference in practice.

While at it, switch to .L prefixed local labels so they don't clutter up
the symbol tables, kallsyms, etc, and clean up the indentation for
legibility.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Reviewed-by: default avatarMark Rutland <mark.rutland@arm.com>
Tested-by: default avatarMark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20220609174320.4035379-2-ardb@kernel.org


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent a111daf0

arch/arm64/mm/proc.S

+36 −64
Original line number Original line Diff line number Diff line
@@ -202,19 +202,25 @@ SYM_FUNC_END(idmap_cpu_replace_ttbr1) 
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

	.pushsection ".idmap.text", "awx"

	.pushsection ".idmap.text", "awx"





	.macro	__idmap_kpti_get_pgtable_ent, type

	.macro	kpti_mk_tbl_ng, type, num_entries

	dc	cvac, cur_\()\type\()p		// Ensure any existing dirty

	add	end_\type\()p, cur_\type\()p, #\num_entries * 8

.Ldo_\type:

	dc	cvac, cur_\type\()p		// Ensure any existing dirty

	dmb	sy				// lines are written back before

	dmb	sy				// lines are written back before

	ldr	\type, [cur_\()\type\()p]	// loading the entry

	ldr	\type, [cur_\type\()p]		// loading the entry

	tbz	\type, #0, skip_\()\type	// Skip invalid and

	tbz	\type, #0, .Lnext_\type		// Skip invalid and

	tbnz	\type, #11, skip_\()\type	// non-global entries

	tbnz	\type, #11, .Lnext_\type	// non-global entries

	.endm



	.macro __idmap_kpti_put_pgtable_ent_ng, type

	orr	\type, \type, #PTE_NG		// Same bit for blocks and pages

	orr	\type, \type, #PTE_NG		// Same bit for blocks and pages

	str	\type, [cur_\()\type\()p]	// Update the entry and ensure

	str	\type, [cur_\type\()p]		// Update the entry and ensure

	dmb	sy				// that it is visible to all

	dmb	sy				// that it is visible to all

	dc	civac, cur_\()\type\()p		// CPUs.

	dc	civac, cur_\()\type\()p		// CPUs.

	.ifnc	\type, pte

	tbnz	\type, #1, .Lderef_\type

	.endif

.Lnext_\type:

	add	cur_\type\()p, cur_\type\()p, #8

	cmp	cur_\type\()p, end_\type\()p

	b.ne	.Ldo_\type

	.endm

	.endm





/*

/*
@@ -235,10 +241,8 @@ SYM_FUNC_START(idmap_kpti_install_ng_mappings) 
	pgd		.req	x7

	pgd		.req	x7

	cur_pudp	.req	x8

	cur_pudp	.req	x8

	end_pudp	.req	x9

	end_pudp	.req	x9

	pud		.req	x10

	cur_pmdp	.req	x11

	cur_pmdp	.req	x11

	end_pmdp	.req	x12

	end_pmdp	.req	x12

	pmd		.req	x13

	cur_ptep	.req	x14

	cur_ptep	.req	x14

	end_ptep	.req	x15

	end_ptep	.req	x15

	pte		.req	x16

	pte		.req	x16
@@ -266,15 +270,7 @@ SYM_FUNC_START(idmap_kpti_install_ng_mappings) 
	/* Everybody is enjoying the idmap, so we can rewrite swapper. */

	/* Everybody is enjoying the idmap, so we can rewrite swapper. */

	/* PGD */

	/* PGD */

	mov		cur_pgdp, swapper_pa

	mov		cur_pgdp, swapper_pa

	add	end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)

	kpti_mk_tbl_ng	pgd, PTRS_PER_PGD

do_pgd:	__idmap_kpti_get_pgtable_ent	pgd

	tbnz	pgd, #1, walk_puds

next_pgd:

	__idmap_kpti_put_pgtable_ent_ng	pgd

skip_pgd:

	add	cur_pgdp, cur_pgdp, #8

	cmp	cur_pgdp, end_pgdp

	b.ne	do_pgd





	/* Publish the updated tables and nuke all the TLBs */

	/* Publish the updated tables and nuke all the TLBs */

	dsb	sy

	dsb	sy
@@ -291,59 +287,35 @@ skip_pgd: 
	str	wzr, [flag_ptr]

	str	wzr, [flag_ptr]

	ret

	ret





.Lderef_pgd:

	/* PUD */

	/* PUD */

walk_puds:

	.if		CONFIG_PGTABLE_LEVELS > 3

	.if		CONFIG_PGTABLE_LEVELS > 3

	pud		.req	x10

	pte_to_phys	cur_pudp, pgd

	pte_to_phys	cur_pudp, pgd

	add	end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)

	kpti_mk_tbl_ng	pud, PTRS_PER_PUD

do_pud:	__idmap_kpti_get_pgtable_ent	pud

	b		.Lnext_pgd

	tbnz	pud, #1, walk_pmds

next_pud:

	__idmap_kpti_put_pgtable_ent_ng	pud

skip_pud:

	add	cur_pudp, cur_pudp, 8

	cmp	cur_pudp, end_pudp

	b.ne	do_pud

	b	next_pgd

	.else		/* CONFIG_PGTABLE_LEVELS <= 3 */

	.else		/* CONFIG_PGTABLE_LEVELS <= 3 */

	mov	pud, pgd

	pud		.req	pgd

	b	walk_pmds

	.set		.Lnext_pud, .Lnext_pgd

next_pud:

	b	next_pgd

	.endif

	.endif





.Lderef_pud:

	/* PMD */

	/* PMD */

walk_pmds:

	.if		CONFIG_PGTABLE_LEVELS > 2

	.if		CONFIG_PGTABLE_LEVELS > 2

	pmd		.req	x13

	pte_to_phys	cur_pmdp, pud

	pte_to_phys	cur_pmdp, pud

	add	end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)

	kpti_mk_tbl_ng	pmd, PTRS_PER_PMD

do_pmd:	__idmap_kpti_get_pgtable_ent	pmd

	b		.Lnext_pud

	tbnz	pmd, #1, walk_ptes

next_pmd:

	__idmap_kpti_put_pgtable_ent_ng	pmd

skip_pmd:

	add	cur_pmdp, cur_pmdp, #8

	cmp	cur_pmdp, end_pmdp

	b.ne	do_pmd

	b	next_pud

	.else		/* CONFIG_PGTABLE_LEVELS <= 2 */

	.else		/* CONFIG_PGTABLE_LEVELS <= 2 */

	mov	pmd, pud

	pmd		.req	pgd

	b	walk_ptes

	.set		.Lnext_pmd, .Lnext_pgd

next_pmd:

	b	next_pud

	.endif

	.endif





.Lderef_pmd:

	/* PTE */

	/* PTE */

walk_ptes:

	pte_to_phys	cur_ptep, pmd

	pte_to_phys	cur_ptep, pmd

	add	end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)

	kpti_mk_tbl_ng	pte, PTRS_PER_PTE

do_pte:	__idmap_kpti_get_pgtable_ent	pte

	b		.Lnext_pmd

	__idmap_kpti_put_pgtable_ent_ng	pte

skip_pte:

	add	cur_ptep, cur_ptep, #8

	cmp	cur_ptep, end_ptep

	b.ne	do_pte

	b	next_pmd





	.unreq	cpu

	.unreq	cpu

	.unreq	num_cpus

	.unreq	num_cpus