Commit 492449ae authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch irq/gic-v3-nmi-fixes-5.19 into irq/irqchip-next 



* irq/gic-v3-nmi-fixes-5.19:
  : .
  : GICv3 pseudo-NMI fixes from Mark Rutland:
  :
  : "These patches fix a couple of issues with the way GICv3 pseudo-NMIs are
  : handled:
  :
  : * The first patch adds a barrier we missed from NMI handling due to an
  :   oversight.
  :
  : * The second patch refactors some logic around reads from ICC_IAR1_EL1
  :   and adds commentary to explain what's going on.
  :
  : * The third patch descends into madness, reworking gic_handle_irq() to
  :   consistently manage ICC_PMR_EL1 + DAIF and avoid cases where these can
  :   be left in an inconsistent state while softirqs are processed."
  : .
  irqchip/gic-v3: Fix priority mask handling
  irqchip/gic-v3: Refactor ISB + EOIR at ack time
  irqchip/gic-v3: Ensure pseudo-NMIs have an ISB between ack and handling

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents 61299e18 614ab80c

arch/arm/include/asm/arch_gicv3.h

+1 −6
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ static inline u32 read_ ## a64(void) \ 
	return read_sysreg(a32); 		\

}						\



CPUIF_MAP(ICC_EOIR1, ICC_EOIR1_EL1)

CPUIF_MAP(ICC_PMR, ICC_PMR_EL1)

CPUIF_MAP(ICC_AP0R0, ICC_AP0R0_EL1)

CPUIF_MAP(ICC_AP0R1, ICC_AP0R1_EL1)
@@ -63,12 +64,6 @@ CPUIF_MAP(ICC_AP1R3, ICC_AP1R3_EL1) 


/* Low-level accessors */



static inline void gic_write_eoir(u32 irq)

{

	write_sysreg(irq, ICC_EOIR1);

	isb();

}



static inline void gic_write_dir(u32 val)

{

	write_sysreg(val, ICC_DIR);

arch/arm64/include/asm/arch_gicv3.h

+0 −6
Original line number Diff line number Diff line
@@ -26,12 +26,6 @@ 
 * sets the GP register's most significant bits to 0 with an explicit cast.

 */



static inline void gic_write_eoir(u32 irq)

{

	write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);

	isb();

}



static __always_inline void gic_write_dir(u32 irq)

{

	write_sysreg_s(irq, SYS_ICC_DIR_EL1);

drivers/irqchip/irq-gic-v3.c

+120 −63
Original line number Diff line number Diff line
@@ -559,7 +559,8 @@ static void gic_irq_nmi_teardown(struct irq_data *d) 


static void gic_eoi_irq(struct irq_data *d)

{

	gic_write_eoir(gic_irq(d));

	write_gicreg(gic_irq(d), ICC_EOIR1_EL1);

	isb();

}



static void gic_eoimode1_eoi_irq(struct irq_data *d)
@@ -639,40 +640,125 @@ static void gic_deactivate_unhandled(u32 irqnr) 
		if (irqnr < 8192)

			gic_write_dir(irqnr);

	} else {

		gic_write_eoir(irqnr);

		write_gicreg(irqnr, ICC_EOIR1_EL1);

		isb();

	}

}



static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)

/*

 * Follow a read of the IAR with any HW maintenance that needs to happen prior

 * to invoking the relevant IRQ handler. We must do two things:

 *

 * (1) Ensure instruction ordering between a read of IAR and subsequent

 *     instructions in the IRQ handler using an ISB.

 *

 *     It is possible for the IAR to report an IRQ which was signalled *after*

 *     the CPU took an IRQ exception as multiple interrupts can race to be

 *     recognized by the GIC, earlier interrupts could be withdrawn, and/or

 *     later interrupts could be prioritized by the GIC.

 *

 *     For devices which are tightly coupled to the CPU, such as PMUs, a

 *     context synchronization event is necessary to ensure that system

 *     register state is not stale, as these may have been indirectly written

 *     *after* exception entry.

 *

 * (2) Deactivate the interrupt when EOI mode 1 is in use.

 */

static inline void gic_complete_ack(u32 irqnr)

{

	bool irqs_enabled = interrupts_enabled(regs);

	int err;

	if (static_branch_likely(&supports_deactivate_key))

		write_gicreg(irqnr, ICC_EOIR1_EL1);



	if (irqs_enabled)

		nmi_enter();

	isb();

}



static bool gic_rpr_is_nmi_prio(void)

{

	if (!gic_supports_nmi())

		return false;



	return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI));

}



static bool gic_irqnr_is_special(u32 irqnr)

{

	return irqnr >= 1020 && irqnr <= 1023;

}



static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs)

{

	if (gic_irqnr_is_special(irqnr))

		return;



	gic_complete_ack(irqnr);



	if (generic_handle_domain_irq(gic_data.domain, irqnr)) {

		WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr);

		gic_deactivate_unhandled(irqnr);

	}

}



static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs)

{

	if (gic_irqnr_is_special(irqnr))

		return;



	gic_complete_ack(irqnr);



	if (generic_handle_domain_nmi(gic_data.domain, irqnr)) {

		WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr);

		gic_deactivate_unhandled(irqnr);

	}

}



	if (static_branch_likely(&supports_deactivate_key))

		gic_write_eoir(irqnr);

/*

	 * Leave the PSR.I bit set to prevent other NMIs to be

	 * received while handling this one.

	 * PSR.I will be restored when we ERET to the

	 * interrupted context.

 * An exception has been taken from a context with IRQs enabled, and this could

 * be an IRQ or an NMI.

 *

 * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear

 * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning,

 * after handling any NMI but before handling any IRQ.

 *

 * The entry code has performed IRQ entry, and if an NMI is detected we must

 * perform NMI entry/exit around invoking the handler.

 */

	err = generic_handle_domain_nmi(gic_data.domain, irqnr);

	if (err)

		gic_deactivate_unhandled(irqnr);

static void __gic_handle_irq_from_irqson(struct pt_regs *regs)

{

	bool is_nmi;

	u32 irqnr;



	if (irqs_enabled)

	irqnr = gic_read_iar();



	is_nmi = gic_rpr_is_nmi_prio();



	if (is_nmi) {

		nmi_enter();

		__gic_handle_nmi(irqnr, regs);

		nmi_exit();

	}



static u32 do_read_iar(struct pt_regs *regs)

{

	u32 iar;

	if (gic_prio_masking_enabled()) {

		gic_pmr_mask_irqs();

		gic_arch_enable_irqs();

	}



	if (gic_supports_nmi() && unlikely(!interrupts_enabled(regs))) {

	if (!is_nmi)

		__gic_handle_irq(irqnr, regs);

}



/*

 * An exception has been taken from a context with IRQs disabled, which can only

 * be an NMI.

 *

 * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave

 * DAIF.IF (and ICC_PMR_EL1) unchanged.

 *

 * The entry code has performed NMI entry.

 */

static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs)

{

	u64 pmr;

	u32 irqnr;



	/*

	 * We were in a context with IRQs disabled. However, the
@@ -690,47 +776,18 @@ static u32 do_read_iar(struct pt_regs *regs) 
	pmr = gic_read_pmr();

	gic_pmr_mask_irqs();

	isb();



		iar = gic_read_iar();



	irqnr = gic_read_iar();

	gic_write_pmr(pmr);

	} else {

		iar = gic_read_iar();

	}



	return iar;

	__gic_handle_nmi(irqnr, regs);

}



static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)

{

	u32 irqnr;



	irqnr = do_read_iar(regs);



	/* Check for special IDs first */

	if ((irqnr >= 1020 && irqnr <= 1023))

		return;



	if (gic_supports_nmi() &&

	    unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI))) {

		gic_handle_nmi(irqnr, regs);

		return;

	}



	if (gic_prio_masking_enabled()) {

		gic_pmr_mask_irqs();

		gic_arch_enable_irqs();

	}



	if (static_branch_likely(&supports_deactivate_key))

		gic_write_eoir(irqnr);

	if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs)))

		__gic_handle_irq_from_irqsoff(regs);

	else

		isb();



	if (generic_handle_domain_irq(gic_data.domain, irqnr)) {

		WARN_ONCE(true, "Unexpected interrupt received!\n");

		gic_deactivate_unhandled(irqnr);

	}

		__gic_handle_irq_from_irqson(regs);

}



static u32 gic_get_pribits(void)