cpufeature.c

	/*
	 * Mismatched CPU features are a recipe for disaster. Don't even
	 * pretend to support them.
	 */
	if (taint) {
		pr_warn_once("Unsupported CPU feature variation detected.\n");
		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
	}
}

u64 read_sanitised_ftr_reg(u32 id)
{
	struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);

	/* We shouldn't get a request for an unsupported register */
	BUG_ON(!regp);
	return regp->sys_val;
}

#define read_sysreg_case(r)	\
	case r:		return read_sysreg_s(r)

/*
 * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
 * Read the system register on the current CPU
 */
static u64 __read_sysreg_by_encoding(u32 sys_id)
{
	switch (sys_id) {
	read_sysreg_case(SYS_ID_PFR0_EL1);
	read_sysreg_case(SYS_ID_PFR1_EL1);
	read_sysreg_case(SYS_ID_PFR2_EL1);
	read_sysreg_case(SYS_ID_DFR0_EL1);
	read_sysreg_case(SYS_ID_DFR1_EL1);
	read_sysreg_case(SYS_ID_MMFR0_EL1);
	read_sysreg_case(SYS_ID_MMFR1_EL1);
	read_sysreg_case(SYS_ID_MMFR2_EL1);
	read_sysreg_case(SYS_ID_MMFR3_EL1);
	read_sysreg_case(SYS_ID_MMFR5_EL1);
	read_sysreg_case(SYS_ID_ISAR0_EL1);
	read_sysreg_case(SYS_ID_ISAR1_EL1);
	read_sysreg_case(SYS_ID_ISAR2_EL1);
	read_sysreg_case(SYS_ID_ISAR3_EL1);
	read_sysreg_case(SYS_ID_ISAR4_EL1);
	read_sysreg_case(SYS_ID_ISAR5_EL1);
	read_sysreg_case(SYS_ID_ISAR6_EL1);
	read_sysreg_case(SYS_MVFR0_EL1);
	read_sysreg_case(SYS_MVFR1_EL1);
	read_sysreg_case(SYS_MVFR2_EL1);

	read_sysreg_case(SYS_ID_AA64PFR0_EL1);
	read_sysreg_case(SYS_ID_AA64PFR1_EL1);
	read_sysreg_case(SYS_ID_AA64ZFR0_EL1);
	read_sysreg_case(SYS_ID_AA64DFR0_EL1);
	read_sysreg_case(SYS_ID_AA64DFR1_EL1);
	read_sysreg_case(SYS_ID_AA64MMFR0_EL1);
	read_sysreg_case(SYS_ID_AA64MMFR1_EL1);
	read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
	read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
	read_sysreg_case(SYS_ID_AA64ISAR1_EL1);

	read_sysreg_case(SYS_CNTFRQ_EL0);
	read_sysreg_case(SYS_CTR_EL0);
	read_sysreg_case(SYS_DCZID_EL0);

	default:
		BUG();
		return 0;
	}
}

#include <linux/irqchip/arm-gic-v3.h>

static bool
feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
{
	int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign);

	return val >= entry->min_field_value;
}

static bool
has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
{
	u64 val;

	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
	if (scope == SCOPE_SYSTEM)
		val = read_sanitised_ftr_reg(entry->sys_reg);
	else
		val = __read_sysreg_by_encoding(entry->sys_reg);

	return feature_matches(val, entry);
}

static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
{
	bool has_sre;

	if (!has_cpuid_feature(entry, scope))
		return false;

	has_sre = gic_enable_sre();
	if (!has_sre)
		pr_warn_once("%s present but disabled by higher exception level\n",
			     entry->desc);

	return has_sre;
}

static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
{
	u32 midr = read_cpuid_id();

	/* Cavium ThunderX pass 1.x and 2.x */
	return midr_is_cpu_model_range(midr, MIDR_THUNDERX,
		MIDR_CPU_VAR_REV(0, 0),
		MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
}

static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
{
	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);

	return cpuid_feature_extract_signed_field(pfr0,
					ID_AA64PFR0_FP_SHIFT) < 0;
}

static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
			  int scope)
{
	u64 ctr;

	if (scope == SCOPE_SYSTEM)
		ctr = arm64_ftr_reg_ctrel0.sys_val;
	else
		ctr = read_cpuid_effective_cachetype();

	return ctr & BIT(CTR_IDC_SHIFT);
}

static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused)
{
	/*
	 * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively
	 * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses
	 * to the CTR_EL0 on this CPU and emulate it with the real/safe
	 * value.
	 */
	if (!(read_cpuid_cachetype() & BIT(CTR_IDC_SHIFT)))
		sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
}

static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
			  int scope)
{
	u64 ctr;

	if (scope == SCOPE_SYSTEM)
		ctr = arm64_ftr_reg_ctrel0.sys_val;
	else
		ctr = read_cpuid_cachetype();

	return ctr & BIT(CTR_DIC_SHIFT);
}

static bool __maybe_unused
has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope)
{
	/*
	 * Kdump isn't guaranteed to power-off all secondary CPUs, CNP
	 * may share TLB entries with a CPU stuck in the crashed
	 * kernel.
	 */
	 if (is_kdump_kernel())
		return false;

	return has_cpuid_feature(entry, scope);
}

/*
 * This check is triggered during the early boot before the cpufeature
 * is initialised. Checking the status on the local CPU allows the boot
 * CPU to detect the need for non-global mappings and thus avoiding a
 * pagetable re-write after all the CPUs are booted. This check will be
 * anyway run on individual CPUs, allowing us to get the consistent
 * state once the SMP CPUs are up and thus make the switch to non-global
 * mappings if required.
 */
bool kaslr_requires_kpti(void)
{
	if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
		return false;

	/*
	 * E0PD does a similar job to KPTI so can be used instead
	 * where available.
	 */
	if (IS_ENABLED(CONFIG_ARM64_E0PD)) {
		u64 mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
		if (cpuid_feature_extract_unsigned_field(mmfr2,
						ID_AA64MMFR2_E0PD_SHIFT))
			return false;
	}

	/*
	 * Systems affected by Cavium erratum 24756 are incompatible
	 * with KPTI.
	 */
	if (IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456)) {
		extern const struct midr_range cavium_erratum_27456_cpus[];

		if (is_midr_in_range_list(read_cpuid_id(),
					  cavium_erratum_27456_cpus))
			return false;
	}

	return kaslr_offset() > 0;
}

static bool __meltdown_safe = true;
static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */

static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
				int scope)
{
	/* List of CPUs that are not vulnerable and don't need KPTI */
	static const struct midr_range kpti_safe_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
		MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
		MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
		MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
		{ /* sentinel */ }
	};
	char const *str = "kpti command line option";
	bool meltdown_safe;

	meltdown_safe = is_midr_in_range_list(read_cpuid_id(), kpti_safe_list);

	/* Defer to CPU feature registers */
	if (has_cpuid_feature(entry, scope))
		meltdown_safe = true;

	if (!meltdown_safe)
		__meltdown_safe = false;

	/*
	 * For reasons that aren't entirely clear, enabling KPTI on Cavium
	 * ThunderX leads to apparent I-cache corruption of kernel text, which
	 * ends as well as you might imagine. Don't even try.
	 */
	if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
		str = "ARM64_WORKAROUND_CAVIUM_27456";
		__kpti_forced = -1;
	}

	/* Useful for KASLR robustness */
	if (kaslr_requires_kpti()) {
		if (!__kpti_forced) {
			str = "KASLR";
			__kpti_forced = 1;
		}
	}

	if (cpu_mitigations_off() && !__kpti_forced) {
		str = "mitigations=off";
		__kpti_forced = -1;
	}

	if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
		pr_info_once("kernel page table isolation disabled by kernel configuration\n");
		return false;
	}

	/* Forced? */
	if (__kpti_forced) {
		pr_info_once("kernel page table isolation forced %s by %s\n",
			     __kpti_forced > 0 ? "ON" : "OFF", str);
		return __kpti_forced > 0;
	}

	return !meltdown_safe;
}

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
static void
kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
{
	typedef void (kpti_remap_fn)(int, int, phys_addr_t);
	extern kpti_remap_fn idmap_kpti_install_ng_mappings;
	kpti_remap_fn *remap_fn;

	int cpu = smp_processor_id();

	/*
	 * We don't need to rewrite the page-tables if either we've done
	 * it already or we have KASLR enabled and therefore have not
	 * created any global mappings at all.
	 */
	if (arm64_use_ng_mappings)
		return;

	remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);

	cpu_install_idmap();
	remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
	cpu_uninstall_idmap();

	if (!cpu)
		arm64_use_ng_mappings = true;

	return;
}
#else
static void
kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
{
}
#endif	/* CONFIG_UNMAP_KERNEL_AT_EL0 */

static int __init parse_kpti(char *str)
{
	bool enabled;
	int ret = strtobool(str, &enabled);

	if (ret)
		return ret;

	__kpti_forced = enabled ? 1 : -1;
	return 0;
}
early_param("kpti", parse_kpti);

#ifdef CONFIG_ARM64_HW_AFDBM
static inline void __cpu_enable_hw_dbm(void)
{
	u64 tcr = read_sysreg(tcr_el1) | TCR_HD;

	write_sysreg(tcr, tcr_el1);
	isb();
}

static bool cpu_has_broken_dbm(void)
{
	/* List of CPUs which have broken DBM support. */
	static const struct midr_range cpus[] = {
#ifdef CONFIG_ARM64_ERRATUM_1024718
		MIDR_RANGE(MIDR_CORTEX_A55, 0, 0, 1, 0),  // A55 r0p0 -r1p0
#endif
		{},
	};

	return is_midr_in_range_list(read_cpuid_id(), cpus);
}

static bool cpu_can_use_dbm(const struct arm64_cpu_capabilities *cap)
{
	return has_cpuid_feature(cap, SCOPE_LOCAL_CPU) &&
	       !cpu_has_broken_dbm();
}

static void cpu_enable_hw_dbm(struct arm64_cpu_capabilities const *cap)
{
	if (cpu_can_use_dbm(cap))
		__cpu_enable_hw_dbm();
}

static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
		       int __unused)
{
	static bool detected = false;
	/*
	 * DBM is a non-conflicting feature. i.e, the kernel can safely
	 * run a mix of CPUs with and without the feature. So, we
	 * unconditionally enable the capability to allow any late CPU
	 * to use the feature. We only enable the control bits on the
	 * CPU, if it actually supports.
	 *
	 * We have to make sure we print the "feature" detection only
	 * when at least one CPU actually uses it. So check if this CPU
	 * can actually use it and print the message exactly once.
	 *
	 * This is safe as all CPUs (including secondary CPUs - due to the
	 * LOCAL_CPU scope - and the hotplugged CPUs - via verification)
	 * goes through the "matches" check exactly once. Also if a CPU
	 * matches the criteria, it is guaranteed that the CPU will turn
	 * the DBM on, as the capability is unconditionally enabled.
	 */
	if (!detected && cpu_can_use_dbm(cap)) {
		detected = true;
		pr_info("detected: Hardware dirty bit management\n");
	}

	return true;
}

#endif

#ifdef CONFIG_ARM64_AMU_EXTN

/*
 * The "amu_cpus" cpumask only signals that the CPU implementation for the
 * flagged CPUs supports the Activity Monitors Unit (AMU) but does not provide
 * information regarding all the events that it supports. When a CPU bit is
 * set in the cpumask, the user of this feature can only rely on the presence
 * of the 4 fixed counters for that CPU. But this does not guarantee that the
 * counters are enabled or access to these counters is enabled by code
 * executed at higher exception levels (firmware).
 */
static struct cpumask amu_cpus __read_mostly;

bool cpu_has_amu_feat(int cpu)
{
	return cpumask_test_cpu(cpu, &amu_cpus);
}

/* Initialize the use of AMU counters for frequency invariance */
extern void init_cpu_freq_invariance_counters(void);

static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap)
{
	if (has_cpuid_feature(cap, SCOPE_LOCAL_CPU)) {
		pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n",
			smp_processor_id());
		cpumask_set_cpu(smp_processor_id(), &amu_cpus);
		init_cpu_freq_invariance_counters();
	}
}

static bool has_amu(const struct arm64_cpu_capabilities *cap,
		    int __unused)
{
	/*
	 * The AMU extension is a non-conflicting feature: the kernel can
	 * safely run a mix of CPUs with and without support for the
	 * activity monitors extension. Therefore, unconditionally enable
	 * the capability to allow any late CPU to use the feature.
	 *
	 * With this feature unconditionally enabled, the cpu_enable
	 * function will be called for all CPUs that match the criteria,
	 * including secondary and hotplugged, marking this feature as
	 * present on that respective CPU. The enable function will also
	 * print a detection message.
	 */

	return true;
}
#endif

#ifdef CONFIG_ARM64_VHE
static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
{
	return is_kernel_in_hyp_mode();
}

static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
{
	/*
	 * Copy register values that aren't redirected by hardware.
	 *
	 * Before code patching, we only set tpidr_el1, all CPUs need to copy
	 * this value to tpidr_el2 before we patch the code. Once we've done
	 * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
	 * do anything here.
	 */
	if (!alternative_is_applied(ARM64_HAS_VIRT_HOST_EXTN))
		write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
}
#endif

static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused)
{
	u64 val = read_sysreg_s(SYS_CLIDR_EL1);

	/* Check that CLIDR_EL1.LOU{U,IS} are both 0 */
	WARN_ON(val & (7 << 27 | 7 << 21));
}

#ifdef CONFIG_ARM64_SSBD
static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
{
	if (user_mode(regs))
		return 1;

	if (instr & BIT(PSTATE_Imm_shift))
		regs->pstate |= PSR_SSBS_BIT;
	else
		regs->pstate &= ~PSR_SSBS_BIT;

	arm64_skip_faulting_instruction(regs, 4);
	return 0;
}

static struct undef_hook ssbs_emulation_hook = {
	.instr_mask	= ~(1U << PSTATE_Imm_shift),
	.instr_val	= 0xd500401f | PSTATE_SSBS,
	.fn		= ssbs_emulation_handler,
};

static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused)
{
	static bool undef_hook_registered = false;
	static DEFINE_RAW_SPINLOCK(hook_lock);

	raw_spin_lock(&hook_lock);
	if (!undef_hook_registered) {
		register_undef_hook(&ssbs_emulation_hook);
		undef_hook_registered = true;
	}
	raw_spin_unlock(&hook_lock);

	if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
		sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
		arm64_set_ssbd_mitigation(false);
	} else {
		arm64_set_ssbd_mitigation(true);
	}
}
#endif /* CONFIG_ARM64_SSBD */

#ifdef CONFIG_ARM64_PAN
static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
{
	/*
	 * We modify PSTATE. This won't work from irq context as the PSTATE
	 * is discarded once we return from the exception.
	 */
	WARN_ON_ONCE(in_interrupt());

	sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
	asm(SET_PSTATE_PAN(1));
}
#endif /* CONFIG_ARM64_PAN */

#ifdef CONFIG_ARM64_RAS_EXTN
static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
{
	/* Firmware may have left a deferred SError in this register. */
	write_sysreg_s(0, SYS_DISR_EL1);
}
#endif /* CONFIG_ARM64_RAS_EXTN */

#ifdef CONFIG_ARM64_PTR_AUTH
static bool has_address_auth(const struct arm64_cpu_capabilities *entry,
			     int __unused)
{
	return __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_ARCH) ||
	       __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF);
}

static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
			     int __unused)
{
	return __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH) ||
	       __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF);
}
#endif /* CONFIG_ARM64_PTR_AUTH */

#ifdef CONFIG_ARM64_E0PD
static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap)
{
	if (this_cpu_has_cap(ARM64_HAS_E0PD))
		sysreg_clear_set(tcr_el1, 0, TCR_E0PD1);
}
#endif /* CONFIG_ARM64_E0PD */

#ifdef CONFIG_ARM64_PSEUDO_NMI
static bool enable_pseudo_nmi;

static int __init early_enable_pseudo_nmi(char *p)
{
	return strtobool(p, &enable_pseudo_nmi);
}
early_param("irqchip.gicv3_pseudo_nmi", early_enable_pseudo_nmi);

static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
				   int scope)
{
	return enable_pseudo_nmi && has_useable_gicv3_cpuif(entry, scope);
}
#endif

/* Internal helper functions to match cpu capability type */
static bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
{
	return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
}

static bool
cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
{
	return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
}

static bool
cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
{
	return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT);
}

static const struct arm64_cpu_capabilities arm64_features[] = {
	{
		.desc = "GIC system register CPU interface",
		.capability = ARM64_HAS_SYSREG_GIC_CPUIF,
		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
		.matches = has_useable_gicv3_cpuif,
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.field_pos = ID_AA64PFR0_GIC_SHIFT,
		.sign = FTR_UNSIGNED,
		.min_field_value = 1,
	},
#ifdef CONFIG_ARM64_PAN
	{
		.desc = "Privileged Access Never",
		.capability = ARM64_HAS_PAN,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64MMFR1_EL1,
		.field_pos = ID_AA64MMFR1_PAN_SHIFT,
		.sign = FTR_UNSIGNED,
		.min_field_value = 1,
		.cpu_enable = cpu_enable_pan,
	},
#endif /* CONFIG_ARM64_PAN */
#ifdef CONFIG_ARM64_LSE_ATOMICS
	{
		.desc = "LSE atomic instructions",
		.capability = ARM64_HAS_LSE_ATOMICS,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64ISAR0_EL1,
		.field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
		.sign = FTR_UNSIGNED,
		.min_field_value = 2,
	},
#endif /* CONFIG_ARM64_LSE_ATOMICS */
	{
		.desc = "Software prefetching using PRFM",
		.capability = ARM64_HAS_NO_HW_PREFETCH,
		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
		.matches = has_no_hw_prefetch,
	},
#ifdef CONFIG_ARM64_UAO
	{
		.desc = "User Access Override",
		.capability = ARM64_HAS_UAO,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64MMFR2_EL1,
		.field_pos = ID_AA64MMFR2_UAO_SHIFT,
		.min_field_value = 1,
		/*
		 * We rely on stop_machine() calling uao_thread_switch() to set
		 * UAO immediately after patching.
		 */
	},
#endif /* CONFIG_ARM64_UAO */
#ifdef CONFIG_ARM64_PAN
	{
		.capability = ARM64_ALT_PAN_NOT_UAO,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = cpufeature_pan_not_uao,
	},
#endif /* CONFIG_ARM64_PAN */
#ifdef CONFIG_ARM64_VHE
	{
		.desc = "Virtualization Host Extensions",
		.capability = ARM64_HAS_VIRT_HOST_EXTN,
		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
		.matches = runs_at_el2,
		.cpu_enable = cpu_copy_el2regs,
	},
#endif	/* CONFIG_ARM64_VHE */
	{
		.desc = "32-bit EL0 Support",
		.capability = ARM64_HAS_32BIT_EL0,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64PFR0_EL0_SHIFT,
		.min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT,
	},
#ifdef CONFIG_KVM
	{
		.desc = "32-bit EL1 Support",
		.capability = ARM64_HAS_32BIT_EL1,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64PFR0_EL1_SHIFT,
		.min_field_value = ID_AA64PFR0_EL1_32BIT_64BIT,
	},
#endif
	{
		.desc = "Kernel page table isolation (KPTI)",
		.capability = ARM64_UNMAP_KERNEL_AT_EL0,
		.type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
		/*
		 * The ID feature fields below are used to indicate that
		 * the CPU doesn't need KPTI. See unmap_kernel_at_el0 for
		 * more details.
		 */
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.field_pos = ID_AA64PFR0_CSV3_SHIFT,
		.min_field_value = 1,
		.matches = unmap_kernel_at_el0,
		.cpu_enable = kpti_install_ng_mappings,
	},
	{
		/* FP/SIMD is not implemented */
		.capability = ARM64_HAS_NO_FPSIMD,
		.type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
		.min_field_value = 0,
		.matches = has_no_fpsimd,
	},
#ifdef CONFIG_ARM64_PMEM
	{
		.desc = "Data cache clean to Point of Persistence",
		.capability = ARM64_HAS_DCPOP,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64ISAR1_EL1,
		.field_pos = ID_AA64ISAR1_DPB_SHIFT,
		.min_field_value = 1,
	},
	{
		.desc = "Data cache clean to Point of Deep Persistence",
		.capability = ARM64_HAS_DCPODP,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64ISAR1_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64ISAR1_DPB_SHIFT,
		.min_field_value = 2,
	},
#endif
#ifdef CONFIG_ARM64_SVE
	{
		.desc = "Scalable Vector Extension",
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.capability = ARM64_SVE,
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64PFR0_SVE_SHIFT,
		.min_field_value = ID_AA64PFR0_SVE,
		.matches = has_cpuid_feature,
		.cpu_enable = sve_kernel_enable,
	},
#endif /* CONFIG_ARM64_SVE */
#ifdef CONFIG_ARM64_RAS_EXTN
	{
		.desc = "RAS Extension Support",
		.capability = ARM64_HAS_RAS_EXTN,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64PFR0_RAS_SHIFT,
		.min_field_value = ID_AA64PFR0_RAS_V1,
		.cpu_enable = cpu_clear_disr,
	},
#endif /* CONFIG_ARM64_RAS_EXTN */
#ifdef CONFIG_ARM64_AMU_EXTN
	{
		/*
		 * The feature is enabled by default if CONFIG_ARM64_AMU_EXTN=y.
		 * Therefore, don't provide .desc as we don't want the detection
		 * message to be shown until at least one CPU is detected to
		 * support the feature.
		 */
		.capability = ARM64_HAS_AMU_EXTN,
		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
		.matches = has_amu,
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64PFR0_AMU_SHIFT,
		.min_field_value = ID_AA64PFR0_AMU,
		.cpu_enable = cpu_amu_enable,
	},
#endif /* CONFIG_ARM64_AMU_EXTN */
	{
		.desc = "Data cache clean to the PoU not required for I/D coherence",
		.capability = ARM64_HAS_CACHE_IDC,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cache_idc,
		.cpu_enable = cpu_emulate_effective_ctr,
	},
	{
		.desc = "Instruction cache invalidation not required for I/D coherence",
		.capability = ARM64_HAS_CACHE_DIC,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cache_dic,
	},
	{
		.desc = "Stage-2 Force Write-Back",
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.capability = ARM64_HAS_STAGE2_FWB,
		.sys_reg = SYS_ID_AA64MMFR2_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64MMFR2_FWB_SHIFT,
		.min_field_value = 1,
		.matches = has_cpuid_feature,
		.cpu_enable = cpu_has_fwb,
	},
#ifdef CONFIG_ARM64_HW_AFDBM
	{
		/*
		 * Since we turn this on always, we don't want the user to
		 * think that the feature is available when it may not be.
		 * So hide the description.
		 *
		 * .desc = "Hardware pagetable Dirty Bit Management",
		 *
		 */
		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
		.capability = ARM64_HW_DBM,
		.sys_reg = SYS_ID_AA64MMFR1_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64MMFR1_HADBS_SHIFT,
		.min_field_value = 2,
		.matches = has_hw_dbm,
		.cpu_enable = cpu_enable_hw_dbm,
	},
#endif
	{
		.desc = "CRC32 instructions",
		.capability = ARM64_HAS_CRC32,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64ISAR0_EL1,
		.field_pos = ID_AA64ISAR0_CRC32_SHIFT,
		.min_field_value = 1,
	},
#ifdef CONFIG_ARM64_SSBD
	{
		.desc = "Speculative Store Bypassing Safe (SSBS)",
		.capability = ARM64_SSBS,
		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64PFR1_EL1,
		.field_pos = ID_AA64PFR1_SSBS_SHIFT,
		.sign = FTR_UNSIGNED,
		.min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,
		.cpu_enable = cpu_enable_ssbs,
	},
#endif
#ifdef CONFIG_ARM64_CNP
	{
		.desc = "Common not Private translations",
		.capability = ARM64_HAS_CNP,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_useable_cnp,
		.sys_reg = SYS_ID_AA64MMFR2_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64MMFR2_CNP_SHIFT,
		.min_field_value = 1,
		.cpu_enable = cpu_enable_cnp,
	},
#endif
	{
		.desc = "Speculation barrier (SB)",
		.capability = ARM64_HAS_SB,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64ISAR1_EL1,
		.field_pos = ID_AA64ISAR1_SB_SHIFT,
		.sign = FTR_UNSIGNED,
		.min_field_value = 1,
	},
#ifdef CONFIG_ARM64_PTR_AUTH
	{
		.desc = "Address authentication (architected algorithm)",
		.capability = ARM64_HAS_ADDRESS_AUTH_ARCH,
		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
		.sys_reg = SYS_ID_AA64ISAR1_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64ISAR1_APA_SHIFT,
		.min_field_value = ID_AA64ISAR1_APA_ARCHITECTED,
		.matches = has_cpuid_feature,
	},
	{
		.desc = "Address authentication (IMP DEF algorithm)",
		.capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
		.sys_reg = SYS_ID_AA64ISAR1_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64ISAR1_API_SHIFT,
		.min_field_value = ID_AA64ISAR1_API_IMP_DEF,
		.matches = has_cpuid_feature,
	},
	{
		.capability = ARM64_HAS_ADDRESS_AUTH,
		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
		.matches = has_address_auth,
	},
	{
		.desc = "Generic authentication (architected algorithm)",
		.capability = ARM64_HAS_GENERIC_AUTH_ARCH,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.sys_reg = SYS_ID_AA64ISAR1_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64ISAR1_GPA_SHIFT,
		.min_field_value = ID_AA64ISAR1_GPA_ARCHITECTED,
		.matches = has_cpuid_feature,
	},
	{
		.desc = "Generic authentication (IMP DEF algorithm)",
		.capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.sys_reg = SYS_ID_AA64ISAR1_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64ISAR1_GPI_SHIFT,
		.min_field_value = ID_AA64ISAR1_GPI_IMP_DEF,
		.matches = has_cpuid_feature,
	},
	{
		.capability = ARM64_HAS_GENERIC_AUTH,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_generic_auth,
	},
#endif /* CONFIG_ARM64_PTR_AUTH */
#ifdef CONFIG_ARM64_PSEUDO_NMI
	{
		/*
		 * Depends on having GICv3
		 */
		.desc = "IRQ priority masking",
		.capability = ARM64_HAS_IRQ_PRIO_MASKING,
		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
		.matches = can_use_gic_priorities,
		.sys_reg = SYS_ID_AA64PFR0_EL1,
		.field_pos = ID_AA64PFR0_GIC_SHIFT,
		.sign = FTR_UNSIGNED,
		.min_field_value = 1,
	},
#endif
#ifdef CONFIG_ARM64_E0PD
	{
		.desc = "E0PD",
		.capability = ARM64_HAS_E0PD,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.sys_reg = SYS_ID_AA64MMFR2_EL1,
		.sign = FTR_UNSIGNED,
		.field_pos = ID_AA64MMFR2_E0PD_SHIFT,
		.matches = has_cpuid_feature,
		.min_field_value = 1,
		.cpu_enable = cpu_enable_e0pd,
	},
#endif
#ifdef CONFIG_ARCH_RANDOM
	{
		.desc = "Random Number Generator",
		.capability = ARM64_HAS_RNG,
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		.matches = has_cpuid_feature,
		.sys_reg = SYS_ID_AA64ISAR0_EL1,
		.field_pos = ID_AA64ISAR0_RNDR_SHIFT,
		.sign = FTR_UNSIGNED,
		.min_field_value = 1,
	},
#endif
	{},
};

#define HWCAP_CPUID_MATCH(reg, field, s, min_value)				\
		.matches = has_cpuid_feature,					\
		.sys_reg = reg,							\
		.field_pos = field,						\
		.sign = s,							\
		.min_field_value = min_value,

#define __HWCAP_CAP(name, cap_type, cap)					\
		.desc = name,							\
		.type = ARM64_CPUCAP_SYSTEM_FEATURE,				\
		.hwcap_type = cap_type,						\
		.hwcap = cap,							\

#define HWCAP_CAP(reg, field, s, min_value, cap_type, cap)			\
	{									\
		__HWCAP_CAP(#cap, cap_type, cap)				\
		HWCAP_CPUID_MATCH(reg, field, s, min_value)			\
	}

#define HWCAP_MULTI_CAP(list, cap_type, cap)					\
	{									\
		__HWCAP_CAP(#cap, cap_type, cap)				\
		.matches = cpucap_multi_entry_cap_matches,			\
		.match_list = list,						\
	}