Commit e36ae229 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'x86_fpu_for_v5.19_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 fpu updates from Borislav Petkov:

 - Add support for XSAVEC - the Compacted XSTATE saving variant - and
   thus allow for guests to use this compacted XSTATE variant when the
   hypervisor exports that support

 - A variable shadowing cleanup

* tag 'x86_fpu_for_v5.19_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/fpu: Cleanup variable shadowing
  x86/fpu/xsave: Support XSAVEC in the kernel
parents de8ac817 b91c0922

arch/x86/include/asm/cpufeatures.h

+1 −1
Original line number Diff line number Diff line
@@ -201,7 +201,7 @@ 
#define X86_FEATURE_INVPCID_SINGLE	( 7*32+ 7) /* Effectively INVPCID && CR4.PCIDE=1 */

#define X86_FEATURE_HW_PSTATE		( 7*32+ 8) /* AMD HW-PState */

#define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */

/* FREE!                                ( 7*32+10) */

#define X86_FEATURE_XCOMPACTED		( 7*32+10) /* "" Use compacted XSTATE (XSAVES or XSAVEC) */

#define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */

#define X86_FEATURE_RETPOLINE		( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */

#define X86_FEATURE_RETPOLINE_LFENCE	( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */

arch/x86/kernel/fpu/xstate.c

+39 −21
Original line number Diff line number Diff line
@@ -142,7 +142,8 @@ static unsigned int xfeature_get_offset(u64 xcomp_bv, int xfeature) 
	 * Non-compacted format and legacy features use the cached fixed

	 * offsets.

	 */

	if (!cpu_feature_enabled(X86_FEATURE_XSAVES) || xfeature <= XFEATURE_SSE)

	if (!cpu_feature_enabled(X86_FEATURE_XCOMPACTED) ||

	    xfeature <= XFEATURE_SSE)

		return xstate_offsets[xfeature];



	/*
@@ -369,12 +370,12 @@ static void __init setup_init_fpu_buf(void) 
	/*

	 * All components are now in init state. Read the state back so

	 * that init_fpstate contains all non-zero init state. This only

	 * works with XSAVE, but not with XSAVEOPT and XSAVES because

	 * works with XSAVE, but not with XSAVEOPT and XSAVEC/S because

	 * those use the init optimization which skips writing data for

	 * components in init state.

	 *

	 * XSAVE could be used, but that would require to reshuffle the

	 * data when XSAVES is available because XSAVES uses xstate

	 * data when XSAVEC/S is available because XSAVEC/S uses xstate

	 * compaction. But doing so is a pointless exercise because most

	 * components have an all zeros init state except for the legacy

	 * ones (FP and SSE). Those can be saved with FXSAVE into the
@@ -584,7 +585,8 @@ static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) 
 */

static bool __init paranoid_xstate_size_valid(unsigned int kernel_size)

{

	bool compacted = cpu_feature_enabled(X86_FEATURE_XSAVES);

	bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);

	bool xsaves = cpu_feature_enabled(X86_FEATURE_XSAVES);

	unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE;

	int i;
@@ -595,7 +597,7 @@ static bool __init paranoid_xstate_size_valid(unsigned int kernel_size) 
		 * Supervisor state components can be managed only by

		 * XSAVES.

		 */

		if (!compacted && xfeature_is_supervisor(i)) {

		if (!xsaves && xfeature_is_supervisor(i)) {

			XSTATE_WARN_ON(1);

			return false;

		}
@@ -612,8 +614,11 @@ static bool __init paranoid_xstate_size_valid(unsigned int kernel_size) 
 * the size of the *user* states.  If we use it to size a buffer

 * that we use 'XSAVES' on, we could potentially overflow the

 * buffer because 'XSAVES' saves system states too.

 *

 * This also takes compaction into account. So this works for

 * XSAVEC as well.

 */

static unsigned int __init get_xsaves_size(void)

static unsigned int __init get_compacted_size(void)

{

	unsigned int eax, ebx, ecx, edx;

	/*
@@ -623,6 +628,10 @@ static unsigned int __init get_xsaves_size(void) 
	 *    containing all the state components

	 *    corresponding to bits currently set in

	 *    XCR0 | IA32_XSS.

	 *

	 * When XSAVES is not available but XSAVEC is (virt), then there

	 * are no supervisor states, but XSAVEC still uses compacted

	 * format.

	 */

	cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);

	return ebx;
@@ -632,13 +641,13 @@ static unsigned int __init get_xsaves_size(void) 
 * Get the total size of the enabled xstates without the independent supervisor

 * features.

 */

static unsigned int __init get_xsaves_size_no_independent(void)

static unsigned int __init get_xsave_compacted_size(void)

{

	u64 mask = xfeatures_mask_independent();

	unsigned int size;



	if (!mask)

		return get_xsaves_size();

		return get_compacted_size();



	/* Disable independent features. */

	wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
@@ -647,7 +656,7 @@ static unsigned int __init get_xsaves_size_no_independent(void) 
	 * Ask the hardware what size is required of the buffer.

	 * This is the size required for the task->fpu buffer.

	 */

	size = get_xsaves_size();

	size = get_compacted_size();



	/* Re-enable independent features so XSAVES will work on them again. */

	wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
@@ -687,20 +696,21 @@ static int __init init_xstate_size(void) 
{

	/* Recompute the context size for enabled features: */

	unsigned int user_size, kernel_size, kernel_default_size;

	bool compacted = cpu_feature_enabled(X86_FEATURE_XSAVES);

	bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);



	/* Uncompacted user space size */

	user_size = get_xsave_size_user();



	/*

	 * XSAVES kernel size includes supervisor states and

	 * uses compacted format when available.

	 * XSAVES kernel size includes supervisor states and uses compacted

	 * format. XSAVEC uses compacted format, but does not save

	 * supervisor states.

	 *

	 * XSAVE does not support supervisor states so

	 * kernel and user size is identical.

	 * XSAVE[OPT] do not support supervisor states so kernel and user

	 * size is identical.

	 */

	if (compacted)

		kernel_size = get_xsaves_size_no_independent();

		kernel_size = get_xsave_compacted_size();

	else

		kernel_size = user_size;
@@ -813,6 +823,9 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) 
	if (!cpu_feature_enabled(X86_FEATURE_XFD))

		fpu_kernel_cfg.max_features &= ~XFEATURE_MASK_USER_DYNAMIC;



	if (!cpu_feature_enabled(X86_FEATURE_XSAVES))

		fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED;

	else

		fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED |

					XFEATURE_MASK_SUPERVISOR_SUPPORTED;
@@ -837,6 +850,11 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) 
	 */

	init_fpstate.xfd = fpu_user_cfg.max_features & XFEATURE_MASK_USER_DYNAMIC;



	/* Set up compaction feature bit */

	if (cpu_feature_enabled(X86_FEATURE_XSAVEC) ||

	    cpu_feature_enabled(X86_FEATURE_XSAVES))

		setup_force_cpu_cap(X86_FEATURE_XCOMPACTED);



	/* Enable xstate instructions to be able to continue with initialization: */

	fpu__init_cpu_xstate();
@@ -873,7 +891,7 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) 
	pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",

		fpu_kernel_cfg.max_features,

		fpu_kernel_cfg.max_size,

		boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");

		boot_cpu_has(X86_FEATURE_XCOMPACTED) ? "compacted" : "standard");

	return;



out_disable:
@@ -917,7 +935,7 @@ static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) 
	if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr)))

		return NULL;



	if (cpu_feature_enabled(X86_FEATURE_XSAVES)) {

	if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED)) {

		if (WARN_ON_ONCE(!(xcomp_bv & BIT_ULL(xfeature_nr))))

			return NULL;

	}
@@ -1215,7 +1233,7 @@ static int copy_uabi_to_xstate(struct fpstate *fpstate, const void *kbuf, 
	}



	for (i = 0; i < XFEATURE_MAX; i++) {

		u64 mask = ((u64)1 << i);

		mask = BIT_ULL(i);



		if (hdr.xfeatures & mask) {

			void *dst = __raw_xsave_addr(xsave, i);
@@ -1525,7 +1543,7 @@ static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) 
	 * vendors into extending XFD for the pre AMX states, especially

	 * AVX512.

	 */

	bool compacted = cpu_feature_enabled(X86_FEATURE_XSAVES);

	bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);

	struct fpu *fpu = &current->group_leader->thread.fpu;

	struct fpu_state_perm *perm;

	unsigned int ksize, usize;

arch/x86/kernel/fpu/xstate.h

+9 −5
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask) 
	 * XRSTORS requires these bits set in xcomp_bv, or it will

	 * trigger #GP:

	 */

	if (cpu_feature_enabled(X86_FEATURE_XSAVES))

	if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED))

		xsave->header.xcomp_bv = mask | XCOMP_BV_COMPACTED_FORMAT;

}
@@ -79,6 +79,7 @@ static inline u64 xfeatures_mask_independent(void) 
/* These macros all use (%edi)/(%rdi) as the single memory argument. */

#define XSAVE		".byte " REX_PREFIX "0x0f,0xae,0x27"

#define XSAVEOPT	".byte " REX_PREFIX "0x0f,0xae,0x37"

#define XSAVEC		".byte " REX_PREFIX "0x0f,0xc7,0x27"

#define XSAVES		".byte " REX_PREFIX "0x0f,0xc7,0x2f"

#define XRSTOR		".byte " REX_PREFIX "0x0f,0xae,0x2f"

#define XRSTORS		".byte " REX_PREFIX "0x0f,0xc7,0x1f"
@@ -97,9 +98,11 @@ static inline u64 xfeatures_mask_independent(void) 
		     : "memory")



/*

 * If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact

 * format and supervisor states in addition to modified optimization in

 * XSAVEOPT.

 * If XSAVES is enabled, it replaces XSAVEC because it supports supervisor

 * states in addition to XSAVEC.

 *

 * Otherwise if XSAVEC is enabled, it replaces XSAVEOPT because it supports

 * compacted storage format in addition to XSAVEOPT.

 *

 * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT

 * supports modified optimization which is not supported by XSAVE.
@@ -111,8 +114,9 @@ static inline u64 xfeatures_mask_independent(void) 
 * address of the instruction where we might get an exception at.

 */

#define XSTATE_XSAVE(st, lmask, hmask, err)				\

	asm volatile(ALTERNATIVE_2(XSAVE,				\

	asm volatile(ALTERNATIVE_3(XSAVE,				\

				   XSAVEOPT, X86_FEATURE_XSAVEOPT,	\

				   XSAVEC,   X86_FEATURE_XSAVEC,	\

				   XSAVES,   X86_FEATURE_XSAVES)	\

		     "\n"						\

		     "xor %[err], %[err]\n"				\