test (3b833167) · Commits · jan.koester / libnetplus

src/crypto/aes.cpp

+107 −70

Changes for src/crypto/aes.cpp: 107 added lines, 70 removed lines.
Original line number	Diff line number	Diff line
		@@ -24,8 +24,13 @@
		// Bulldozer and later) also has SSE4.1/4.2 — there is no real-world AES-NI
		// CPU lacking it.
		#define AESNI_FUNC __attribute__((target("aes,ssse3,sse4.1,pclmul")))
		// Single-block AES-NI helpers only: must not pull in pclmul (or anything
		// beyond AES-NI) so they stay usable on CPUs/VMs that expose AES-NI
		// without PCLMULQDQ -- see g_has_aesni below.
		#define AESNI_BLOCK_FUNC __attribute__((target("aes,sse2")))
		#else
		#define AESNI_FUNC
		#define AESNI_BLOCK_FUNC
		#endif

		static inline std::uint32_t byteSwap32(std::uint32_t value) noexcept
		@@ -57,29 +62,43 @@ static inline std::uint32_t byteSwap32(std::uint32_t value) noexcept
		// core AEAD path in a crypto library. The existing 128-bit/4-block path
		// remains the baseline fallback and covers every AES-NI-capable CPU.

		static bool detect_aesni_pclmul() {
		struct AesCpuFeatures {
		bool aesni = false;
		bool pclmul = false;
		};

		static AesCpuFeatures detect_aes_features() {
		AesCpuFeatures f;
		#ifdef _MSC_VER
		int info[4];
		__cpuid(info, 1);
		bool aes = (info[2] & (1 << 25)) != 0;
		bool pclmul = (info[2] & (1 << 1)) != 0;
		return aes && pclmul;
		f.aesni = (info[2] & (1 << 25)) != 0;
		f.pclmul = (info[2] & (1 << 1)) != 0;
		#else
		unsigned eax, ebx, ecx, edx;
		if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
		bool aes = (ecx & (1 << 25)) != 0;
		bool pclmul = (ecx & (1 << 1)) != 0;
		return aes && pclmul;
		f.aesni = (ecx & (1 << 25)) != 0;
		f.pclmul = (ecx & (1 << 1)) != 0;
		}
		return false;
		#endif
		return f;
		}

		static const bool g_has_hw_aes = detect_aesni_pclmul();
		static const AesCpuFeatures g_aes_cpu = detect_aes_features();

		// Block encrypt/decrypt via AES-NI. Deliberately independent of PCLMULQDQ:
		// hypervisors' generic CPU models (e.g. QEMU "kvm64"/"qemu64" with +aes)
		// commonly expose AES-NI without PCLMULQDQ, and gating AES-NI on both used
		// to drop such hosts onto the constant-time software AES -- roughly three
		// orders of magnitude slower (TLS throughput collapsed to <1 MB/s/core).
		static const bool g_has_aesni = g_aes_cpu.aesni;

		// The fused AES-NI CTR + PCLMULQDQ GHASH GCM paths need both.
		static const bool g_has_hw_aes = g_aes_cpu.aesni && g_aes_cpu.pclmul;

		// ---- AES-NI encrypt/decrypt for AES-128 (10 rounds) ----

		AESNI_FUNC
		AESNI_BLOCK_FUNC
		static void aesni_encrypt_128(const uint8_t in[16], uint8_t out[16],
		const uint8_t* round_keys) {
		__m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(in));
		@@ -93,7 +112,7 @@ static void aesni_encrypt_128(const uint8_t in[16], uint8_t out[16],
		_mm_storeu_si128(reinterpret_cast<__m128i*>(out), block);
		}

		AESNI_FUNC
		AESNI_BLOCK_FUNC
		static void aesni_decrypt_128(const uint8_t in[16], uint8_t out[16],
		const uint8_t* round_keys) {
		__m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(in));
		@@ -110,7 +129,7 @@ static void aesni_decrypt_128(const uint8_t in[16], uint8_t out[16],

		// ---- AES-NI encrypt/decrypt for AES-256 (14 rounds) ----

		AESNI_FUNC
		AESNI_BLOCK_FUNC
		static void aesni_encrypt_256(const uint8_t in[16], uint8_t out[16],
		const uint8_t* round_keys) {
		__m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(in));
		@@ -124,7 +143,7 @@ static void aesni_encrypt_256(const uint8_t in[16], uint8_t out[16],
		_mm_storeu_si128(reinterpret_cast<__m128i*>(out), block);
		}

		AESNI_FUNC
		AESNI_BLOCK_FUNC
		static void aesni_decrypt_256(const uint8_t in[16], uint8_t out[16],
		const uint8_t* round_keys) {
		__m128i block = _mm_loadu_si128(reinterpret_cast<const __m128i*>(in));
		@@ -823,6 +842,71 @@ static void aesni_gcm_ctr_ghash_decrypt_256(

		#endif // NETPLUS_X86

		// ============================================================
		// Constant-time software GHASH multiply (no PCLMULQDQ)
		// ============================================================
		// Carry-less 64x64 multiply built from ordinary integer multiplies with
		// "holes" (every 4th bit kept, so carries can't spill into kept bits) --
		// the ctmul64 technique from BearSSL's ghash_ctmul64.c. Branch- and
		// table-free, so constant-time like the bit loop it replaces, but ~10x
		// fewer operations per block than walking all 128 bits of X.

		static inline uint64_t gf_bmul64(uint64_t x, uint64_t y) {
		const uint64_t m0 = 0x1111111111111111ULL, m1 = 0x2222222222222222ULL,
		m2 = 0x4444444444444444ULL, m3 = 0x8888888888888888ULL;
		uint64_t x0 = x & m0, x1 = x & m1, x2 = x & m2, x3 = x & m3;
		uint64_t y0 = y & m0, y1 = y & m1, y2 = y & m2, y3 = y & m3;
		uint64_t z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
		uint64_t z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
		uint64_t z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
		uint64_t z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
		return (z0 & m0) \| (z1 & m1) \| (z2 & m2) \| (z3 & m3);
		}

		static inline uint64_t gf_rev64(uint64_t x) {
		x = ((x & 0x5555555555555555ULL) << 1) \| ((x >> 1) & 0x5555555555555555ULL);
		x = ((x & 0x3333333333333333ULL) << 2) \| ((x >> 2) & 0x3333333333333333ULL);
		x = ((x & 0x0F0F0F0F0F0F0F0FULL) << 4) \| ((x >> 4) & 0x0F0F0F0F0F0F0F0FULL);
		x = ((x & 0x00FF00FF00FF00FFULL) << 8) \| ((x >> 8) & 0x00FF00FF00FF00FFULL);
		x = ((x & 0x0000FFFF0000FFFFULL) << 16) \| ((x >> 16) & 0x0000FFFF0000FFFFULL);
		return (x << 32) \| (x >> 32);
		}

		// X <- X * H in GCM's bit-reflected GF(2^128); hi/lo are the big-endian
		// first/second halves of each 16-byte block (as load_be64 yields them).
		static void gf128_mul_ct(uint64_t &Xhi, uint64_t &Xlo, uint64_t Hhi, uint64_t Hlo) {
		const uint64_t h0 = Hlo, h1 = Hhi, h2 = h0 ^ h1;
		const uint64_t h0r = gf_rev64(h0), h1r = gf_rev64(h1), h2r = h0r ^ h1r;
		const uint64_t y0 = Xlo, y1 = Xhi, y2 = y0 ^ y1;
		const uint64_t y0r = gf_rev64(y0), y1r = gf_rev64(y1), y2r = y0r ^ y1r;

		uint64_t z0 = gf_bmul64(y0, h0);
		uint64_t z1 = gf_bmul64(y1, h1);
		uint64_t z2 = gf_bmul64(y2, h2);
		uint64_t z0h = gf_bmul64(y0r, h0r);
		uint64_t z1h = gf_bmul64(y1r, h1r);
		uint64_t z2h = gf_bmul64(y2r, h2r);
		z2 ^= z0 ^ z1;
		z2h ^= z0h ^ z1h;
		z0h = gf_rev64(z0h) >> 1;
		z1h = gf_rev64(z1h) >> 1;
		z2h = gf_rev64(z2h) >> 1;

		uint64_t v0 = z0, v1 = z0h ^ z2, v2 = z1 ^ z2h, v3 = z1h;
		v3 = (v3 << 1) \| (v2 >> 63);
		v2 = (v2 << 1) \| (v1 >> 63);
		v1 = (v1 << 1) \| (v0 >> 63);
		v0 = (v0 << 1);

		v2 ^= v0 ^ (v0 >> 1) ^ (v0 >> 2) ^ (v0 >> 7);
		v1 ^= (v0 << 63) ^ (v0 << 62) ^ (v0 << 57);
		v3 ^= v1 ^ (v1 >> 1) ^ (v1 >> 2) ^ (v1 >> 7);
		v2 ^= (v1 << 63) ^ (v1 << 62) ^ (v1 << 57);

		Xlo = v2;
		Xhi = v3;
		}

		namespace netplus {

		// ============================================================
		@@ -1029,7 +1113,7 @@ aes128::aes128(const key128& key) {
		// AES block encrypt/decrypt
		void aes128::encrypt_block(const uint8_t in[16], uint8_t out[16]) const {
		#ifdef NETPLUS_X86
		if (g_has_hw_aes) {
		if (g_has_aesni) {
		aesni_encrypt_128(in, out, m_roundKeyBytes.data());
		return;
		}
		@@ -1053,7 +1137,7 @@ void aes128::encrypt_block(const uint8_t in[16], uint8_t out[16]) const {

		void aes128::decrypt_block(const uint8_t in[16], uint8_t out[16]) const {
		#ifdef NETPLUS_X86
		if (g_has_hw_aes) {
		if (g_has_aesni) {
		aesni_decrypt_128(in, out, m_roundKeyBytes.data());
		return;
		}
		@@ -1259,36 +1343,10 @@ void aes128::gcm_mul(std::array<uint8_t,16>& X) const {
		uint64_t Hhi = load_be64(m_H);
		uint64_t Hlo = load_be64(m_H + 8);

		uint64_t Zhi = 0, Zlo = 0;

		// Process each bit of X from MSB to LSB
		for (int i = 0; i < 128; i++) {
		// Get bit i of X (MSB first)
		uint64_t xi;
		if (i < 64)
		xi = (Xhi >> (63 - i)) & 1;
		else
		xi = (Xlo >> (63 - (i - 64))) & 1;
		gf128_mul_ct(Xhi, Xlo, Hhi, Hlo);

		// If bit is set, XOR H into Z
		if (xi) {
		Zhi ^= Hhi;
		Zlo ^= Hlo;
		}

		// Multiply H by x (shift right, reduce if needed)
		uint64_t lsb = Hlo & 1;
		Hlo = (Hlo >> 1) \| (Hhi << 63);
		Hhi = Hhi >> 1;

		// If LSB was 1, reduce by XORing with R (0xE1 << 56)
		if (lsb) {
		Hhi ^= 0xE100000000000000ULL;
		}
		}

		store_be64(X.data(), Zhi);
		store_be64(X.data() + 8, Zlo);
		store_be64(X.data(), Xhi);
		store_be64(X.data() + 8, Xlo);
		}

		void aes128::ghash(std::array<uint8_t,16>& out,
		@@ -1635,7 +1693,7 @@ aes256::aes256(const std::vector<uint8_t>& key) {

		void aes256::encrypt_block(const uint8_t in[16], uint8_t out[16]) const {
		#ifdef NETPLUS_X86
		if (g_has_hw_aes) {
		if (g_has_aesni) {
		aesni_encrypt_256(in, out, m_roundKeyBytes.data());
		return;
		}
		@@ -1659,7 +1717,7 @@ void aes256::encrypt_block(const uint8_t in[16], uint8_t out[16]) const {

		void aes256::decrypt_block(const uint8_t in[16], uint8_t out[16]) const {
		#ifdef NETPLUS_X86
		if (g_has_hw_aes) {
		if (g_has_aesni) {
		aesni_decrypt_256(in, out, m_roundKeyBytes.data());
		return;
		}
		@@ -1844,31 +1902,10 @@ void aes256::gcm_mul(std::array<uint8_t,16>& X) const {
		uint64_t Hhi = load_be64(m_H);
		uint64_t Hlo = load_be64(m_H + 8);

		uint64_t Zhi = 0, Zlo = 0;

		for (int i = 0; i < 128; i++) {
		uint64_t xi;
		if (i < 64)
		xi = (Xhi >> (63 - i)) & 1;
		else
		xi = (Xlo >> (63 - (i - 64))) & 1;

		if (xi) {
		Zhi ^= Hhi;
		Zlo ^= Hlo;
		}

		uint64_t lsb = Hlo & 1;
		Hlo = (Hlo >> 1) \| (Hhi << 63);
		Hhi = Hhi >> 1;

		if (lsb) {
		Hhi ^= 0xE100000000000000ULL;
		}
		}
		gf128_mul_ct(Xhi, Xlo, Hhi, Hlo);

		store_be64(X.data(), Zhi);
		store_be64(X.data() + 8, Zlo);
		store_be64(X.data(), Xhi);
		store_be64(X.data() + 8, Xlo);
		}

		void aes256::ghash(std::array<uint8_t,16>& out,