test (3d2b640c) · Commits · jan.koester / libhttppp

src/hpack.cpp

+46 −6

Changes for src/hpack.cpp: 46 added lines, 6 removed lines.
Original line number	Diff line number	Diff line
		@@ -463,16 +463,57 @@ static void appendInt(std::string &out, uint8_t prefix_bits, uint8_t prefix_mask
		out.push_back(static_cast<char>(value));
		}

		// Huffman-encode using the same HUFF_TABLE the decoder uses (indexed by
		// symbol value 0-255, giving each byte's canonical {code, bits} directly --
		// no separate encode table to keep in sync). Previously there was no encoder
		// at all; every literal was sent raw, leaving ~20-30% bandwidth on the table
		// for typical ASCII header values (paths, cookies, tokens). Shared with QPACK
		// (RFC 9204 reuses this exact table), see hpack.h.
		std::string Decoder::huffmanEncode(const std::string &in) {
		std::string out;
		out.reserve(in.size()); // Huffman output is never larger than the input for real text
		uint64_t bitbuf = 0;
		int nbits = 0;
		for (unsigned char c : in) {
		bitbuf = (bitbuf << HUFF_TABLE[c].bits) \| HUFF_TABLE[c].code;
		nbits += HUFF_TABLE[c].bits;
		while (nbits >= 8) {
		nbits -= 8;
		out.push_back(static_cast<char>((bitbuf >> nbits) & 0xFF));
		}
		bitbuf &= (nbits > 0) ? ((1ULL << nbits) - 1) : 0;
		}
		if (nbits > 0) {
		// Pad with 1-bits -- the most significant bits of the EOS code
		// (0x3fffffff, i.e. all-ones), per RFC 7541 §5.2.
		uint8_t pad = static_cast<uint8_t>((bitbuf << (8 - nbits)) \| ((1u << (8 - nbits)) - 1));
		out.push_back(static_cast<char>(pad));
		}
		return out;
		}

		// Appends a string literal (RFC 7541 §5.2), choosing whichever of raw vs.
		// Huffman-encoded is shorter -- the H-bit is the encoder's choice, and
		// preferring the smaller representation is what real HPACK encoders do.
		static void appendString(std::string &out, const std::string &s) {
		std::string huff = Decoder::huffmanEncode(s);
		if (huff.size() < s.size()) {
		appendInt(out, 7, 0x80, static_cast<uint32_t>(huff.size()));
		out.append(huff);
		} else {
		appendInt(out, 7, 0x00, static_cast<uint32_t>(s.size()));
		out.append(s);
		}
		}

		void Encoder::appendLiteral(std::string &out,
		const std::string &name,
		const std::string &value,
		bool incremental_index) {
		uint8_t prefix_mask = incremental_index ? 0x40 : 0x00;
		appendInt(out, 6, prefix_mask, 0);
		appendInt(out, 7, 0x00, static_cast<uint32_t>(name.size()));
		out.append(name);
		appendInt(out, 7, 0x00, static_cast<uint32_t>(value.size()));
		out.append(value);
		appendString(out, name);
		appendString(out, value);
		}

		void Encoder::appendLiteralWithNameIndex(std::string &out,
		@@ -481,8 +522,7 @@ void Encoder::appendLiteralWithNameIndex(std::string &out,
		bool incremental_index) {
		uint8_t prefix_mask = incremental_index ? 0x40 : 0x00;
		appendInt(out, 6, prefix_mask, name_index);
		appendInt(out, 7, 0x00, static_cast<uint32_t>(value.size()));
		out.append(value);
		appendString(out, value);
		}

		std::string Encoder::encodeResponseHeaders(uint16_t status_code,

src/hpack.h

+4 −0

Changes for src/hpack.h: 4 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -91,6 +91,10 @@ public:
		static std::string decodeString(const uint8_t *data, size_t len, size_t &bytes_read);
		static std::string huffmanDecode(const uint8_t *data, size_t len);

		// RFC 9204 (QPACK) reuses RFC 7541 Appendix B's Huffman table verbatim,
		// so this is shared with the QPACK encoder too rather than duplicated.
		static std::string huffmanEncode(const std::string &in);

		private:
		// Dynamic table (RFC 7541 §2.3.2): newest entry at front
		struct DynEntry { std::string name; std::string value; };

src/http.cpp

+46 −64

Changes for src/http.cpp: 46 added lines, 64 removed lines.
Original line number	Diff line number	Diff line
		@@ -564,17 +564,16 @@ std::vector<char> libhttppp::HttpClient::_h1ReadResponse(const std::string &labe
		if (header_end != std::string::npos)
		break;

		netplus::buffer buf(CHUNKSIZE);
		// Honor setTimeout() (_recvTimeoutSec) instead of a fixed 10s: a slow
		// backend (cold model load/swap) can take much longer than 10s to
		// send back even the first byte of the response.
		size_t n = _recvBlocking(buf);
		size_t n = _recvBlocking(_recvScratch);
		if (n == 0) {
		netplus::NetException ne;
		ne[netplus::NetException::Error] << "HTTP " << label << ": EOF while reading header";
		throw ne;
		}
		raw.insert(raw.end(), buf.data.buf, buf.data.buf + n);
		raw.insert(raw.end(), _recvScratch.data.buf, _recvScratch.data.buf + n);
		}

		// 2) Parse header
		@@ -648,15 +647,14 @@ std::vector<char> libhttppp::HttpClient::_h1ReadResponse(const std::string &labe
		remaining -= can;

		while (remaining > 0) {
		netplus::buffer buf(CHUNKSIZE);
		size_t n = _recvBlocking(buf);
		size_t n = _recvBlocking(_recvScratch);
		if (n == 0) {
		netplus::NetException ne;
		ne[netplus::NetException::Error] << "HTTP " << label << ": EOF before Content-Length complete";
		throw ne;
		}
		size_t take = (std::min)(remaining, n);
		ret.insert(ret.end(), buf.data.buf, buf.data.buf + take);
		ret.insert(ret.end(), _recvScratch.data.buf, _recvScratch.data.buf + take);
		remaining -= take;
		}
		return ret;
		@@ -668,14 +666,13 @@ std::vector<char> libhttppp::HttpClient::_h1ReadResponse(const std::string &labe

		auto ensure_bytes = [&](size_t need) {
		while (raw.size() - pos < need) {
		netplus::buffer buf(CHUNKSIZE);
		size_t n = _recvBlocking(buf);
		size_t n = _recvBlocking(_recvScratch);
		if (n == 0) {
		netplus::NetException ne;
		ne[netplus::NetException::Error] << "HTTP " << label << ": EOF while reading chunked body";
		throw ne;
		}
		raw.insert(raw.end(), buf.data.buf, buf.data.buf + n);
		raw.insert(raw.end(), _recvScratch.data.buf, _recvScratch.data.buf + n);
		}
		};

		@@ -689,14 +686,13 @@ std::vector<char> libhttppp::HttpClient::_h1ReadResponse(const std::string &labe
		return line;
		}
		}
		netplus::buffer buf(CHUNKSIZE);
		size_t n = _recvBlocking(buf);
		size_t n = _recvBlocking(_recvScratch);
		if (n == 0) {
		netplus::NetException ne;
		ne[netplus::NetException::Error] << "HTTP " << label << ": EOF while reading chunk size line";
		throw ne;
		}
		raw.insert(raw.end(), buf.data.buf, buf.data.buf + n);
		raw.insert(raw.end(), _recvScratch.data.buf, _recvScratch.data.buf + n);
		}
		};

		@@ -749,15 +745,14 @@ std::vector<char> libhttppp::HttpClient::_h1ReadResponse(const std::string &labe
		ret.insert(ret.end(), raw.begin() + (ptrdiff_t)pos, raw.end());
		}
		for (;;) {
		netplus::buffer buf(CHUNKSIZE);
		size_t n = 0;
		try {
		n = _recvBlocking(buf);
		n = _recvBlocking(_recvScratch);
		} catch (...) {
		throw;
		}
		if (n == 0) break;
		ret.insert(ret.end(), buf.data.buf, buf.data.buf + n);
		ret.insert(ret.end(), _recvScratch.data.buf, _recvScratch.data.buf + n);
		}
		}

		@@ -1044,16 +1039,15 @@ libhttppp::HttpResponse libhttppp::HttpClient::GetStream(libhttppp::HttpRequest

		auto h2_ensure_bytes = [&](size_t need) {
		while (_streamH2Raw.size() - h2_off < need) {
		netplus::buffer buf(CHUNKSIZE);
		size_t n = _recvBlocking(buf, 15);
		size_t n = _recvBlocking(_recvScratch, 15);
		if (n == 0) {
		HTTPException he;
		he[HTTPException::Error] << "HTTP/2 GetStream: EOF while reading frames";
		throw he;
		}
		_streamH2Raw.insert(_streamH2Raw.end(),
		reinterpret_cast<uint8_t*>(buf.data.buf),
		reinterpret_cast<uint8_t*>(buf.data.buf) + n);
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf),
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf) + n);
		}
		};

		@@ -1222,14 +1216,13 @@ libhttppp::HttpResponse libhttppp::HttpClient::GetStream(libhttppp::HttpRequest
		if (remaining_sec <= 0)
		remaining_sec = 1;

		netplus::buffer buf(CHUNKSIZE);
		size_t n = _recvBlocking(buf, remaining_sec);
		size_t n = _recvBlocking(_recvScratch, remaining_sec);
		if (n == 0) {
		netplus::NetException ne;
		ne[netplus::NetException::Error] << "HTTP GetStream: EOF while reading header";
		throw ne;
		}
		raw.insert(raw.end(), buf.data.buf, buf.data.buf + n);
		raw.insert(raw.end(), _recvScratch.data.buf, _recvScratch.data.buf + n);
		}

		// Parse header
		@@ -1399,14 +1392,13 @@ libhttppp::HttpResponse libhttppp::HttpClient::PostStream(libhttppp::HttpRequest
		if (remaining_sec <= 0)
		remaining_sec = 1;

		netplus::buffer buf(CHUNKSIZE);
		size_t n = _recvBlocking(buf, remaining_sec);
		size_t n = _recvBlocking(_recvScratch, remaining_sec);
		if (n == 0) {
		netplus::NetException ne;
		ne[netplus::NetException::Error] << "HTTP PostStream: EOF while reading header";
		throw ne;
		}
		raw.insert(raw.end(), buf.data.buf, buf.data.buf + n);
		raw.insert(raw.end(), _recvScratch.data.buf, _recvScratch.data.buf + n);
		}

		libhttppp::HttpResponse res;
		@@ -1493,8 +1485,7 @@ size_t libhttppp::HttpClient::readBodyChunk(char *buf, size_t bufsize) {

		// Helper: receive more data into internal buffer
		auto recvMore = [&]() -> size_t {
		netplus::buffer nb(CHUNKSIZE);
		size_t n = _recvBlocking(nb);
		size_t n = _recvBlocking(_recvScratch);
		if (n > 0) {
		// Compact only when consumed portion exceeds half the buffer
		// to amortize the O(n) memmove cost
		@@ -1503,7 +1494,7 @@ size_t libhttppp::HttpClient::readBodyChunk(char *buf, size_t bufsize) {
		_streamBuf.begin() + (ptrdiff_t)_streamBufPos);
		_streamBufPos = 0;
		}
		_streamBuf.insert(_streamBuf.end(), nb.data.buf, nb.data.buf + n);
		_streamBuf.insert(_streamBuf.end(), _recvScratch.data.buf, _recvScratch.data.buf + n);
		}
		return n;
		};
		@@ -1527,19 +1518,18 @@ size_t libhttppp::HttpClient::readBodyChunk(char *buf, size_t bufsize) {
		}

		// Read from socket
		netplus::buffer nb(CHUNKSIZE);
		size_t n = _recvBlocking(nb);
		size_t n = _recvBlocking(_recvScratch);
		if (n == 0) {
		_streamMode = STREAM_NONE;
		return 0;
		}
		size_t take = (std::min)({n, want, bufsize});
		std::memcpy(buf, nb.data.buf, take);
		std::memcpy(buf, _recvScratch.data.buf, take);
		_streamRemaining -= take;

		// Save excess data for next call
		if (take < n) {
		_streamBuf.assign(nb.data.buf + take, nb.data.buf + n);
		_streamBuf.assign(_recvScratch.data.buf + take, _recvScratch.data.buf + n);
		_streamBufPos = 0;
		}

		@@ -1593,18 +1583,17 @@ size_t libhttppp::HttpClient::readBodyChunk(char *buf, size_t bufsize) {
		return got;
		}
		// Read from socket directly
		netplus::buffer nb(CHUNKSIZE);
		size_t n = _recvBlocking(nb);
		size_t n = _recvBlocking(_recvScratch);
		if (n == 0) {
		_streamChunkDone = true;
		_streamMode = STREAM_NONE;
		return 0;
		}
		size_t take = (std::min)({n, want, bufsize});
		std::memcpy(buf, nb.data.buf, take);
		std::memcpy(buf, _recvScratch.data.buf, take);
		_streamChunkRemaining -= take;
		if (take < n) {
		_streamBuf.assign(nb.data.buf + take, nb.data.buf + n);
		_streamBuf.assign(_recvScratch.data.buf + take, _recvScratch.data.buf + n);
		_streamBufPos = 0;
		}
		if (_streamChunkRemaining == 0) {
		@@ -1660,10 +1649,9 @@ size_t libhttppp::HttpClient::readBodyChunk(char *buf, size_t bufsize) {
		if (got > 0) return got;

		// Read from socket; EOF signals end of body
		netplus::buffer nb(CHUNKSIZE);
		size_t n = 0;
		try {
		n = _recvBlocking(nb);
		n = _recvBlocking(_recvScratch);
		} catch (...) {
		_streamMode = STREAM_NONE;
		return 0;
		@@ -1673,9 +1661,9 @@ size_t libhttppp::HttpClient::readBodyChunk(char *buf, size_t bufsize) {
		return 0;
		}
		size_t take = (std::min)(n, bufsize);
		std::memcpy(buf, nb.data.buf, take);
		std::memcpy(buf, _recvScratch.data.buf, take);
		if (take < n) {
		_streamBuf.assign(nb.data.buf + take, nb.data.buf + n);
		_streamBuf.assign(_recvScratch.data.buf + take, _recvScratch.data.buf + n);
		_streamBufPos = 0;
		}
		return take;
		@@ -1697,15 +1685,14 @@ size_t libhttppp::HttpClient::readBodyChunk(char *buf, size_t bufsize) {

		auto h2_ensure_bytes = [&](size_t need) {
		while (_streamH2Raw.size() - h2_off < need) {
		netplus::buffer fbuf(CHUNKSIZE);
		size_t n = _recvBlocking(fbuf);
		size_t n = _recvBlocking(_recvScratch);
		if (n == 0) {
		_streamH2EndStream = true;
		return;
		}
		_streamH2Raw.insert(_streamH2Raw.end(),
		reinterpret_cast<uint8_t*>(fbuf.data.buf),
		reinterpret_cast<uint8_t*>(fbuf.data.buf) + n);
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf),
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf) + n);
		}
		};

		@@ -2003,17 +1990,16 @@ size_t libhttppp::HttpClient::readBodyChunkNonBlocking(char *buf, size_t bufsize
		}

		// Non-blocking read from socket
		netplus::buffer nb(CHUNKSIZE);
		size_t n = _recvNonBlocking(nb);
		size_t n = _recvNonBlocking(_recvScratch);
		if (n == 0)
		return 0; // No data yet — caller should retry later

		size_t take = (std::min)({n, want, bufsize});
		std::memcpy(buf, nb.data.buf, take);
		std::memcpy(buf, _recvScratch.data.buf, take);
		_streamRemaining -= take;

		if (take < n) {
		_streamBuf.assign(nb.data.buf + take, nb.data.buf + n);
		_streamBuf.assign(_recvScratch.data.buf + take, _recvScratch.data.buf + n);
		_streamBufPos = 0;
		}

		@@ -2175,12 +2161,11 @@ size_t libhttppp::HttpClient::readBodyChunkNonBlocking(char *buf, size_t bufsize

		// Non-blocking read from socket — always try, append to _streamH2Raw
		{
		netplus::buffer nbuf(CHUNKSIZE);
		size_t n = _recvNonBlocking(nbuf);
		size_t n = _recvNonBlocking(_recvScratch);
		if (n > 0) {
		_streamH2Raw.insert(_streamH2Raw.end(),
		reinterpret_cast<uint8_t*>(nbuf.data.buf),
		reinterpret_cast<uint8_t*>(nbuf.data.buf) + n);
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf),
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf) + n);
		}
		}

		@@ -2594,13 +2579,12 @@ const std::vector<char> libhttppp::HttpClient::_h2Request(
		// Try a non-blocking read to pick up the server's SETTINGS
		// (which usually arrives right after TLS handshake).
		{
		netplus::buffer nbuf(CHUNKSIZE);
		size_t n = 0;
		try { n = _cltsock->recvData(nbuf, 0); } catch (...) {}
		try { n = _cltsock->recvData(_recvScratch, 0); } catch (...) {}
		if (n > 0) {
		raw.insert(raw.end(),
		reinterpret_cast<uint8_t*>(nbuf.data.buf),
		reinterpret_cast<uint8_t*>(nbuf.data.buf) + n);
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf),
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf) + n);
		process_fc_frames();
		}
		}
		@@ -2611,16 +2595,15 @@ const std::vector<char> libhttppp::HttpClient::_h2Request(

		// If window exhausted, block-read for WINDOW_UPDATE
		while (allowed <= 0) {
		netplus::buffer nbuf(CHUNKSIZE);
		size_t n = recv_blocking(nbuf);
		size_t n = recv_blocking(_recvScratch);
		if (n == 0) {
		HTTPException he;
		he[HTTPException::Error] << "HTTP/2: EOF waiting for WINDOW_UPDATE";
		throw he;
		}
		raw.insert(raw.end(),
		reinterpret_cast<uint8_t*>(nbuf.data.buf),
		reinterpret_cast<uint8_t*>(nbuf.data.buf) + n);
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf),
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf) + n);
		process_fc_frames();
		allowed = std::min(peer_conn_window, peer_stream_window);
		}
		@@ -2649,16 +2632,15 @@ const std::vector<char> libhttppp::HttpClient::_h2Request(
		// Read enough data into raw buffer
		auto ensure_bytes = [&](size_t need) {
		while (raw.size() < need) {
		netplus::buffer buf(CHUNKSIZE);
		size_t n = recv_blocking(buf);
		size_t n = recv_blocking(_recvScratch);
		if (n == 0) {
		HTTPException he;
		he[HTTPException::Error] << "HTTP/2: EOF while reading frames";
		throw he;
		}
		raw.insert(raw.end(),
		reinterpret_cast<uint8_t*>(buf.data.buf),
		reinterpret_cast<uint8_t*>(buf.data.buf) + n);
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf),
		reinterpret_cast<uint8_t*>(_recvScratch.data.buf) + n);
		}
		};

src/http.h

+10 −0

Changes for src/http.h: 10 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -226,6 +226,16 @@ namespace libhttppp {
		std::unique_ptr<netplus::socket> _cltsock;
		netplus::socketwait _sw;
		netplus::x509cert _cert;
		// Reused across every recv() call in this object's lifetime instead of
		// constructing a fresh netplus::buffer per read: the Linux buffer ctor
		// value-initializes (zero-fills) the whole CHUNKSIZE (64KB) allocation,
		// so a fresh one per network read cost an alloc+memset+free even when
		// only a few KB actually arrived. Safe to share across all call sites
		// since each one copies out exactly the bytes recvData() reports before
		// doing anything else with the buffer (nothing relies on unwritten
		// bytes being zero, and HttpClient is not used from multiple threads
		// concurrently -- _cltsock/_streamMode etc. already assume that).
		netplus::buffer _recvScratch{CHUNKSIZE};
		int _recvTimeoutSec = 60;
		int _sendTimeoutSec = 30;
		int _vers = 2; // HTTP version preference (0=h1 only, 1=h1+h2, 2=h2 preferred, 3=h3, 4=internal h2-only probe)

src/qpack.cpp

+31 −3

Changes for src/qpack.cpp: 31 added lines, 3 removed lines.
Original line number	Diff line number	Diff line
		@@ -241,11 +241,20 @@ void Encoder::encodeInt(std::vector<uint8_t> &out, uint8_t prefix_byte,
		}
		}

		// Encode a QPACK string literal: H=0 (no Huffman), 7-bit prefix length
		// Encode a QPACK string literal (RFC 9204 §4.1.2), choosing whichever of raw
		// vs. Huffman-encoded is shorter -- same table RFC 7541 Appendix B defines,
		// which RFC 9204 reuses verbatim, so this shares hpack::Decoder's encoder
		// rather than duplicating the bit-packing logic.
		void Encoder::encodeString(std::vector<uint8_t> &out, const std::string &s) {
		std::string huff = hpack::Decoder::huffmanEncode(s);
		if (huff.size() < s.size()) {
		encodeInt(out, 0x80, 7, huff.size());
		out.insert(out.end(), huff.begin(), huff.end());
		} else {
		encodeInt(out, 0x00, 7, s.size());
		out.insert(out.end(), s.begin(), s.end());
		}
		}

		// Indexed Field Line referencing static table (RFC 9204 §4.5.2)
		// \| 1 \| T=1 \| Index (6+) \|
		@@ -277,6 +286,25 @@ void Encoder::encodeLiteralName(std::vector<uint8_t> &out,
		encodeString(out, value);
		}

		int Encoder::findStaticNameIndex(const std::string &name) {
		for (size_t i = 0; i < Decoder::STATIC_TABLE_SIZE; ++i) {
		if (name == Decoder::STATIC_TABLE[i].name) return static_cast<int>(i);
		}
		return -1;
		}

		// Emits a name-referenced literal when the name is in the static table (RFC
		// 9204 §4.5.4), otherwise falls back to a fully literal name+value (§4.5.6).
		void Encoder::encodeExtraHeader(std::vector<uint8_t> &out, const std::string &name,
		const std::string &value) {
		int idx = findStaticNameIndex(name);
		if (idx >= 0) {
		encodeLiteralStaticNameRef(out, static_cast<uint64_t>(idx), value);
		} else {
		encodeLiteralName(out, name, value);
		}
		}

		std::vector<uint8_t> Encoder::encodeResponseHeaders(uint16_t status_code,
		const std::string &content_type,
		size_t content_length,
		@@ -331,7 +359,7 @@ std::vector<uint8_t> Encoder::encodeResponseHeaders(uint16_t status_code,
		if (h.name.empty()) {
		continue;
		}
		encodeLiteralName(out, h.name, h.value);
		encodeExtraHeader(out, h.name, h.value);
		}

		return out;
		@@ -385,7 +413,7 @@ std::vector<uint8_t> Encoder::encodeRequestHeaders(const std::string &method,
		if (h.name.empty()) {
		continue;
		}
		encodeLiteralName(out, h.name, h.value);
		encodeExtraHeader(out, h.name, h.value);
		}

		return out;