test (f5e00ade) · Commits · tuxist / proxyplus

config/proxyplus.yaml

+6 −0

Changes for config/proxyplus.yaml: 6 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -81,6 +81,12 @@ proxyplus:
		# every proxied request. Raise this for a legitimately slow-but-alive upstream (a
		# cold-starting backend, a long-running API call, ...) that needs more than 60s.
		upstream_request_timeout_seconds: 60
		# Caps how many H2 requests may queue waiting for a free offload worker (shared across every
		# vhost/route on the listener) before proxyplus answers 503 instead of queuing further. Only
		# matters once the offload pool is fully saturated for a sustained stretch -- e.g. several
		# long-lived streaming responses (a live camera feed, a large file transfer) or a genuinely
		# stuck backend call tying up every worker at once; a healthy pool never gets close to this.
		h2_dispatch_queue_max: 256

		pid_file: "/run/proxyplus.pid"
		recovery_mode: false

src/config/BootstrapConfig.cpp

+3 −0

Changes for src/config/BootstrapConfig.cpp: 3 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -53,6 +53,7 @@ const std::set<std::string> &knownLeafPatterns() {
		"/proxyplus/process/upstream_pool_max_idle_per_origin",
		"/proxyplus/process/upstream_pool_idle_timeout_seconds",
		"/proxyplus/process/upstream_request_timeout_seconds",
		"/proxyplus/process/h2_dispatch_queue_max",
		"/proxyplus/pid_file",
		"/proxyplus/recovery_mode",
		"/proxyplus/maintenance_mode",
		@@ -235,6 +236,8 @@ void BootstrapConfig::loadAndValidate() {
		readInt(conf, "/proxyplus/process/upstream_pool_idle_timeout_seconds", 60);
		cfg.upstreamRequestTimeoutSeconds =
		readInt(conf, "/proxyplus/process/upstream_request_timeout_seconds", 60);
		cfg.h2DispatchQueueMax =
		readInt(conf, "/proxyplus/process/h2_dispatch_queue_max", 256);
		cfg.pidFile = readValue(conf, "/proxyplus/pid_file", "");

		cfg.recoveryMode = readBool(conf, "/proxyplus/recovery_mode", false) ? RecoveryMode::Recover

src/config/BootstrapConfig.h

+14 −0

Changes for src/config/BootstrapConfig.h: 14 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -119,6 +119,20 @@ struct AppConfig {
		// so an unset config keeps today's behavior.
		int upstreamRequestTimeoutSeconds = 60;

		// Caps how many H2 streams may sit queued in the H2 offload pool (see
		// libhttppp::HttpEvent's h2DispatchQueueMax ctor param) waiting for a worker thread, across
		// every vhost/route sharing this listener. Root-caused live on 10.1.2.99: that pool used to
		// have an unbounded queue, so once its (CPU-count-sized) workers all got tied up -- e.g. a
		// burst of concurrently proxied slow/streaming responses -- every subsequent request on
		// every connection, including brand new ones, queued forever with no response and no way to
		// notice (looked identical to a total outage from the outside, even though the process was
		// still up and technically "working"). Positive by default so proxyplus degrades to a fast
		// 503 under that condition out of the box instead of requiring an opt-in; 256 is generous
		// headroom above what a legitimate traffic burst should ever need to queue -- if this is
		// ever actually being hit under normal load, the fix is more/faster workers or moving
		// long-lived streaming responses off this pool entirely, not raising this further.
		int h2DispatchQueueMax = 256;

		RecoveryMode recoveryMode = RecoveryMode::Normal;
		MaintenanceMode maintenanceMode = MaintenanceMode::Off;
		};

src/http/HttpProxyServer.cpp

+20 −9

Changes for src/http/HttpProxyServer.cpp: 20 added lines, 9 removed lines.
Original line number	Diff line number	Diff line
		@@ -117,7 +117,8 @@ HttpProxyController::HttpProxyController(std::vector<netplus::socket *> serverSo
		http::UpstreamConnectionPool &connectionPool,
		tls::AcmeChallengeStore *acmeChallenges,
		int idleTimeoutSeconds,
		int upstreamTimeoutSeconds)
		int upstreamTimeoutSeconds,
		int h2DispatchQueueMax)
		// H2 dispatch now runs through the same offload pool as H1 (see shouldOffloadH2Dispatch
		// below): both RequestEvent calls do real backend I/O, so both need off the shared
		// epoll/kqueue listener workers to avoid one bursty connection starving new TLS
		@@ -125,11 +126,15 @@ HttpProxyController::HttpProxyController(std::vector<netplus::socket *> serverSo
		// try/catch around RequestEvent (see httpd.cpp's _dispatchH2Stream) -- without it, a
		// backend exception left the stream queued forever with no HEADERS/DATA/RST_STREAM ever
		// sent. ALPN is left at HttpEvent's own default (prefer h2, fall back to http/1.1) below;
		// H3 stays independent via QUIC/Alt-Svc regardless.
		// H3 stays independent via QUIC/Alt-Svc regardless. h2DispatchQueueMax bounds that same
		// pool's queue (see httpd.h) so a saturated/stuck pool degrades to 503 instead of every
		// subsequent stream -- on every connection -- queuing forever with no way to notice; see
		// BootstrapConfig.h's h2DispatchQueueMax doc comment for the incident this fixes.
		: libhttppp::HttpEvent(serverSockets, /timeout=/1000,
		/h2OffloadThreads=/(std::max<size_t>)(4, std::thread::hardware_concurrency() * 2),
		idleTimeoutSeconds,
		/h1OffloadThreads=/(std::max<size_t>)(4, std::thread::hardware_concurrency() * 2))
		/h1OffloadThreads=/(std::max<size_t>)(4, std::thread::hardware_concurrency() * 2),
		h2DispatchQueueMax)
		, _listenerLabel(std::move(listenerLabel))
		, _tlsActive(tlsActive)
		, _configStore(configStore)
		@@ -138,7 +143,8 @@ HttpProxyController::HttpProxyController(std::vector<netplus::socket *> serverSo
		, _loadBalancer(loadBalancer)
		, _connectionPool(connectionPool)
		, _acmeChallenges(acmeChallenges)
		, _upstreamTimeoutSeconds(upstreamTimeoutSeconds) {
		, _upstreamTimeoutSeconds(upstreamTimeoutSeconds)
		, _h2DispatchQueueMax(h2DispatchQueueMax) {
		}

		bool HttpProxyController::shouldOffloadH2Dispatch(libhttppp::HttpRequest &, uint32_t) const {
		@@ -663,7 +669,8 @@ HttpProxyServer::HttpProxyServer(config::ListenerConfig listener, std::string ce
		http::UpstreamConnectionPool &connectionPool,
		tls::AcmeChallengeStore *acmeChallenges,
		int idleTimeoutSeconds,
		int upstreamTimeoutSeconds)
		int upstreamTimeoutSeconds,
		int h2DispatchQueueMax)
		: _listener(listener)
		, _tlsActive(!listener.tlsProfile.empty())
		, _httpd(std::make_unique<libhttppp::HttpD>(listener.address, listener.port, 512, certPath,
		@@ -675,7 +682,8 @@ HttpProxyServer::HttpProxyServer(config::ListenerConfig listener, std::string ce
		, _connectionPool(connectionPool)
		, _acmeChallenges(acmeChallenges)
		, _idleTimeoutSeconds(idleTimeoutSeconds)
		, _upstreamTimeoutSeconds(upstreamTimeoutSeconds) {
		, _upstreamTimeoutSeconds(upstreamTimeoutSeconds)
		, _h2DispatchQueueMax(h2DispatchQueueMax) {
		bindAndListen();
		}

		@@ -686,7 +694,8 @@ HttpProxyServer::HttpProxyServer(config::ListenerConfig listener,
		http::UpstreamConnectionPool &connectionPool,
		tls::AcmeChallengeStore *acmeChallenges,
		int idleTimeoutSeconds,
		int upstreamTimeoutSeconds)
		int upstreamTimeoutSeconds,
		int h2DispatchQueueMax)
		: _listener(listener)
		, _tlsActive(true)
		, _httpd(std::make_unique<libhttppp::HttpD>(listener.address, listener.port, 512, certsByHostname))
		@@ -697,7 +706,8 @@ HttpProxyServer::HttpProxyServer(config::ListenerConfig listener,
		, _connectionPool(connectionPool)
		, _acmeChallenges(acmeChallenges)
		, _idleTimeoutSeconds(idleTimeoutSeconds)
		, _upstreamTimeoutSeconds(upstreamTimeoutSeconds) {
		, _upstreamTimeoutSeconds(upstreamTimeoutSeconds)
		, _h2DispatchQueueMax(h2DispatchQueueMax) {
		bindAndListen();
		}

		@@ -717,7 +727,8 @@ void HttpProxyServer::run() {
		_listener.address + ":" + std::to_string(_listener.port),
		_tlsActive, _configStore, _authService, _pluginManager,
		_loadBalancer, _connectionPool, _acmeChallenges,
		_idleTimeoutSeconds, _upstreamTimeoutSeconds);
		_idleTimeoutSeconds, _upstreamTimeoutSeconds,
		_h2DispatchQueueMax);
		controller.runEventloop();
		}

src/http/HttpProxyServer.h

+15 −5

Changes for src/http/HttpProxyServer.h: 15 added lines, 5 removed lines.
Original line number	Diff line number	Diff line
		@@ -33,6 +33,10 @@ public:
		// UpstreamConnectionPool::acquire()'s timeoutSec) -- default 60 matches the value that used
		// to be hardcoded at the RequestEvent call site; main.cpp's real call sites pass
		// AppConfig::upstreamRequestTimeoutSeconds.
		// h2DispatchQueueMax is forwarded to libhttppp::HttpEvent's own h2DispatchQueueMax ctor
		// param (see httpd.h) -- caps how many H2 streams queue for a free offload worker before
		// proxyplus answers 503 instead of piling up behind a saturated (or fully stuck) pool;
		// main.cpp's real call sites pass AppConfig::h2DispatchQueueMax.
		HttpProxyController(std::vector<netplus::socket *> serverSockets,
		std::string listenerLabel,
		bool tlsActive,
		@@ -43,7 +47,8 @@ public:
		http::UpstreamConnectionPool &connectionPool,
		tls::AcmeChallengeStore *acmeChallenges = nullptr,
		int idleTimeoutSeconds = 0,
		int upstreamTimeoutSeconds = 60);
		int upstreamTimeoutSeconds = 60,
		int h2DispatchQueueMax = 256);

		void RequestEvent(libhttppp::HttpRequest &curreq, const int tid, ULONG_PTR args) override;

		@@ -105,6 +110,7 @@ private:
		http::UpstreamConnectionPool &_connectionPool;
		tls::AcmeChallengeStore *_acmeChallenges;
		int _upstreamTimeoutSeconds;
		int _h2DispatchQueueMax;
		};

		// Binds the listener socket(s) at construction time (see main.cpp / spec "Privilege
		@@ -114,8 +120,9 @@ private:
		// from a worker thread after privileges have been dropped.
		class HttpProxyServer {
		public:
		// idleTimeoutSeconds/upstreamTimeoutSeconds: see HttpProxyController's own doc comment
		// above -- threaded down into the controller it constructs internally in run().
		// idleTimeoutSeconds/upstreamTimeoutSeconds/h2DispatchQueueMax: see HttpProxyController's
		// own doc comment above -- threaded down into the controller it constructs internally in
		// run().
		HttpProxyServer(config::ListenerConfig listener,
		std::string certPath,
		std::string keyPath,
		@@ -126,7 +133,8 @@ public:
		http::UpstreamConnectionPool &connectionPool,
		tls::AcmeChallengeStore *acmeChallenges = nullptr,
		int idleTimeoutSeconds = 0,
		int upstreamTimeoutSeconds = 60);
		int upstreamTimeoutSeconds = 60,
		int h2DispatchQueueMax = 256);

		// SNI virtual hosting variant: certsByHostname carries one CertificateBundle per domain
		// sharing this listener (see ListenerConfig::tlsProfiles) instead of the single cert/key
		@@ -140,7 +148,8 @@ public:
		http::UpstreamConnectionPool &connectionPool,
		tls::AcmeChallengeStore *acmeChallenges = nullptr,
		int idleTimeoutSeconds = 0,
		int upstreamTimeoutSeconds = 60);
		int upstreamTimeoutSeconds = 60,
		int h2DispatchQueueMax = 256);

		void run();

		@@ -170,6 +179,7 @@ private:
		tls::AcmeChallengeStore *_acmeChallenges;
		int _idleTimeoutSeconds;
		int _upstreamTimeoutSeconds;
		int _h2DispatchQueueMax;
		};

		} // namespace proxyplus::httpproxy