Commit 8f9e9caf authored by jan.koester's avatar jan.koester
Browse files

test

parent a5024b80
Loading
Loading
Loading
Loading
+4 −4
Changes for config/proxyplus.yaml: 4 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -87,10 +87,10 @@ proxyplus:
    # long-lived streaming responses (a live camera feed, a large file transfer) or a genuinely
    # stuck backend call tying up every worker at once; a healthy pool never gets close to this.
    h2_dispatch_queue_max: 256
    # Largest request body (upload, git push, ...) accepted per request, in MiB. The body is
    # buffered in memory before being forwarded upstream, so this is also the per-upload
    # memory cost. 0 = libhttppp's compiled-in default (~4 MB).
    max_request_body_mb: 512
    # Largest request body (upload, git push, ...) relayed per request, in MiB; larger ones get
    # 413. Bodies are streamed to the upstream, not buffered, so this is a policy cap, not a
    # memory one. 0 = no limit.
    max_request_body_mb: 0

  pid_file: "/run/proxyplus.pid"
  recovery_mode: false
+1 −1
Changes for src/config/BootstrapConfig.cpp: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -243,7 +243,7 @@ void BootstrapConfig::loadAndValidate() {
    cfg.h1DispatchQueueMax =
        readInt(conf, "/proxyplus/process/h1_dispatch_queue_max", 256);
    cfg.maxRequestBodyMb =
        readInt(conf, "/proxyplus/process/max_request_body_mb", 512);
        readInt(conf, "/proxyplus/process/max_request_body_mb", 0);
    if (cfg.maxRequestBodyMb < 0)
        throw ConfigValidationError("process.max_request_body_mb must be >= 0");
    cfg.pidFile = readValue(conf, "/proxyplus/pid_file", "");
+7 −8
Changes for src/config/BootstrapConfig.h: 7 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -143,14 +143,13 @@ struct AppConfig {
    // default for the same reason h2DispatchQueueMax is: degrade to a fast 503 out of the box.
    int h1DispatchQueueMax = 256;

    // Per-connection request-body ceiling, applied to every accepted HTTP connection via
    // libhttppp::HttpRequest::setMaxUploadSize() (see HttpProxyController::CreateConnection).
    // libhttppp's own compiled-in default (DEFAULT_UPLOADSIZE, ~4 MB) is far too small for a
    // reverse proxy: an H2 request body over it gets RST_STREAM ENHANCE_YOUR_CALM, which broke
    // a 5 MiB `git push` to git.tuxist.de. The body is still fully buffered in memory before
    // being forwarded upstream, so this is also the per-upload memory cost. In MiB;
    // 0 = keep libhttppp's compiled-in default.
    int maxRequestBodyMb = 512;
    // Largest request body proxyplus relays, in MiB -- a policy cap only: bodies are streamed
    // to the upstream as they arrive (see HttpProxyController::shouldStreamRequestBody), so
    // their size no longer costs memory. Over it: 413, up front for a declared Content-Length,
    // mid-upload otherwise. 0 = no limit -- the default, since libhttppp's old buffered-body
    // ceiling (DEFAULT_UPLOADSIZE, ~4 MB) is exactly what broke a 5 MiB `git push` to
    // git.tuxist.de (H2 RST_STREAM ENHANCE_YOUR_CALM).
    int maxRequestBodyMb = 0;

    RecoveryMode recoveryMode = RecoveryMode::Normal;
    MaintenanceMode maintenanceMode = MaintenanceMode::Off;
+69 −7
Changes for src/http/HttpProxyServer.cpp: 69 added lines, 7 removed lines.
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@
#include <algorithm>
#include <cctype>
#include <iostream>
#include <limits>
#include <set>
#include <stdexcept>
#include <thread>
@@ -159,10 +160,11 @@ bool HttpProxyController::shouldOffloadH2Dispatch(libhttppp::HttpRequest &, uint

void HttpProxyController::CreateConnection(std::shared_ptr<netplus::con> &res) {
    libhttppp::HttpEvent::CreateConnection(res);
    if (_maxRequestBodyMb > 0) {
    // Only reached by bodies libhttppp still buffers itself (an H2 stream that couldn't be
    // streamed); streamed ones are capped in RequestEvent. 0 = no limit there either.
    static_cast<libhttppp::HttpRequest &>(*res).setMaxUploadSize(
            static_cast<size_t>(_maxRequestBodyMb) * 1024 * 1024);
    }
        _maxRequestBodyMb > 0 ? static_cast<size_t>(_maxRequestBodyMb) * 1024 * 1024
                              : std::numeric_limits<size_t>::max());
}

bool HttpProxyController::tryServeAcmeChallenge(libhttppp::HttpRequest &curreq,
@@ -237,6 +239,30 @@ std::vector<char> drainStreamedBody(libhttppp::HttpClient &client) {
    }
    return body;
}

// Raised from the request-body source handed to HttpClient::RequestStream() -- separates a
// failure on the *client's* side of a streamed upload (it disconnected, sent broken framing,
// or exceeded max_request_body_mb) from a genuine upstream failure, which is the only kind
// that should count against the upstream's passive health.
struct ClientBodyError : std::runtime_error {
    ClientBodyError(const std::string &msg, bool tooLarge)
        : std::runtime_error(msg), tooLarge(tooLarge) {}
    bool tooLarge;
};

// The client's declared body length, or -1 when it didn't declare one (chunked H1, or an
// H2/H3 request without content-length) -- RequestStream() then frames it accordingly.
int64_t declaredContentLength(libhttppp::HttpRequest &req) {
    if (req.isChunkedRequest()) return -1;
    auto *hd = req.getHeaderData("content-length");
    if (!hd || !hd->getfirstValue()) return -1;
    try {
        long long v = std::stoll(hd->getfirstValue()->getvalue());
        return v >= 0 ? v : -1;
    } catch (...) {
        return -1;
    }
}
} // namespace

namespace {
@@ -539,6 +565,16 @@ void HttpProxyController::RequestEvent(libhttppp::HttpRequest &curreq, const int
            return;
        }

        // Reject a declared-too-large upload before touching any upstream; a streamed body
        // without a declared length is capped as it's read (see bodySource below).
        const uint64_t bodyLimit =
            _maxRequestBodyMb > 0 ? static_cast<uint64_t>(_maxRequestBodyMb) << 20 : 0;
        const int64_t declaredLength = declaredContentLength(curreq);
        if (bodyLimit && declaredLength > 0 && static_cast<uint64_t>(declaredLength) > bodyLimit) {
            sendError(curreq, HTTP413, "request body exceeds the configured limit");
            return;
        }

        std::string upstreamBase;
        ScopedLbRelease lbGuard; // inert unless a pool actually picks a target below
        if (!matched.upstreamPool.empty()) {
@@ -571,6 +607,7 @@ void HttpProxyController::RequestEvent(libhttppp::HttpRequest &curreq, const int
        }

        bool upstreamOk = false;
        bool clientBodyFailed = false; // see ClientBodyError
        ScopedPoolRelease poolGuard; // inert unless acquire() below actually hands out a connection
        try {
            libhttppp::HttpUrl upstream(upstreamBase + forwardPath);
@@ -603,8 +640,9 @@ void HttpProxyController::RequestEvent(libhttppp::HttpRequest &curreq, const int
                "host", "connection", "content-length", "transfer-encoding",
                "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer",
                "upgrade",
                // The full request body is already buffered here; a forwarded
                // "Expect: 100-continue" only adds an interim 100 round-trip.
                // libhttppp answers the client's 100-continue itself once the body is first
                // read, and RequestStream() sends the body right behind the headers -- a
                // forwarded "Expect" would only add an interim 100 round-trip upstream.
                "expect",
            };
            for (auto *hd = curreq.getfirstHeaderData(); hd; hd = hd->nextHeaderData()) {
@@ -704,7 +742,26 @@ void HttpProxyController::RequestEvent(libhttppp::HttpRequest &curreq, const int
                upstreamOk = status < 500;
            };

            if (method == "POST" || method == "PUT" || method == "PATCH") {
            // A body the client is still sending (shouldStreamRequestBody() is on for every
            // request) goes upstream as it's read: memory per upload stays at one read buffer
            // instead of the whole body, whatever its size.
            uint64_t bodyReceived = 0;
            libhttppp::HttpClient::BodySource bodySource = [&](char *buf, size_t len) -> size_t {
                size_t n;
                try {
                    n = curreq.readBody(buf, len);
                } catch (const std::exception &e) {
                    throw ClientBodyError(std::string("client request body: ") + e.what(), false);
                }
                bodyReceived += n;
                if (bodyLimit && bodyReceived > bodyLimit)
                    throw ClientBodyError("request body exceeds the configured limit", true);
                return n;
            };

            if (curreq.hasStreamingBody() && method != "GET") {
                body = client.RequestStream(requestType, outreq, bodySource, declaredLength);
            } else if (method == "POST" || method == "PUT" || method == "PATCH") {
                outreq.setRequestType(method == "POST" ? POSTREQUEST
                                       : method == "PUT" ? PUTREQUEST
                                                          : PATCHREQUEST);
@@ -905,11 +962,16 @@ void HttpProxyController::RequestEvent(libhttppp::HttpRequest &curreq, const int
                relayBuffered(&streamResp);
            }
            if (method != "GET") relayBuffered(client.lastResponse());
        } catch (const ClientBodyError &e) {
            // Not the upstream's fault -- no passive-health penalty. RequestStream() already
            // reset the half-sent upstream connection, so it isn't pooled either.
            clientBodyFailed = true;
            sendError(curreq, e.tooLarge ? HTTP413 : HTTP400, e.what());
        } catch (const std::exception &e) {
            sendError(curreq, HTTP502, std::string("upstream error: ") + e.what());
        }

        lbGuard.ok = upstreamOk;
        lbGuard.ok = upstreamOk || clientBodyFailed;
        poolGuard.ok = upstreamOk;
    } catch (const libhttppp::HTTPException &e) {
        std::cerr << "[http] request handling failed: " << e.what() << std::endl;
+6 −0
Changes for src/http/HttpProxyServer.h: 6 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -85,6 +85,12 @@ public:
    // that fix, a backend exception left the stream queued forever with no response ever sent.
    bool shouldOffloadH2Dispatch(libhttppp::HttpRequest &, uint32_t) const override;

    // Every body is relayed to the upstream as it arrives (HttpClient::RequestStream() in
    // RequestEvent) instead of being buffered whole first -- a multi-hundred-MB upload or git
    // push costs one read buffer, not its own size. Route/auth/plugin checks in RequestEvent
    // all run before the first body byte is read, so a rejected upload is never read at all.
    bool shouldStreamRequestBody(libhttppp::HttpRequest &) const override { return true; }

protected:
    // Applies maxRequestBodyMb (see BootstrapConfig.h) to each newly accepted connection --
    // libhttppp enforces HttpRequest::getMaxUploadSize() on the connection-level request,
Loading