Loading src/quic.cpp +34 −26 Original line number Diff line number Diff line Loading @@ -270,16 +270,24 @@ quic::quic(const std::string& addr, int port, int maxconnections, int sockopts) _last_activity = std::chrono::steady_clock::now(); // Enlarge the receive buffer so high-throughput clients don't // overrun the kernel queue between epoll_wait cycles. 16 MB keeps // up with ~13000 in-flight 1200-byte QUIC datagrams -- confirmed via // /proc/net/snmp's Udp:RcvbufErrors counter that the previous 4 MB // default was overflowing (and silently dropping ~2% of packets) on a // single-threaded loopback bulk transfer at ~180k pkt/s; the kernel // clamps this to net.core.rmem_max/wmem_max regardless, so requesting // more here is always safe -- it just won't help until that sysctl // ceiling is also raised. int rcvbuf = 16 * 1024 * 1024; int sndbuf = 16 * 1024 * 1024; // overrun the kernel queue between epoll_wait cycles. 4 MB keeps // up with ~3000 in-flight 1200-byte QUIC datagrams. // // Tried 16 MB (2026-08-15): confirmed via /proc/net/snmp's // Udp:RcvbufErrors that 4 MB was overflowing under a single-threaded // loopback bulk transfer at ~180k pkt/s (~2% real packet loss). Raising // to 16 MB (with net.core.rmem_max/wmem_max also raised to allow it) // did eliminate that loss entirely, but removing the loss let cwnd grow // unchecked into the larger buffer instead of being throttled by it — // RTT went from ~23ms to ~98ms (classic bufferbloat) for only ~10-15% // more throughput, because the real ceiling was never buffer size, it's // this thread's fixed packet-processing rate. Reverted to 4 MB: not // worth quadrupling latency for a partial throughput gain. See // [[project_libnetplus_tls_tcp_throughput_optimization]] memory entry; // the real fix needs to raise that processing-rate ceiling itself // (e.g. parallelizing per-packet decrypt), not the buffer in front of it. int rcvbuf = 4 * 1024 * 1024; int sndbuf = 4 * 1024 * 1024; #ifdef _WIN32 const char* rcvopt = reinterpret_cast<const char*>(&rcvbuf); Loading Loading @@ -338,12 +346,12 @@ quic::quic(const std::map<std::string, ssl::CertificateBundle>& certs, _last_activity = std::chrono::steady_clock::now(); // Enlarge the receive/send buffers so high-throughput clients don't // overrun the kernel queue between epoll_wait cycles. 16 MB keeps up // with ~13000 in-flight 1200-byte QUIC datagrams -- see the matching // comment in quic's default constructor for why this was raised from // 4 MB (confirmed real kernel-level drops via /proc/net/snmp). int rcvbuf = 16 * 1024 * 1024; int sndbuf = 16 * 1024 * 1024; // overrun the kernel queue between epoll_wait cycles. 4 MB keeps // up with ~3000 in-flight 1200-byte QUIC datagrams. See the matching // comment in quic's default constructor for the 16 MB experiment this // was bumped to and then reverted from (bufferbloat tradeoff). int rcvbuf = 4 * 1024 * 1024; int sndbuf = 4 * 1024 * 1024; #ifdef _WIN32 const char* rcvopt = reinterpret_cast<const char*>(&rcvbuf); Loading Loading @@ -3898,16 +3906,16 @@ void quic::connect(const std::string& addr, int port, bool nonblock) { // source (processFrame()'s ack_eliciting out-param), so the buffer // increase is safe to re-apply. // // Bumped from 4 MB to 16 MB: even at 4 MB, a single-threaded loopback // bulk transfer at ~180k pkt/s still measurably overflowed this (same // RcvbufErrors counter, ~2% of packets dropped per run) because the // kernel's per-datagram accounting overhead means the *usable* payload // capacity is well under the raw byte count requested. The kernel // clamps this to net.core.rmem_max/wmem_max regardless (4 MB on a // stock/unconfigured box), so asking for more here is always safe -- // it simply has no effect until that sysctl ceiling is also raised. int rcvbuf = 16 * 1024 * 1024; int sndbuf = 16 * 1024 * 1024; // Tried 16 MB (2026-08-15), with net.core.rmem_max/wmem_max also raised // to allow it: did eliminate the RcvbufErrors drops entirely, but // removing that loss let cwnd grow unchecked instead of being throttled // by it -- RTT went from ~23ms to ~98ms (bufferbloat) for only ~10-15% // more throughput. Reverted back to 4 MB: the real ceiling here is this // thread's fixed packet-processing rate, not the buffer size, so // quadrupling latency wasn't worth the partial gain. See // [[project_libnetplus_tls_tcp_throughput_optimization]] memory entry. int rcvbuf = 4 * 1024 * 1024; int sndbuf = 4 * 1024 * 1024; #ifdef _WIN32 const char* rcvopt = reinterpret_cast<const char*>(&rcvbuf); const char* sndopt = reinterpret_cast<const char*>(&sndbuf); Loading Loading
src/quic.cpp +34 −26 Original line number Diff line number Diff line Loading @@ -270,16 +270,24 @@ quic::quic(const std::string& addr, int port, int maxconnections, int sockopts) _last_activity = std::chrono::steady_clock::now(); // Enlarge the receive buffer so high-throughput clients don't // overrun the kernel queue between epoll_wait cycles. 16 MB keeps // up with ~13000 in-flight 1200-byte QUIC datagrams -- confirmed via // /proc/net/snmp's Udp:RcvbufErrors counter that the previous 4 MB // default was overflowing (and silently dropping ~2% of packets) on a // single-threaded loopback bulk transfer at ~180k pkt/s; the kernel // clamps this to net.core.rmem_max/wmem_max regardless, so requesting // more here is always safe -- it just won't help until that sysctl // ceiling is also raised. int rcvbuf = 16 * 1024 * 1024; int sndbuf = 16 * 1024 * 1024; // overrun the kernel queue between epoll_wait cycles. 4 MB keeps // up with ~3000 in-flight 1200-byte QUIC datagrams. // // Tried 16 MB (2026-08-15): confirmed via /proc/net/snmp's // Udp:RcvbufErrors that 4 MB was overflowing under a single-threaded // loopback bulk transfer at ~180k pkt/s (~2% real packet loss). Raising // to 16 MB (with net.core.rmem_max/wmem_max also raised to allow it) // did eliminate that loss entirely, but removing the loss let cwnd grow // unchecked into the larger buffer instead of being throttled by it — // RTT went from ~23ms to ~98ms (classic bufferbloat) for only ~10-15% // more throughput, because the real ceiling was never buffer size, it's // this thread's fixed packet-processing rate. Reverted to 4 MB: not // worth quadrupling latency for a partial throughput gain. See // [[project_libnetplus_tls_tcp_throughput_optimization]] memory entry; // the real fix needs to raise that processing-rate ceiling itself // (e.g. parallelizing per-packet decrypt), not the buffer in front of it. int rcvbuf = 4 * 1024 * 1024; int sndbuf = 4 * 1024 * 1024; #ifdef _WIN32 const char* rcvopt = reinterpret_cast<const char*>(&rcvbuf); Loading Loading @@ -338,12 +346,12 @@ quic::quic(const std::map<std::string, ssl::CertificateBundle>& certs, _last_activity = std::chrono::steady_clock::now(); // Enlarge the receive/send buffers so high-throughput clients don't // overrun the kernel queue between epoll_wait cycles. 16 MB keeps up // with ~13000 in-flight 1200-byte QUIC datagrams -- see the matching // comment in quic's default constructor for why this was raised from // 4 MB (confirmed real kernel-level drops via /proc/net/snmp). int rcvbuf = 16 * 1024 * 1024; int sndbuf = 16 * 1024 * 1024; // overrun the kernel queue between epoll_wait cycles. 4 MB keeps // up with ~3000 in-flight 1200-byte QUIC datagrams. See the matching // comment in quic's default constructor for the 16 MB experiment this // was bumped to and then reverted from (bufferbloat tradeoff). int rcvbuf = 4 * 1024 * 1024; int sndbuf = 4 * 1024 * 1024; #ifdef _WIN32 const char* rcvopt = reinterpret_cast<const char*>(&rcvbuf); Loading Loading @@ -3898,16 +3906,16 @@ void quic::connect(const std::string& addr, int port, bool nonblock) { // source (processFrame()'s ack_eliciting out-param), so the buffer // increase is safe to re-apply. // // Bumped from 4 MB to 16 MB: even at 4 MB, a single-threaded loopback // bulk transfer at ~180k pkt/s still measurably overflowed this (same // RcvbufErrors counter, ~2% of packets dropped per run) because the // kernel's per-datagram accounting overhead means the *usable* payload // capacity is well under the raw byte count requested. The kernel // clamps this to net.core.rmem_max/wmem_max regardless (4 MB on a // stock/unconfigured box), so asking for more here is always safe -- // it simply has no effect until that sysctl ceiling is also raised. int rcvbuf = 16 * 1024 * 1024; int sndbuf = 16 * 1024 * 1024; // Tried 16 MB (2026-08-15), with net.core.rmem_max/wmem_max also raised // to allow it: did eliminate the RcvbufErrors drops entirely, but // removing that loss let cwnd grow unchecked instead of being throttled // by it -- RTT went from ~23ms to ~98ms (bufferbloat) for only ~10-15% // more throughput. Reverted back to 4 MB: the real ceiling here is this // thread's fixed packet-processing rate, not the buffer size, so // quadrupling latency wasn't worth the partial gain. See // [[project_libnetplus_tls_tcp_throughput_optimization]] memory entry. int rcvbuf = 4 * 1024 * 1024; int sndbuf = 4 * 1024 * 1024; #ifdef _WIN32 const char* rcvopt = reinterpret_cast<const char*>(&rcvbuf); const char* sndopt = reinterpret_cast<const char*>(&sndbuf); Loading