Loading src/posix/udp.cpp +28 −3 Original line number Diff line number Diff line Loading @@ -602,9 +602,26 @@ constexpr size_t kRecvCmsgBufSize = CMSG_SPACE(sizeof(uint16_t)); thread_local char g_recv_cmsg_bufs[kRecvMaxBatch * kRecvCmsgBufSize]; int recvBatchRaw(SOCKET sock, int batch, bool gro_enabled) { std::memset(g_recv_msgs, 0, sizeof(struct mmsghdr) * batch); std::memset(g_recv_peer_addrs, 0, sizeof(sockaddr_storage) * batch); // No blanket memset of g_recv_msgs/g_recv_peer_addrs here: profiling // (callgrind) showed these two zeroings alone cost ~36% of this // benchmark's entire instruction count, because they always zero the // full requested `batch` (up to 64 slots x sizeof(sockaddr_storage) // etc.) regardless of how many datagrams actually arrive -- which per // QUIC_PERF is typically just 1 on a CPU-bound receive loop. Every // field recvmmsg()/its callers actually depend on is either (a) set // unconditionally below for every slot before the syscall (msg_iov, // msg_iovlen, msg_name, msg_namelen), (b) a pure kernel-output field // (msg_len, msg_flags) that's only ever read for slots the kernel // just filled in and always overwrites, or (c) msg_control/ // msg_controllen, handled explicitly below since it's the one field // this thread_local array's cross-call/cross-socket reuse (see its // declaration comment) can leave genuinely stale. g_recv_peer_addrs // needs no reset at all: consumers (recvBatchAddr/recvBatchAddrViews) // only ever read slots [0, received) that the kernel just wrote, and // only ever interpret them via ss_family-gated accessors // (sockaddrLen(), serializePeerAddr()) or a memcpy bounded to the // family-correct length -- never a raw whole-struct read that could // observe stale padding from a previous call. for (int i = 0; i < batch; ++i) { g_recv_iovecs[i].iov_base = g_recv_flat_buf.data() + i * 65535; g_recv_iovecs[i].iov_len = 65535; Loading @@ -615,6 +632,14 @@ int recvBatchRaw(SOCKET sock, int batch, bool gro_enabled) { if (gro_enabled) { g_recv_msgs[i].msg_hdr.msg_control = &g_recv_cmsg_bufs[i * kRecvCmsgBufSize]; g_recv_msgs[i].msg_hdr.msg_controllen = kRecvCmsgBufSize; } else { // A prior call on this thread may have serviced a // GRO-enabled socket and left a valid msg_control/ // msg_controllen in this slot -- must be cleared, not just // left as whatever the last call set, or the kernel would // write ancillary data into a buffer this call never offered. g_recv_msgs[i].msg_hdr.msg_control = nullptr; g_recv_msgs[i].msg_hdr.msg_controllen = 0; } } Loading Loading
src/posix/udp.cpp +28 −3 Original line number Diff line number Diff line Loading @@ -602,9 +602,26 @@ constexpr size_t kRecvCmsgBufSize = CMSG_SPACE(sizeof(uint16_t)); thread_local char g_recv_cmsg_bufs[kRecvMaxBatch * kRecvCmsgBufSize]; int recvBatchRaw(SOCKET sock, int batch, bool gro_enabled) { std::memset(g_recv_msgs, 0, sizeof(struct mmsghdr) * batch); std::memset(g_recv_peer_addrs, 0, sizeof(sockaddr_storage) * batch); // No blanket memset of g_recv_msgs/g_recv_peer_addrs here: profiling // (callgrind) showed these two zeroings alone cost ~36% of this // benchmark's entire instruction count, because they always zero the // full requested `batch` (up to 64 slots x sizeof(sockaddr_storage) // etc.) regardless of how many datagrams actually arrive -- which per // QUIC_PERF is typically just 1 on a CPU-bound receive loop. Every // field recvmmsg()/its callers actually depend on is either (a) set // unconditionally below for every slot before the syscall (msg_iov, // msg_iovlen, msg_name, msg_namelen), (b) a pure kernel-output field // (msg_len, msg_flags) that's only ever read for slots the kernel // just filled in and always overwrites, or (c) msg_control/ // msg_controllen, handled explicitly below since it's the one field // this thread_local array's cross-call/cross-socket reuse (see its // declaration comment) can leave genuinely stale. g_recv_peer_addrs // needs no reset at all: consumers (recvBatchAddr/recvBatchAddrViews) // only ever read slots [0, received) that the kernel just wrote, and // only ever interpret them via ss_family-gated accessors // (sockaddrLen(), serializePeerAddr()) or a memcpy bounded to the // family-correct length -- never a raw whole-struct read that could // observe stale padding from a previous call. for (int i = 0; i < batch; ++i) { g_recv_iovecs[i].iov_base = g_recv_flat_buf.data() + i * 65535; g_recv_iovecs[i].iov_len = 65535; Loading @@ -615,6 +632,14 @@ int recvBatchRaw(SOCKET sock, int batch, bool gro_enabled) { if (gro_enabled) { g_recv_msgs[i].msg_hdr.msg_control = &g_recv_cmsg_bufs[i * kRecvCmsgBufSize]; g_recv_msgs[i].msg_hdr.msg_controllen = kRecvCmsgBufSize; } else { // A prior call on this thread may have serviced a // GRO-enabled socket and left a valid msg_control/ // msg_controllen in this slot -- must be cleared, not just // left as whatever the last call set, or the kernel would // write ancillary data into a buffer this call never offered. g_recv_msgs[i].msg_hdr.msg_control = nullptr; g_recv_msgs[i].msg_hdr.msg_controllen = 0; } } Loading