diff --git a/README.md b/README.md index 4235b17f..e70f18d2 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,18 @@ result before returning. Listener shutdown still allows accepted connections to drain before releasing the socket. Treat this as an experimental option: measure playback quality and CPU under your workload before enabling it. +For two parallel writers, also pass `-udp-send-writers 2` (or +`relaynet.WithUDPSendWriters(2)`). This is opt-in and requires batching. +Destinations stay on a fixed writer; both Go handles share the source socket +and port. The total queue remains 256 writes, divided between the writers. +The default is one writer. With two writers an observer can be called +concurrently, and summed socket-call durations can exceed elapsed time. + +Subgroup fanout reuses empty inbox storage after each writer has finished. +The relay-wide cache retains at most 1024 queues, only for queue capacities up +to 256 objects; larger queues use the ordinary allocation path. Closing a +subgroup still drains its queued objects before completing. + For send-path diagnostics, pass `relaynet.WithUDPSendMetrics(observer, 128)` alongside batching. `UDPSendMetrics` observes sampled queue/admission wait and every socket call's requested/sent message counts, duration and error. Partial diff --git a/cmd/relay/main.go b/cmd/relay/main.go index 21e0c966..c7d91073 100644 --- a/cmd/relay/main.go +++ b/cmd/relay/main.go @@ -43,6 +43,7 @@ func main() { addr := flag.String("addr", "0.0.0.0:4433", "listen address") udpSendBatchSize := flag.Int("udp-send-batch-size", 0, "maximum ready UDP writes per Linux batch (0 disables, 1-64 enables)") + udpSendWriters := flag.Int("udp-send-writers", 1, "Linux UDP batch writers (1 or 2; two requires batching)") certFile := flag.String("cert", "", "TLS certificate file (PEM); generated if empty") keyFile := flag.String("key", "", "TLS private key file (PEM); generated if empty") wtPath := flag.String( @@ -137,7 +138,7 @@ func main() { } listener, err := relaynet.Listen(*addr, *wtPath, tlsCfg, logger, - relaynet.WithUDPSendBatchSize(*udpSendBatchSize)) + relaynet.WithUDPSendBatchSize(*udpSendBatchSize), relaynet.WithUDPSendWriters(*udpSendWriters)) if err != nil { log.Fatalf("listen %s: %v", *addr, err) } diff --git a/pkg/relay/handler_fanout.go b/pkg/relay/handler_fanout.go index 3bf598cf..1ac76d59 100644 --- a/pkg/relay/handler_fanout.go +++ b/pkg/relay/handler_fanout.go @@ -801,13 +801,23 @@ func (h *sessionHandler) openWriterForSub( // reading (see [joinWriters]), and resets the open stream of one that // cancelled (§3.1.2). ioCtx, cancelIO := context.WithCancel(ctx) + var inbox chan fwdObject + var inboxDone chan struct{} + if h.subgroupInboxes != nil { + inbox = h.subgroupInboxes.get() + inboxDone = make(chan struct{}) + } else { + inbox = make(chan fwdObject, h.sendQueueSize) + } w := &subgroupWriter{ sub: sub, ctx: ioCtx, cancelIO: cancelIO, unwatchSub: context.AfterFunc(sub.Cancelled(), cancelIO), hdr: subHdr, - inbox: make(chan fwdObject, h.sendQueueSize), + inbox: inbox, + inboxDone: inboxDone, + inboxPool: h.subgroupInboxes, done: make(chan struct{}), log: h.log, metrics: h.metrics, @@ -852,6 +862,8 @@ type subgroupWriter struct { out *session.OutgoingSubgroupStream // nil until run opens it lazily unbridge func() bool // stops the current stream's ctx→Cancel bridge inbox chan fwdObject + inboxDone chan struct{} + inboxPool *subgroupInboxPool done chan struct{} log *slog.Logger metrics Metrics @@ -981,27 +993,28 @@ func (w *subgroupWriter) publish(fwd fwdObject) { w.dropsMu.Unlock() return } - w.dropsMu.Unlock() - fwd.enqueuedAt = time.Now() select { case w.inbox <- fwd: + w.dropsMu.Unlock() w.metrics.ObjectForwarded(w.ref, w.hdr.SubgroupID) default: - w.metrics.ObjectDropped(w.ref, w.hdr.SubgroupID) - w.lastPos = inboundPos{} // the next Object does not follow a sent one - w.dropsMu.Lock() + w.lastPos = inboundPos{} w.drops++ w.markIncompleteLocked(moqt.StreamResetExcessiveLoad) drops := w.drops capped := w.maxDropsBeforeReset > 0 && w.drops > w.maxDropsBeforeReset w.dropsMu.Unlock() - - w.log.Debug("fanout: dropped object on full inbox", - "sub_id", w.sub.ID, "drops", drops) + w.metrics.ObjectDropped(w.ref, w.hdr.SubgroupID) + w.log.Debug("fanout: dropped object on full inbox", "sub_id", w.sub.ID, "drops", drops) if capped { - w.log.Warn("fanout: subscriber hit MaxDropsBeforeReset cap, terminating", - "sub_id", w.sub.ID, "drops", drops) + w.log.Warn( + "fanout: subscriber hit MaxDropsBeforeReset cap, terminating", + "sub_id", + w.sub.ID, + "drops", + drops, + ) w.close(true, moqt.StreamResetExcessiveLoad) } } @@ -1063,6 +1076,7 @@ func (w *subgroupWriter) dropOut() { // draining, so publish never blocks. func (w *subgroupWriter) run() { defer close(w.done) + defer w.releaseInbox() defer w.unwatchSub() var ( @@ -1120,7 +1134,7 @@ func (w *subgroupWriter) run() { }() // failWrites latches this writer broken and stops contributors - // enqueueing. Only close, under sg.Mu, closes the inbox. + // enqueueing. Only close, under sg.Mu, ends normal draining. var writeFailedLatched bool failWrites := func() { writeFailed = true @@ -1134,7 +1148,7 @@ func (w *subgroupWriter) run() { var lagExceeded bool queueMetrics, _ := w.metrics.(ObjectQueueMetrics) - for fwd := range w.inbox { + for fwd, ok := w.nextObject(); ok; fwd, ok = w.nextObject() { if queueMetrics != nil { queueMetrics.ObjectDequeued(w.ref, false, fwd.enqueuedAt, fwd.headerAt) } @@ -1221,6 +1235,11 @@ func (w *subgroupWriter) run() { // it SHOULD. } + w.finishStream(lagExceeded, writeFailed) +} + +// finishStream applies the terminal stream/reset state after the inbox drained. +func (w *subgroupWriter) finishStream(lagExceeded, writeFailed bool) { w.dropsMu.Lock() dropCapped := w.maxDropsBeforeReset > 0 && w.drops > w.maxDropsBeforeReset inboundReset := w.inboundReset @@ -1238,7 +1257,7 @@ func (w *subgroupWriter) run() { cause = ResetCauseExcessiveLoad } w.metrics.SubscriptionResetSlowReader(w.ref, cause) - // Refuse further enqueues; only close may close the inbox. + // Refuse further enqueues; close still owns the drain boundary. w.dropsMu.Lock() w.closed = true w.dropsMu.Unlock() @@ -1349,7 +1368,7 @@ func (w *subgroupWriter) applyPriority() { )) } -// close closes the inbox, recording whether the writer ends its stream with +// close stops admission, recording whether the writer ends its stream with // a reset (and code) or a FIN. The first call wins. Queued objects are still // written; a wedged writer is bounded by [joinWriters]. func (w *subgroupWriter) close(reset bool, code moqt.StreamResetCode) { @@ -1359,7 +1378,11 @@ func (w *subgroupWriter) close(reset bool, code moqt.StreamResetCode) { w.inboundReset = reset w.inboundResetCode = code w.dropsMu.Unlock() - close(w.inbox) + if w.inboxDone != nil { + close(w.inboxDone) + } else { + close(w.inbox) + } }) } diff --git a/pkg/relay/relay.go b/pkg/relay/relay.go index b09da015..e5766630 100644 --- a/pkg/relay/relay.go +++ b/pkg/relay/relay.go @@ -169,7 +169,9 @@ type Config struct { // more memory locked up. // Datagrams share one queue of this size per downstream session, across // all its subscriptions, so a congested connection cannot stall others. - // Zero means: use the default of 64. + // Zero means: use the default of 64. Subgroup inbox storage is reused in + // a relay-wide cache of at most 1024 empty queues when this is at most 256; + // larger configured queues keep the ordinary allocation path. SendQueueSize int // MaxFanoutLag bounds how far behind the live edge a downstream @@ -331,7 +333,8 @@ type Relay struct { // statusRounds shares forwarded TRACK_STATUS rounds across every // session handler; see [sessionHandler.forwardTrackStatus]. - statusRounds *trackStatusRounds + statusRounds *trackStatusRounds + subgroupInboxes *subgroupInboxPool // upstreams dials and pools relay-to-relay sessions for Discovery-driven // cross-relay upstream SUBSCRIBE. nil when Config.Dialer is unset (the @@ -432,15 +435,16 @@ func New(listener Listener, cfg Config) *Relay { } r := &Relay{ - listener: listener, - cfg: cfg, - log: log.With("component", "relay"), - tracks: registry.NewTrackRegistry(trackOpts...), - names: registry.NewNamespaceRegistry(nameOpts...), - fetch: registry.NewFetchRouter(), - statusRounds: newTrackStatusRounds(), - sessions: make(map[*session.Session]struct{}), - stopCh: make(chan struct{}), + listener: listener, + cfg: cfg, + log: log.With("component", "relay"), + tracks: registry.NewTrackRegistry(trackOpts...), + names: registry.NewNamespaceRegistry(nameOpts...), + fetch: registry.NewFetchRouter(), + statusRounds: newTrackStatusRounds(), + subgroupInboxes: newSubgroupInboxPool(cfg.SendQueueSize), + sessions: make(map[*session.Session]struct{}), + stopCh: make(chan struct{}), } if cfg.PauseUnwatchedPublishes > 0 { r.pauser = &unwatchedPauser{delay: cfg.PauseUnwatchedPublishes, stop: r.stopCh, run: r.handlers.Go} @@ -724,6 +728,7 @@ func (r *Relay) serveSession(ctx context.Context, sess *session.Session, leg Leg auth, r.cfg.Metrics, leg, r.fetch, r.statusRounds, r.upstreams, r.cfg.Discovery, r.cfg.RelayAddr, r.cfg.SendQueueSize, r.cfg.MaxDropsBeforeReset, r.cfg.MaxFanoutLag, + r.subgroupInboxes, r.cfg.MaxSubscriptionsPerSession, r.cfg.MaxNamespaceRequestsPerSession, max(r.cfg.MaxRendezvousTimeout, 0), r.handlers.Go, diff --git a/pkg/relay/relaynet/dual.go b/pkg/relay/relaynet/dual.go index 1c6fc8fd..0623b7a2 100644 --- a/pkg/relay/relaynet/dual.go +++ b/pkg/relay/relaynet/dual.go @@ -78,7 +78,7 @@ func Listen(addr, wtPath string, tlsCfg *tls.Config, logger *slog.Logger, opts . // SETUP as 0.5-RTT data to a peer whose certificate is unverified. 0-RTT would // need Config.Allow0RTT, which defaultQUICConfig deliberately leaves unset. ql, closeSocket, err := listenQUIC( - addr, tlsCfg, qcfg, cfg.udpSendBatchSize, cfg.udpSendMetrics, cfg.udpSendSampleEvery, + addr, tlsCfg, qcfg, cfg.udpSendBatchSize, cfg.udpSendWriters, cfg.udpSendMetrics, cfg.udpSendSampleEvery, ) if err != nil { return nil, fmt.Errorf("relaynet: listen %s: %w", addr, err) diff --git a/pkg/relay/relaynet/dual_test.go b/pkg/relay/relaynet/dual_test.go index f24910cd..c41f9fb9 100644 --- a/pkg/relay/relaynet/dual_test.go +++ b/pkg/relay/relaynet/dual_test.go @@ -19,19 +19,29 @@ import ( // keep dialing raw QUIC directly: the two do not have to agree on a transport. func TestDualListenerServesBothTransports(t *testing.T) { t.Parallel() - for _, size := range []int{0, 32} { - t.Run(fmt.Sprintf("batch-%d", size), func(t *testing.T) { testDualListenerTransports(t, size) }) + for _, mode := range []struct{ batch, writers int }{{0, 1}, {32, 1}, {32, 2}} { + t.Run( + fmt.Sprintf("batch-%d-writers-%d", mode.batch, mode.writers), + func(t *testing.T) { testDualListenerTransports(t, mode.batch, mode.writers) }, + ) } } -func testDualListenerTransports(t *testing.T, batchSize int) { +func testDualListenerTransports(t *testing.T, batchSize, writers int) { const wtPath = "/moq" tlsCfg, err := relaynet.TLSConfig("", "", relaynet.DualALPNs) if err != nil { t.Fatalf("TLSConfig: %v", err) } - l, err := relaynet.Listen("127.0.0.1:0", wtPath, tlsCfg, nil, relaynet.WithUDPSendBatchSize(batchSize)) + l, err := relaynet.Listen( + "127.0.0.1:0", + wtPath, + tlsCfg, + nil, + relaynet.WithUDPSendBatchSize(batchSize), + relaynet.WithUDPSendWriters(writers), + ) if err != nil { t.Fatalf("Listen: %v", err) } @@ -102,17 +112,27 @@ func testDualListenerTransports(t *testing.T, batchSize int) { // GOAWAY. An earlier version of this listener did exactly that. func TestDualListenerCloseKeepsLiveSessions(t *testing.T) { t.Parallel() - for _, size := range []int{0, 32} { - t.Run(fmt.Sprintf("batch-%d", size), func(t *testing.T) { testDualListenerDrain(t, size) }) + for _, mode := range []struct{ batch, writers int }{{0, 1}, {32, 1}, {32, 2}} { + t.Run( + fmt.Sprintf("batch-%d-writers-%d", mode.batch, mode.writers), + func(t *testing.T) { testDualListenerDrain(t, mode.batch, mode.writers) }, + ) } } -func testDualListenerDrain(t *testing.T, batchSize int) { +func testDualListenerDrain(t *testing.T, batchSize, writers int) { tlsCfg, err := relaynet.TLSConfig("", "", relaynet.DualALPNs) if err != nil { t.Fatalf("TLSConfig: %v", err) } - l, err := relaynet.Listen("127.0.0.1:0", "/moq", tlsCfg, nil, relaynet.WithUDPSendBatchSize(batchSize)) + l, err := relaynet.Listen( + "127.0.0.1:0", + "/moq", + tlsCfg, + nil, + relaynet.WithUDPSendBatchSize(batchSize), + relaynet.WithUDPSendWriters(writers), + ) if err != nil { t.Fatalf("Listen: %v", err) } diff --git a/pkg/relay/relaynet/options.go b/pkg/relay/relaynet/options.go index b59fea7e..329a311c 100644 --- a/pkg/relay/relaynet/options.go +++ b/pkg/relay/relaynet/options.go @@ -16,6 +16,7 @@ type config struct { quic *quic.Config wt webtransport.Config udpSendBatchSize int + udpSendWriters int udpSendMetrics UDPSendMetrics udpSendSampleEvery uint32 } @@ -32,6 +33,15 @@ func WithUDPSendBatchSize(size int) Option { return func(c *config) { c.udpSendBatchSize = size } } +// WithUDPSendWriters opts Listen into one or two Linux UDP batch writers. +// Zero and one select the default single writer. Two requires a positive +// WithUDPSendBatchSize and uses separate Go handles to the same socket, with +// stable destination affinity and a shared total budget of 256 queued writes. +// Other operating systems keep their ordinary socket. Dial options ignore it. +func WithUDPSendWriters(count int) Option { + return func(c *config) { c.udpSendWriters = count } +} + // WithQUICConfig returns an [Option] that runs tune over the [quic.Config] the // entry point is about to use, once the relay's defaults have been populated. // Passing it more than once applies the hooks in order. diff --git a/pkg/relay/relaynet/udp_batch.go b/pkg/relay/relaynet/udp_batch.go index f07f67b1..269730dc 100644 --- a/pkg/relay/relaynet/udp_batch.go +++ b/pkg/relay/relaynet/udp_batch.go @@ -66,12 +66,22 @@ type udpBatchWriter struct { } func newUDPBatchWriter(socket udpBatchSocket, size int, metrics UDPSendMetrics, sampleEvery uint32) *udpBatchWriter { + return newUDPBatchWriterWithQueue(socket, size, metrics, sampleEvery, udpSendQueueSize) +} + +func newUDPBatchWriterWithQueue( + socket udpBatchSocket, + size int, + metrics UDPSendMetrics, + sampleEvery uint32, + queueSize int, +) *udpBatchWriter { w := &udpBatchWriter{ socket: socket, metrics: metrics, sampleEvery: sampleEvery, size: size, - queue: make(chan *udpWriteRequest, udpSendQueueSize), + queue: make(chan *udpWriteRequest, queueSize), closed: make(chan struct{}), done: make(chan struct{}), pool: sync.Pool{New: func() any { @@ -206,19 +216,7 @@ func (w *udpBatchWriter) flush(requests []*udpWriteRequest, messages []ipv4.Mess return } if !*singleOnly { - start := w.beforeSend(requests) - n, err := w.socket.WriteBatch(messages, 0) - if w.metrics != nil { - w.metrics.UDPSendCall(true, len(messages), n, time.Since(start), err) - } - for i := range n { - r := requests[i] - result := udpWriteResult{n: messages[i].N, oobn: len(r.oob)} - if result.n != len(r.buffer[0]) { - result.err = io.ErrShortWrite - } - completeUDPWrite(requests, messages, i, result) - } + n, err := w.sendBatch(requests, messages) requests, messages = requests[n:], messages[n:] if len(requests) == 0 { return @@ -253,6 +251,24 @@ func (w *udpBatchWriter) flush(requests []*udpWriteRequest, messages []ipv4.Mess } } +// sendBatch completes only the accepted prefix; flush owns unsent retries. +func (w *udpBatchWriter) sendBatch(requests []*udpWriteRequest, messages []ipv4.Message) (int, error) { + start := w.beforeSend(requests) + n, err := w.socket.WriteBatch(messages, 0) + if w.metrics != nil { + w.metrics.UDPSendCall(true, len(messages), n, time.Since(start), err) + } + for i := range n { + r := requests[i] + result := udpWriteResult{n: messages[i].N, oobn: len(r.oob)} + if result.n != len(r.buffer[0]) { + result.err = io.ErrShortWrite + } + completeUDPWrite(requests, messages, i, result) + } + return n, err +} + func completeUDPWrite(requests []*udpWriteRequest, messages []ipv4.Message, i int, result udpWriteResult) { r := requests[i] // Drop every borrowed reference before waking the caller, which may diff --git a/pkg/relay/relaynet/udp_batch_linux_test.go b/pkg/relay/relaynet/udp_batch_linux_test.go index f2bce7ca..aca9b5ea 100644 --- a/pkg/relay/relaynet/udp_batch_linux_test.go +++ b/pkg/relay/relaynet/udp_batch_linux_test.go @@ -5,7 +5,9 @@ import ( "context" "encoding/binary" "errors" + "hash/maphash" "net" + "strconv" "testing" "time" "unsafe" @@ -24,6 +26,12 @@ func udpControlMessage(level, kind int32, data []byte) []byte { } func TestUDPBatchConnGSOMetadata(t *testing.T) { + t.Run("one", func(t *testing.T) { testUDPBatchConnGSOMetadata(t, 1, 0) }) + t.Run("two-shard-0", func(t *testing.T) { testUDPBatchConnGSOMetadata(t, 2, 0) }) + t.Run("two-shard-1", func(t *testing.T) { testUDPBatchConnGSOMetadata(t, 2, 1) }) +} + +func testUDPBatchConnGSOMetadata(t *testing.T, writers, shard int) { t.Parallel() receiver, err := net.ListenUDP("udp6", &net.UDPAddr{IP: net.IPv6loopback}) if err != nil { @@ -51,10 +59,25 @@ func TestUDPBatchConnGSOMetadata(t *testing.T) { t.Fatal(err) } m := &udpMetricSamples{} - c := &udpBatchConn{ - UDPConn: socket, - writer: newUDPBatchWriter(udpBatchBackend{UDPConn: socket, batch: ipv4.NewPacketConn(socket)}, 32, m, 1), + var writer udpPacketWriter + if writers == 1 { + writer = newUDPBatchWriter(udpBatchBackend{UDPConn: socket, batch: ipv4.NewPacketConn(socket)}, 32, m, 1) + } else { + group, err := newUDPWriterGroup(socket, 32, m, 1) + if err != nil { + socket.Close() + t.Fatal(err) + } + for attempt := 0; group.shard(receiver.LocalAddr().(*net.UDPAddr)) != shard && attempt < 1000; attempt++ { + group.seed = maphash.MakeSeed() + } + if group.shard(receiver.LocalAddr().(*net.UDPAddr)) != shard { + group.Close() + t.Fatal("could not select metadata shard") + } + writer = group } + c := &udpBatchConn{UDPConn: socket, writer: writer} t.Cleanup(func() { c.Close() }) const segmentSize = 900 gso := udpControlMessage(unix.IPPROTO_UDP, unix.UDP_SEGMENT, binary.NativeEndian.AppendUint16(nil, segmentSize)) @@ -107,13 +130,19 @@ func TestUDPBatchConnGSOMetadata(t *testing.T) { } func TestUDPBatchListenerReleasesSocket(t *testing.T) { + for _, writers := range []int{1, 2} { + t.Run(strconv.Itoa(writers), func(t *testing.T) { testUDPBatchListenerReleasesSocket(t, writers) }) + } +} + +func testUDPBatchListenerReleasesSocket(t *testing.T, writers int) { for _, acceptConnection := range []bool{false, true} { t.Run(map[bool]string{false: "idle", true: "drained"}[acceptConnection], func(t *testing.T) { tlsCfg, err := TLSConfig("", "", DualALPNs) if err != nil { t.Fatal(err) } - l, err := Listen("127.0.0.1:0", "/moq", tlsCfg, nil, WithUDPSendBatchSize(32)) + l, err := Listen("127.0.0.1:0", "/moq", tlsCfg, nil, WithUDPSendBatchSize(32), WithUDPSendWriters(writers)) if err != nil { t.Fatal(err) } @@ -155,13 +184,19 @@ func TestUDPBatchListenerReleasesSocket(t *testing.T) { } func TestUDPBatchListenerFailureClosesSocket(t *testing.T) { + for _, writers := range []int{1, 2} { + t.Run(strconv.Itoa(writers), func(t *testing.T) { testUDPBatchListenerFailureClosesSocket(t, writers) }) + } +} + +func testUDPBatchListenerFailureClosesSocket(t *testing.T, writers int) { s, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) if err != nil { t.Fatal(err) } addr := s.LocalAddr().String() s.Close() - if l, _, err := listenQUIC(addr, nil, defaultQUICConfig(), 32, nil, 0); err == nil { + if l, _, err := listenQUIC(addr, nil, defaultQUICConfig(), 32, writers, nil, 0); err == nil { l.Close() t.Fatal("nil TLS config accepted") } diff --git a/pkg/relay/relaynet/udp_group.go b/pkg/relay/relaynet/udp_group.go new file mode 100644 index 00000000..f29070c7 --- /dev/null +++ b/pkg/relay/relaynet/udp_group.go @@ -0,0 +1,114 @@ +package relaynet + +import ( + "errors" + "hash/maphash" + "net" + "net/netip" + "sync" + "time" + + "golang.org/x/net/ipv4" +) + +// Each destination stays on one FIFO, including writes outside QUIC's send +// queue (for example path probes). Independent Go FDs allow concurrent socket +// writes while retaining the listener's source address and port. +type udpWriterGroup struct { + writers [2]*udpBatchWriter + seed maphash.Seed + closed chan struct{} + deadlineMu sync.Mutex + closeOnce sync.Once + closeErr error +} + +func newUDPWriterGroup( + socket *net.UDPConn, + size int, + metrics UDPSendMetrics, + sampleEvery uint32, +) (*udpWriterGroup, error) { + file, err := socket.File() + if err != nil { + return nil, err + } + packetConn, err := net.FilePacketConn(file) + fileErr := file.Close() + if err != nil { + return nil, err + } + if fileErr != nil { + _ = packetConn.Close() + return nil, fileErr + } + duplicate, ok := packetConn.(*net.UDPConn) + if !ok { + _ = packetConn.Close() + return nil, errors.New("duplicated socket is not UDP") + } + g := &udpWriterGroup{seed: maphash.MakeSeed(), closed: make(chan struct{})} + for i, conn := range []*net.UDPConn{socket, duplicate} { + g.writers[i] = newUDPBatchWriterWithQueue( + udpBatchBackend{UDPConn: conn, batch: ipv4.NewPacketConn(conn)}, + size, metrics, sampleEvery, udpSendQueueSize/len(g.writers), + ) + } + return g, nil +} + +func (g *udpWriterGroup) shard(addr *net.UDPAddr) int { + if addr == nil || addr.Port < 0 || addr.Port > 65535 { + return 0 // Keep invalid-address errors in the socket's native path. + } + ip, ok := netip.AddrFromSlice(addr.IP) + if !ok { + return 0 + } + ip = ip.Unmap() + if addr.Zone != "" { + ip = ip.WithZone(addr.Zone) + } + return int(maphash.Comparable(g.seed, netip.AddrPortFrom(ip, uint16(addr.Port))) & 1) +} + +func (g *udpWriterGroup) WriteMsgUDP(b, oob []byte, addr *net.UDPAddr) (int, int, error) { + select { + case <-g.closed: + return 0, 0, net.ErrClosed + default: + } + return g.writers[g.shard(addr)].WriteMsgUDP(b, oob, addr) +} + +func (g *udpWriterGroup) SetWriteDeadline(at time.Time) error { + // Concurrent callers must not leave different deadlines on the two FDs. + g.deadlineMu.Lock() + defer g.deadlineMu.Unlock() + select { + case <-g.closed: + return net.ErrClosed + default: + } + var errs [2]error + for i, writer := range g.writers { + errs[i] = writer.SetWriteDeadline(at) + } + return errors.Join(errs[:]...) +} + +func (g *udpWriterGroup) Close() error { + g.closeOnce.Do(func() { + close(g.closed) + // Cancel both admissions and both socket calls before joining either. + // Taking deadlineMu here could block shutdown behind a deadline call. + var wg sync.WaitGroup + var errs [2]error + for i, writer := range g.writers { + wg.Go(func() { errs[i] = writer.Close() }) + } + wg.Wait() + g.closeErr = errors.Join(errs[:]...) + }) + return g.closeErr +} diff --git a/pkg/relay/relaynet/udp_group_test.go b/pkg/relay/relaynet/udp_group_test.go new file mode 100644 index 00000000..fec85a7c --- /dev/null +++ b/pkg/relay/relaynet/udp_group_test.go @@ -0,0 +1,383 @@ +package relaynet + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/maphash" + "net" + "sync" + "syscall" + "testing" + "testing/synctest" + "time" + + "golang.org/x/net/ipv4" +) + +func udpGroupForTest(sockets [2]udpBatchSocket) *udpWriterGroup { + g := &udpWriterGroup{seed: maphash.MakeSeed(), closed: make(chan struct{})} + for i, socket := range sockets { + g.writers[i] = newUDPBatchWriterWithQueue(socket, 32, nil, 0, udpSendQueueSize/2) + } + return g +} + +func udpAddressForShard(t *testing.T, g *udpWriterGroup, shard int) *net.UDPAddr { + t.Helper() + for port := 10000; port < 12000; port++ { + addr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port} + if g.shard(addr) == shard { + return addr + } + } + t.Fatal("destination hash never selected shard", shard) + return nil +} + +func TestUDPWriterGroupParallelAndBorrowedBuffers(t *testing.T) { + var entered [2]chan struct{} + var release [2]chan struct{} + var releaseOnce [2]sync.Once + var sockets [2]udpBatchSocket + for i := range sockets { + entered[i], release[i] = make(chan struct{}), make(chan struct{}) + var once sync.Once + sockets[i] = testUDPSocket{batch: func(ms []ipv4.Message) (int, error) { + once.Do(func() { close(entered[i]) }) + <-release[i] + for j := range ms { + if !bytes.Equal(ms[j].Buffers[0], []byte{byte(i), 42}) || !bytes.Equal(ms[j].OOB, []byte{byte(i), 9}) { + t.Error("caller buffers changed before completion") + } + ms[j].N = len(ms[j].Buffers[0]) + } + return len(ms), nil + }, close: func() error { releaseOnce[i].Do(func() { close(release[i]) }); return nil }} + } + g := udpGroupForTest(sockets) + t.Cleanup(func() { g.Close() }) + results := make(chan error, 2) + for i := range sockets { + addr := udpAddressForShard(t, g, i) + go func() { + b, oob := []byte{byte(i), 42}, []byte{byte(i), 9} + n, on, err := g.WriteMsgUDP(b, oob, addr) + clear(b) + clear(oob) + if err == nil && (n != 2 || on != 2) { + err = errors.New("short write") + } + results <- err + }() + } + for i := range entered { + select { + case <-entered[i]: + case <-time.After(5 * time.Second): + for j := range release { + releaseOnce[j].Do(func() { close(release[j]) }) + } + t.Fatal("writers did not enter independent socket calls") + } + } + select { + case <-results: + t.Fatal("write returned before its socket call completed") + default: + } + for j := range release { + releaseOnce[j].Do(func() { close(release[j]) }) + } + for range sockets { + if err := <-results; err != nil { + t.Fatal(err) + } + } + if got := cap(g.writers[0].queue) + cap(g.writers[1].queue); got != udpSendQueueSize { + t.Fatal("queue budget", got) + } +} + +func TestUDPWriterGroupDeadlineFanoutSerialized(t *testing.T) { + first, second := time.Now().Add(time.Second), time.Now().Add(2*time.Second) + entered, release := make(chan struct{}), make(chan struct{}) + secondStarted, secondEntered := make(chan struct{}), make(chan struct{}, 1) + var unblock sync.Once + var deadlines [2]time.Time + var sockets [2]udpBatchSocket + for i := range sockets { + sockets[i] = testUDPSocket{deadline: func(at time.Time) error { + if i == 0 && at.Equal(first) { + close(entered) + <-release + } + if i == 0 && at.Equal(second) { + secondEntered <- struct{}{} + } + deadlines[i] = at + return nil + }} + } + g := udpGroupForTest(sockets) + var calls sync.WaitGroup + t.Cleanup(func() { unblock.Do(func() { close(release) }); calls.Wait(); g.Close() }) + calls.Go(func() { + if err := g.SetWriteDeadline(first); err != nil { + t.Error(err) + } + }) + <-entered + calls.Go(func() { + close(secondStarted) + if err := g.SetWriteDeadline(second); err != nil { + t.Error(err) + } + }) + <-secondStarted + select { + case <-secondEntered: + t.Fatal("deadline updates interleaved across socket handles") + case <-time.After(10 * time.Millisecond): + } + unblock.Do(func() { close(release) }) + calls.Wait() + if deadlines != [2]time.Time{second, second} { + t.Fatal("inconsistent deadlines", deadlines) + } +} + +func TestUDPWriterGroupCloseInterruptsDeadlineAndWrites(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + sockets := [2]udpBatchSocket{ + testUDPSocket{ + deadline: func(time.Time) error { close(entered); <-release; return nil }, + close: func() error { once.Do(func() { close(release) }); return nil }, + }, + testUDPSocket{}, + } + g := udpGroupForTest(sockets) + var calls sync.WaitGroup + calls.Go(func() { _ = g.SetWriteDeadline(time.Now().Add(time.Second)) }) + <-entered + if err := g.Close(); err != nil { + t.Fatal(err) + } + calls.Wait() + if _, _, err := g.WriteMsgUDP([]byte{1}, nil, nil); !errors.Is(err, net.ErrClosed) { + t.Fatal("write after close", err) + } + if err := g.SetWriteDeadline(time.Time{}); !errors.Is(err, net.ErrClosed) { + t.Fatal("deadline after close", err) + } + }) +} + +func TestUDPWriterGroupCloseJoinsBothWorkers(t *testing.T) { + var closeCount int + var mu sync.Mutex + errOne, errTwo := errors.New("socket one"), errors.New("socket two") + g := udpGroupForTest([2]udpBatchSocket{ + testUDPSocket{close: func() error { mu.Lock(); closeCount++; mu.Unlock(); return errOne }}, + testUDPSocket{close: func() error { mu.Lock(); closeCount++; mu.Unlock(); return errTwo }}, + }) + var callers sync.WaitGroup + for range 10 { + callers.Go(func() { + err := g.Close() + if !errors.Is(err, errOne) || !errors.Is(err, errTwo) { + t.Error("lost close error", err) + } + }) + } + callers.Wait() + if closeCount != 2 { + t.Fatal("closed sockets more than once", closeCount) + } +} + +func TestUDPWriterGroupCanonicalAddressAffinity(t *testing.T) { + g := &udpWriterGroup{seed: maphash.MakeSeed()} + a := &net.UDPAddr{IP: net.IP{127, 0, 0, 1}, Port: 12345} + b := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 12345} + if g.shard(a) != g.shard(b) { + t.Fatal("IPv4 representation changed destination affinity") + } + for range 100 { + if g.shard(a) != g.shard(b) { + t.Fatal("unstable affinity") + } + } +} + +func TestUDPWriterGroupFallbackStaysOnFailedWorker(t *testing.T) { + var calls, singles [2]int + g := udpGroupForTest([2]udpBatchSocket{ + testUDPSocket{ + batch: func([]ipv4.Message) (int, error) { calls[0]++; return 0, syscall.EOPNOTSUPP }, + single: func(b, oob []byte, _ *net.UDPAddr) (int, int, error) { singles[0]++; return len(b), len(oob), nil }, + }, + testUDPSocket{ + batch: func(ms []ipv4.Message) (int, error) { + calls[1]++ + for i := range ms { + ms[i].N = len(ms[i].Buffers[0]) + } + return len(ms), nil + }, + single: func(b, oob []byte, _ *net.UDPAddr) (int, int, error) { singles[1]++; return len(b), len(oob), nil }, + }, + }) + t.Cleanup(func() { g.Close() }) + for range 2 { + for shard := range 2 { + if _, _, err := g.WriteMsgUDP([]byte{1}, nil, udpAddressForShard(t, g, shard)); err != nil { + t.Fatal(err) + } + } + } + if err := g.Close(); err != nil { + t.Fatal(err) + } + if calls != [2]int{1, 2} || singles != [2]int{2, 0} { + t.Fatalf("fallback crossed workers: batches=%v singles=%v", calls, singles) + } +} + +func TestUDPWriterCountValidation(t *testing.T) { + tlsCfg, err := TLSConfig("", "", DualALPNs) + if err != nil { + t.Fatal(err) + } + for _, mode := range []struct{ writers, batch int }{{-1, 32}, {3, 32}, {2, 0}} { + listener, err := Listen( + "127.0.0.1:0", + "/moq", + tlsCfg, + nil, + WithUDPSendBatchSize(mode.batch), + WithUDPSendWriters(mode.writers), + ) + if err == nil { + listener.Close() + t.Fatalf("accepted writers=%d batch=%d", mode.writers, mode.batch) + } + } +} + +func TestUDPWriterGroupSocketSourceOrderingAndDeadline(t *testing.T) { + for _, network := range []string{"udp4", "udp6"} { + t.Run(network, func(t *testing.T) { + ip := net.IPv4(127, 0, 0, 1) + if network == "udp6" { + ip = net.IPv6loopback + } + socket, err := net.ListenUDP(network, &net.UDPAddr{IP: ip}) + if err != nil { + t.Fatal(err) + } + m := &udpMetricSamples{} + g, err := newUDPWriterGroup(socket, 32, m, 1) + if err != nil { + socket.Close() + t.Fatal(err) + } + t.Cleanup(func() { g.Close() }) + var receivers []*net.UDPConn + var shards [2]int + for len(receivers) < 64 && (len(receivers) < 8 || shards[0] == 0 || shards[1] == 0) { + r, err := net.ListenUDP(network, &net.UDPAddr{IP: ip}) + if err != nil { + t.Fatal(err) + } + receivers = append(receivers, r) + t.Cleanup(func() { r.Close() }) + shards[g.shard(r.LocalAddr().(*net.UDPAddr))]++ + } + if shards[0] == 0 || shards[1] == 0 { + t.Fatal("destinations did not reach both workers") + } + if err := g.SetWriteDeadline(time.Now().Add(-time.Second)); err != nil { + t.Fatal(err) + } + for _, receiver := range receivers { + if _, _, err := g.WriteMsgUDP([]byte{1}, nil, receiver.LocalAddr().(*net.UDPAddr)); err == nil { + t.Fatal("expired deadline accepted write") + } + } + if err := g.SetWriteDeadline(time.Time{}); err != nil { + t.Fatal(err) + } + const packets = 20 + var sends, reads sync.WaitGroup + for id, receiver := range receivers { + if err := receiver.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + reads.Go(func() { + b := make([]byte, 16) + for seq := range packets { + n, from, err := receiver.ReadFromUDP(b) + if err != nil { + t.Error(err) + return + } + if n != 16 || binary.LittleEndian.Uint64(b) != uint64(id) || + binary.LittleEndian.Uint64(b[8:]) != uint64(seq) { + t.Error("payload or order changed") + } + if from.Port != socket.LocalAddr().(*net.UDPAddr).Port || !from.IP.Equal(ip) { + t.Error("source address changed") + } + } + }) + sends.Go(func() { + b := make([]byte, 16) + for seq := range packets { + binary.LittleEndian.PutUint64(b, uint64(id)) + binary.LittleEndian.PutUint64(b[8:], uint64(seq)) + if n, _, err := g.WriteMsgUDP( + b, + nil, + receiver.LocalAddr().(*net.UDPAddr), + ); err != nil || + n != len(b) { + t.Errorf("write: %d %v", n, err) + return + } + } + clear(b) + }) + } + sends.Wait() + reads.Wait() + if err := g.Close(); err != nil { + t.Fatal(err) + } + if len(m.waits) != len(receivers)*packets { + t.Fatal("lost queue observations", len(m.waits)) + } + accepted := 0 + for _, sample := range m.calls { + accepted += sample.sent + if sample.err != nil { + t.Error(sample.err) + } + } + if accepted != len(receivers)*packets { + t.Fatal("accepted message count", accepted) + } + }) + } +} + +func BenchmarkUDPWriterDestinationAffinity(b *testing.B) { + g := &udpWriterGroup{seed: maphash.MakeSeed()} + addr := &net.UDPAddr{IP: net.ParseIP("2001:db8::1"), Port: 4433} + b.ReportAllocs() + for b.Loop() { + _ = g.shard(addr) + } +} diff --git a/pkg/relay/relaynet/udp_listener.go b/pkg/relay/relaynet/udp_listener.go index 59dc1aec..af8b5a0a 100644 --- a/pkg/relay/relaynet/udp_listener.go +++ b/pkg/relay/relaynet/udp_listener.go @@ -19,7 +19,13 @@ const maxUDPSendBatchSize = 64 type udpBatchConn struct { *net.UDPConn - writer *udpBatchWriter + writer udpPacketWriter +} + +type udpPacketWriter interface { + WriteMsgUDP([]byte, []byte, *net.UDPAddr) (int, int, error) + SetWriteDeadline(time.Time) error + Close() error } var _ quic.OOBCapablePacketConn = (*udpBatchConn)(nil) @@ -63,9 +69,19 @@ func listenQUIC( tlsCfg *tls.Config, qcfg *quic.Config, batchSize int, + writers int, metrics UDPSendMetrics, sampleEvery uint32, ) (*quic.Listener, func(), error) { + if writers < 0 || writers > 2 { + return nil, nil, errors.New("UDP send writers must be one or two (zero selects one)") + } + if writers == 0 { + writers = 1 + } + if writers == 2 && batchSize == 0 { + return nil, nil, errors.New("two UDP send writers require UDP send batching") + } if metrics != nil && sampleEvery == 0 { return nil, nil, errors.New("UDP send metrics sample interval must be positive") } @@ -85,9 +101,17 @@ func listenQUIC( return nil, nil, err } conn := &udpBatchConn{UDPConn: socket} - conn.writer = newUDPBatchWriter( - udpBatchBackend{UDPConn: socket, batch: ipv4.NewPacketConn(socket)}, batchSize, metrics, sampleEvery, - ) + if writers == 1 { + conn.writer = newUDPBatchWriter( + udpBatchBackend{UDPConn: socket, batch: ipv4.NewPacketConn(socket)}, batchSize, metrics, sampleEvery, + ) + } else { + conn.writer, err = newUDPWriterGroup(socket, batchSize, metrics, sampleEvery) + if err != nil { + _ = socket.Close() + return nil, nil, err + } + } var connections sync.WaitGroup transport := &quic.Transport{ Conn: conn, diff --git a/pkg/relay/relaynet/udp_metrics.go b/pkg/relay/relaynet/udp_metrics.go index 2b7cbd9f..38980c66 100644 --- a/pkg/relay/relaynet/udp_metrics.go +++ b/pkg/relay/relaynet/udp_metrics.go @@ -4,7 +4,7 @@ import "time" // UDPSendMetrics observes the shared Linux UDP writer enabled by // WithUDPSendBatchSize. Hooks run synchronously and must not block. They may -// be called concurrently if the same implementation observes several listeners. +// be called concurrently with two writers or when observing several listeners. // No peer addresses or packet contents are exposed. type UDPSendMetrics interface { // UDPSendQueueWait measures sampled writes from admission in WriteMsgUDP until diff --git a/pkg/relay/relaynet/udp_metrics_test.go b/pkg/relay/relaynet/udp_metrics_test.go index b622c49a..c31404d4 100644 --- a/pkg/relay/relaynet/udp_metrics_test.go +++ b/pkg/relay/relaynet/udp_metrics_test.go @@ -20,12 +20,19 @@ type udpCallSample struct { err error } type udpMetricSamples struct { + mu sync.Mutex waits []time.Duration calls []udpCallSample } -func (m *udpMetricSamples) UDPSendQueueWait(d time.Duration) { m.waits = append(m.waits, d) } +func (m *udpMetricSamples) UDPSendQueueWait(d time.Duration) { + m.mu.Lock() + defer m.mu.Unlock() + m.waits = append(m.waits, d) +} func (m *udpMetricSamples) UDPSendCall(batch bool, requested, sent int, elapsed time.Duration, err error) { + m.mu.Lock() + defer m.mu.Unlock() m.calls = append(m.calls, udpCallSample{batch, requested, sent, elapsed, err}) } @@ -107,7 +114,7 @@ func TestUDPMetricsPartialAndFallback(t *testing.T) { func TestUDPMetricsRejectZeroSampleInterval(t *testing.T) { m := &udpMetricSamples{} - if _, _, err := listenQUIC("127.0.0.1:0", nil, defaultQUICConfig(), 32, m, 0); err == nil { + if _, _, err := listenQUIC("127.0.0.1:0", nil, defaultQUICConfig(), 32, 1, m, 0); err == nil { t.Fatal("accepted zero interval") } } diff --git a/pkg/relay/session_handler.go b/pkg/relay/session_handler.go index d8c7d1f6..1a48bb21 100644 --- a/pkg/relay/session_handler.go +++ b/pkg/relay/session_handler.go @@ -54,6 +54,7 @@ type sessionHandler struct { discovery discovery.DiscoveryStore relayAddr string sendQueueSize int + subgroupInboxes *subgroupInboxPool maxDropsBeforeReset int maxFanoutLag time.Duration // maxRendezvous caps RENDEZVOUS_TIMEOUT (§9.20.6); 0 holds no SUBSCRIBE. @@ -105,6 +106,7 @@ func newSessionHandler( sendQueueSize int, maxDropsBeforeReset int, maxFanoutLag time.Duration, + subgroupInboxes *subgroupInboxPool, maxSubsPerSession int, maxNamespaceReqsPerSession int, maxRendezvous time.Duration, @@ -125,6 +127,7 @@ func newSessionHandler( discovery: discovery, relayAddr: relayAddr, sendQueueSize: sendQueueSize, + subgroupInboxes: subgroupInboxes, maxDropsBeforeReset: maxDropsBeforeReset, maxFanoutLag: maxFanoutLag, maxRendezvous: maxRendezvous, diff --git a/pkg/relay/subgroup_inbox.go b/pkg/relay/subgroup_inbox.go new file mode 100644 index 00000000..32a70971 --- /dev/null +++ b/pkg/relay/subgroup_inbox.go @@ -0,0 +1,91 @@ +package relay + +// Retention is bounded per relay, and unusually large configured queues keep +// the original allocation path. Cached channels are open, empty, and have no +// object references. The writer owns its channel until releaseInbox completes. +const ( + maxCachedSubgroupInboxes = 1024 + maxPooledSubgroupQueueSize = 256 +) + +type subgroupInboxPool struct { + size int + free chan chan fwdObject +} + +func newSubgroupInboxPool(size int) *subgroupInboxPool { + if size <= 0 || size > maxPooledSubgroupQueueSize { + return nil + } + return &subgroupInboxPool{size: size, free: make(chan chan fwdObject, maxCachedSubgroupInboxes)} +} + +func (p *subgroupInboxPool) get() chan fwdObject { + select { + case inbox := <-p.free: + return inbox + default: + return make(chan fwdObject, p.size) + } +} + +func (p *subgroupInboxPool) put(inbox chan fwdObject) { + if cap(inbox) != p.size { + return + } + for { + select { + case _, ok := <-inbox: + if !ok { + return + } // A legacy/test channel was closed; never reuse it. + default: + select { + case p.free <- inbox: + default: // The bounded cache is full; let this empty channel go. + } + return + } + } +} + +func (w *subgroupWriter) nextObject() (fwdObject, bool) { + if w.inboxDone == nil { + fwd, ok := <-w.inbox + return fwd, ok + } + // Avoid a two-channel select when data is already queued. + select { + case fwd, ok := <-w.inbox: + return fwd, ok + default: + } + select { + case fwd, ok := <-w.inbox: + return fwd, ok + case <-w.inboxDone: + // Closing stops producers, but queued objects still drain before FIN. + select { + case fwd, ok := <-w.inbox: + return fwd, ok + default: + return fwdObject{}, false + } + } +} + +func (w *subgroupWriter) releaseInbox() { + if w.inboxPool == nil { + return + } + // publish holds this lock through its nonblocking enqueue. A late producer + // must see closed before the channel can belong to another subgroup. + w.dropsMu.Lock() + w.closed = true + inbox := w.inbox + w.inbox = nil + w.dropsMu.Unlock() + if inbox != nil { + w.inboxPool.put(inbox) + } +} diff --git a/pkg/relay/subgroup_inbox_test.go b/pkg/relay/subgroup_inbox_test.go new file mode 100644 index 00000000..5d7d0488 --- /dev/null +++ b/pkg/relay/subgroup_inbox_test.go @@ -0,0 +1,148 @@ +package relay + +import ( + "log/slog" + "runtime" + "sync" + "testing" + + "github.com/floatdrop/moq-go/pkg/moqt/message" + "github.com/floatdrop/moq-go/pkg/relay/internal/registry" +) + +func pooledWriterForTest(p *subgroupInboxPool) *subgroupWriter { + return &subgroupWriter{ + inbox: p.get(), inboxDone: make(chan struct{}), inboxPool: p, + done: make(chan struct{}), metrics: NopMetrics{}, + log: slog.New(slog.DiscardHandler), + sub: ®istry.DownstreamSub{}, + } +} + +func TestSubgroupInboxCloseDrainsBeforeReuse(t *testing.T) { + p := newSubgroupInboxPool(4) + w := pooledWriterForTest(p) + original := w.inbox + object := &message.SubgroupObject{} + for i := range 4 { + w.publish(fwdObject{obj: object, absID: uint64(i)}) + } + w.close(false, 0) + for i := range 4 { + fwd, ok := w.nextObject() + if !ok || fwd.absID != uint64(i) || fwd.obj != object { + t.Fatalf("drain %d: %+v, %v", i, fwd, ok) + } + } + if _, ok := w.nextObject(); ok { + t.Fatal("closed inbox still produced an object") + } + w.releaseInbox() + w.releaseInbox() // Never cache one channel twice. + if w.inbox != nil || len(p.free) != 1 { + t.Fatal("inbox was not detached exactly once") + } + if next := p.get(); next != original || len(next) != 0 { + t.Fatal("inbox storage was not reused empty") + } +} + +func TestSubgroupInboxEarlyExitDropsReferences(t *testing.T) { + p := newSubgroupInboxPool(4) + w := pooledWriterForTest(p) + original := w.inbox + for range 4 { + w.publish(fwdObject{obj: &message.SubgroupObject{}}) + } + w.releaseInbox() // The lag/error exit need not consume the rest. + if len(original) != 0 { + t.Fatal("cached inbox retained queued objects") + } + w.close(true, 0) // Late parent close must not close the cached data channel. + next := p.get() + if next != original { + t.Fatal("expected reuse") + } + next <- fwdObject{absID: 77} + if got := <-next; got.absID != 77 { + t.Fatal("late close affected reused channel") + } +} + +func TestSubgroupInboxLateProducerCannotReachReusedQueue(t *testing.T) { + p := newSubgroupInboxPool(4) + for range 200 { + w := pooledWriterForTest(p) + start := make(chan struct{}) + first := make(chan struct{}) + var producer sync.WaitGroup + producer.Go(func() { + <-start + w.publish(fwdObject{absID: 99}) + close(first) + for range 100 { + runtime.Gosched() + w.publish(fwdObject{absID: 99}) + } + }) + close(start) + <-first + w.releaseInbox() + reused := p.get() + producer.Wait() + w.publish(fwdObject{absID: 99}) + if len(reused) != 0 { + t.Fatal("old producer wrote into a new subgroup") + } + p.put(reused) + } +} + +func TestSubgroupInboxCacheBoundAndClosedChannel(t *testing.T) { + if newSubgroupInboxPool(maxPooledSubgroupQueueSize+1) != nil { + t.Fatal("large queues should not be retained") + } + p := newSubgroupInboxPool(1) + closed := make(chan fwdObject, 1) + close(closed) + p.put(closed) + if len(p.free) != 0 { + t.Fatal("cached a closed channel") + } + for range maxCachedSubgroupInboxes + 10 { + p.put(make(chan fwdObject, 1)) + } + if len(p.free) != maxCachedSubgroupInboxes { + t.Fatal("unbounded cache", len(p.free)) + } +} + +var benchmarkInboxWriter *subgroupWriter + +func BenchmarkSubgroupInboxLifecycle(b *testing.B) { + for _, reuse := range []bool{false, true} { + name := "fresh" + if reuse { + name = "reused" + } + b.Run(name, func(b *testing.B) { + p := newSubgroupInboxPool(defaultSendQueueSize) + b.ReportAllocs() + for b.Loop() { + w := &subgroupWriter{done: make(chan struct{})} + if reuse { + w.inbox, w.inboxDone, w.inboxPool = p.get(), make(chan struct{}), p + } else { + w.inbox = make(chan fwdObject, defaultSendQueueSize) + } + w.close(false, 0) + if _, ok := w.nextObject(); ok { + b.Fatal("closed queue had an object") + } + w.releaseInbox() + close(w.done) + benchmarkInboxWriter = w + } + }) + } +}