From cd15f43a54a1fa477e2198f4f0e96efbf5c71942 Mon Sep 17 00:00:00 2001 From: tunsuy Date: Mon, 16 Nov 2026 21:20:58 +0800 Subject: [PATCH] fix(envd): open multiple TCP connections to the NFS proxy per mount Without nconnect the guest kernel multiplexes every request of an NFS mount over a single TCP connection, and the proxy serves each connection's requests in turn. One slow operation -- a READDIR of a big directory, a synchronous write -- therefore blocks every other request queued behind it on that connection, stalling unrelated work on the whole volume. Mount volumes with nconnect=8 so the kernel spreads outstanding requests over several connections; the go-nfs server already serves each accepted connection in its own goroutine, so the proxy needs no change. The connections are established lazily and only while requests are outstanding, and the proxy resolves NFSv3 handles per mount rather than per connection, so the extra connections need no additional MOUNT. Fixes #3818 Co-Authored-By: Claude Code --- packages/envd/internal/api/init.go | 17 +++++++++++++++++ packages/envd/internal/api/init_test.go | 21 +++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 8d01b9e38be..033f5bf13e4 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -748,6 +748,15 @@ func writeInitError(w http.ResponseWriter, logger zerolog.Logger, err error) { w.Write([]byte(err.Error())) } +// nfsConnections is the nconnect value: how many TCP connections the guest kernel +// may open to the NFS proxy per mount. The proxy handles one connection's requests +// in turn, so a slow operation (a READDIR of a big directory, a sync write) blocks +// every other request queued behind it on that connection; spreading the mount's +// requests over several connections keeps unrelated work moving. The proxy serves +// every connection in its own goroutine, so a handful of connections costs it +// little; more would mostly add guest-side sockets. The kernel accepts 1-16. +const nfsConnections = 8 + var nfsOptions = strings.Join([]string{ // wait for data to be sent to proxy server before returning. // async might cause issues if the sandbox is shut down suddenly. @@ -762,6 +771,14 @@ var nfsOptions = strings.Join([]string{ "nfsvers=3", // nfs proxy is nfs version 3 "noacl", // no reason for acl in the sandbox + // Without this the kernel client multiplexes every request of a mount over one + // TCP connection, and the proxy's turn-by-turn handling per connection turns a + // single slow operation into head-of-line blocking for the whole volume. The + // extra connections are established lazily, only while requests are + // outstanding. The proxy resolves NFSv3 file handles per mount, not per + // connection, so they need no extra MOUNT. + "nconnect=" + strconv.Itoa(nfsConnections), + // disable caching so that pause/resume works correctly "noac", "lookupcache=none", diff --git a/packages/envd/internal/api/init_test.go b/packages/envd/internal/api/init_test.go index 0951e832a19..c6bd778b359 100644 --- a/packages/envd/internal/api/init_test.go +++ b/packages/envd/internal/api/init_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -635,6 +636,26 @@ func TestShouldRemountNFS(t *testing.T) { } } +// nconnect must be present and inside the kernel's accepted range: an out-of-range +// value fails the whole mount, which would break every volume. +func TestNFSOptionsCarryAValidNconnect(t *testing.T) { + t.Parallel() + + var nconnect string + for _, option := range strings.Split(nfsOptions, ",") { + if value, ok := strings.CutPrefix(option, "nconnect="); ok { + nconnect = value + } + } + + require.NotEmpty(t, nconnect, "nfsOptions must set nconnect so one slow operation does not block the mount's other requests") + + connections, err := strconv.Atoi(nconnect) + require.NoError(t, err) + assert.GreaterOrEqual(t, connections, 1) + assert.LessOrEqual(t, connections, 16, "the kernel accepts nconnect values of 1-16 and fails the mount otherwise") +} + type fakeCgroupManager struct { mu sync.Mutex frozen []cgroups.ProcessType