diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 877317c72a0..f696de51332 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -757,6 +757,15 @@ func writeInitError(w http.ResponseWriter, logger zerolog.Logger, err error) { w.Write([]byte(err.Error())) } +// nfsConnections is the nconnect value: how many TCP connections the guest kernel +// may open to the NFS proxy per mount. The proxy handles one connection's requests +// in turn, so a slow operation (a READDIR of a big directory, a sync write) blocks +// every other request queued behind it on that connection; spreading the mount's +// requests over several connections keeps unrelated work moving. The proxy serves +// every connection in its own goroutine, so a handful of connections costs it +// little; more would mostly add guest-side sockets. The kernel accepts 1-16. +const nfsConnections = 8 + var nfsOptions = strings.Join([]string{ // wait for data to be sent to proxy server before returning. // async might cause issues if the sandbox is shut down suddenly. @@ -771,6 +780,14 @@ var nfsOptions = strings.Join([]string{ "nfsvers=3", // nfs proxy is nfs version 3 "noacl", // no reason for acl in the sandbox + // Without this the kernel client multiplexes every request of a mount over one + // TCP connection, and the proxy's turn-by-turn handling per connection turns a + // single slow operation into head-of-line blocking for the whole volume. The + // extra connections are established lazily, only while requests are + // outstanding. The proxy resolves NFSv3 file handles per mount, not per + // connection, so they need no extra MOUNT. + "nconnect=" + strconv.Itoa(nfsConnections), + // disable caching so that pause/resume works correctly "noac", "lookupcache=none", diff --git a/packages/envd/internal/api/init_test.go b/packages/envd/internal/api/init_test.go index 7c1f4a5eb47..f3b28500d7b 100644 --- a/packages/envd/internal/api/init_test.go +++ b/packages/envd/internal/api/init_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -636,6 +637,26 @@ func TestShouldRemountNFS(t *testing.T) { } } +// nconnect must be present and inside the kernel's accepted range: an out-of-range +// value fails the whole mount, which would break every volume. +func TestNFSOptionsCarryAValidNconnect(t *testing.T) { + t.Parallel() + + var nconnect string + for _, option := range strings.Split(nfsOptions, ",") { + if value, ok := strings.CutPrefix(option, "nconnect="); ok { + nconnect = value + } + } + + require.NotEmpty(t, nconnect, "nfsOptions must set nconnect so one slow operation does not block the mount's other requests") + + connections, err := strconv.Atoi(nconnect) + require.NoError(t, err) + assert.GreaterOrEqual(t, connections, 1) + assert.LessOrEqual(t, connections, 16, "the kernel accepts nconnect values of 1-16 and fails the mount otherwise") +} + type fakeCgroupManager struct { mu sync.Mutex frozen []cgroups.ProcessType