Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions packages/envd/internal/api/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,15 @@ func writeInitError(w http.ResponseWriter, logger zerolog.Logger, err error) {
w.Write([]byte(err.Error()))
}

// nfsConnections is the nconnect value: how many TCP connections the guest kernel
// may open to the NFS proxy per mount. The proxy handles one connection's requests
// in turn, so a slow operation (a READDIR of a big directory, a sync write) blocks
// every other request queued behind it on that connection; spreading the mount's
// requests over several connections keeps unrelated work moving. The proxy serves
// every connection in its own goroutine, so a handful of connections costs it
// little; more would mostly add guest-side sockets. The kernel accepts 1-16.
const nfsConnections = 8

var nfsOptions = strings.Join([]string{
// wait for data to be sent to proxy server before returning.
// async might cause issues if the sandbox is shut down suddenly.
Expand All @@ -771,6 +780,14 @@ var nfsOptions = strings.Join([]string{
"nfsvers=3", // nfs proxy is nfs version 3
"noacl", // no reason for acl in the sandbox

// Without this the kernel client multiplexes every request of a mount over one
// TCP connection, and the proxy's turn-by-turn handling per connection turns a
// single slow operation into head-of-line blocking for the whole volume. The
// extra connections are established lazily, only while requests are
// outstanding. The proxy resolves NFSv3 file handles per mount, not per
// connection, so they need no extra MOUNT.
"nconnect=" + strconv.Itoa(nfsConnections),

// disable caching so that pause/resume works correctly
"noac",
"lookupcache=none",
Expand Down
21 changes: 21 additions & 0 deletions packages/envd/internal/api/init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
Expand Down Expand Up @@ -636,6 +637,26 @@ func TestShouldRemountNFS(t *testing.T) {
}
}

// nconnect must be present and inside the kernel's accepted range: an out-of-range
// value fails the whole mount, which would break every volume.
func TestNFSOptionsCarryAValidNconnect(t *testing.T) {
t.Parallel()

var nconnect string
for _, option := range strings.Split(nfsOptions, ",") {
if value, ok := strings.CutPrefix(option, "nconnect="); ok {
nconnect = value
}
}

require.NotEmpty(t, nconnect, "nfsOptions must set nconnect so one slow operation does not block the mount's other requests")

connections, err := strconv.Atoi(nconnect)
require.NoError(t, err)
assert.GreaterOrEqual(t, connections, 1)
assert.LessOrEqual(t, connections, 16, "the kernel accepts nconnect values of 1-16 and fails the mount otherwise")
}

type fakeCgroupManager struct {
mu sync.Mutex
frozen []cgroups.ProcessType
Expand Down