fix: harden SSH tunnel lifecycle

packages/ssh/src/tunnel.test.ts

@@ -392,4 +392,97 @@ describe("ssh tunnel scripts", () => {
       assert.equal(tunnelKillCount, 1);
     }).pipe(Effect.provide(layer), Effect.scoped);
   });
+
+  it.effect("keeps one authoritative tunnel across concurrent ensures", () => {
+    let tunnelSpawnCount = 0;
+    const spawner = ChildProcessSpawner.make((command) =>
+      Effect.sync(() => {
+        const args = commandArgs(command);
+        if (args.includes("-N")) {
+          tunnelSpawnCount += 1;
+          return makeRunningProcess(() => undefined);
+        }
+        return makeSuccessfulProcess('{"remotePort":3773,"serverKind":"external"}\n');
+      }),
+    );
+    const layer = Layer.mergeAll(
+      NodeServices.layer,
+      Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
+      Layer.succeed(HttpClient.HttpClient, testHttpClient),
+      Layer.succeed(NetService.NetService, testNetService),
+      SshPasswordPrompt.disabledLayer,
+      SshEnvironmentManager.layer(),
+    );
+    const target = {
+      alias: "devbox",
+      hostname: "devbox.example.com",
+      username: "julius",
+      port: 2222,
+    } as const;
+
+    return Effect.gen(function* () {
+      const manager = yield* SshEnvironmentManager;
+      const environments = yield* Effect.all(
+        Array.from({ length: 20 }, () => manager.ensureEnvironment(target)),
+        { concurrency: "unbounded" },
+      );
+      assert.equal(tunnelSpawnCount, 1);
+      assert.equal(new Set(environments.map((environment) => environment.httpBaseUrl)).size, 1);
+    }).pipe(Effect.provide(layer), Effect.scoped);
+  });
+
+  it.effect("does not grow tunnels or stop an external backend across 100 reconnects", () => {
+    let activeTunnels = 0;
+    let maximumActiveTunnels = 0;
+    let tunnelSpawnCount = 0;
+    let tunnelKillCount = 0;
+    let remoteStopCount = 0;
+    const spawner = ChildProcessSpawner.make((command) =>
+      Effect.sync(() => {
+        const args = commandArgs(command);
+        if (args.includes("-N")) {
+          activeTunnels += 1;
+          maximumActiveTunnels = Math.max(maximumActiveTunnels, activeTunnels);
+          tunnelSpawnCount += 1;
+          return makeRunningProcess(() => {
+            activeTunnels -= 1;
+            tunnelKillCount += 1;
+          });
+        }
+        if (args.includes("sh") && !args.includes("--")) {
+          remoteStopCount += 1;
+          return makeSuccessfulProcess('{"stopped":true}\n');
+        }
+        return makeSuccessfulProcess('{"remotePort":3773,"serverKind":"external"}\n');
+      }),
+    );
+    const layer = Layer.mergeAll(
+      NodeServices.layer,
+      Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
+      Layer.succeed(HttpClient.HttpClient, testHttpClient),
+      Layer.succeed(NetService.NetService, testNetService),
+      SshPasswordPrompt.disabledLayer,
+      SshEnvironmentManager.layer(),
+    );
+    const target = {
+      alias: "devbox",
+      hostname: "devbox.example.com",
+      username: "julius",
+      port: 2222,
+    } as const;
+
+    return Effect.gen(function* () {
+      const manager = yield* SshEnvironmentManager;
+      for (let reconnect = 0; reconnect < 100; reconnect += 1) {
+        yield* manager.ensureEnvironment(target);
+        assert.equal(activeTunnels, 1);
+        yield* manager.disconnectEnvironment(target);
+        assert.equal(activeTunnels, 0);
+      }
+      assert.equal(tunnelSpawnCount, 100);
+      assert.equal(tunnelKillCount, 100);
+      assert.equal(maximumActiveTunnels, 1);
+      assert.equal(remoteStopCount, 0);
+    }).pipe(Effect.provide(layer), Effect.scoped);
+  });
 });

packages/ssh/src/tunnel.ts

@@ -52,7 +52,8 @@ import {
 export const DEFAULT_REMOTE_PORT = 3773;
 const REMOTE_PORT_SCAN_WINDOW = 200;
 const SSH_READY_TIMEOUT_MS = 20_000;
-const SSH_READY_PROBE_TIMEOUT_MS = 1_000;
+const SSH_READY_PROBE_TIMEOUT_MS = 5_000;
+const SSH_EXISTING_TUNNEL_READY_TIMEOUT_MS = 15_000;
 const TUNNEL_SHUTDOWN_TIMEOUT_MS = 2_000;
 const REMOTE_READY_TIMEOUT_MS = 15_000;
 const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000;
@@ -1371,29 +1372,33 @@ const makeSshEnvironmentManager = Effect.fn("ssh/tunnel.SshEnvironmentManager.ma
         });
         tunnels.delete(tunnelEntry.key);
         const authSecret = authSecrets.get(tunnelEntry.key) ?? null;
+        const stopOwnedRemoteServer =
+          tunnelEntry.remoteServerKind === "external"
+            ? Effect.void
+            : stopRemoteServer(
+                tunnelEntry.target,
+                authSecret === null
+                  ? {
+                      batchMode: "yes",
+                      interactiveAuth: false,
+                    }
+                  : {
+                      authSecret,
+                      batchMode: "no",
+                      interactiveAuth: true,
+                    },
+              ).pipe(
+                Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawnerService),
+                Effect.provideService(FileSystem.FileSystem, fileSystemService),
+                Effect.provideService(Path.Path, pathService),
+              );
         yield* Effect.all(
           [
             tunnelEntry.process.kill({
               killSignal: "SIGTERM",
               forceKillAfter: TUNNEL_SHUTDOWN_TIMEOUT_MS,
             }),
-            stopRemoteServer(
-              tunnelEntry.target,
-              authSecret === null
-                ? {
-                    batchMode: "yes",
-                    interactiveAuth: false,
-                  }
-                : {
-                    authSecret,
-                    batchMode: "no",
-                    interactiveAuth: true,
-                  },
-            ).pipe(
-              Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawnerService),
-              Effect.provideService(FileSystem.FileSystem, fileSystemService),
-              Effect.provideService(Path.Path, pathService),
-            ),
+            stopOwnedRemoteServer,
           ],
           { concurrency: "unbounded" },
         ).pipe(Effect.ignore);
@@ -1429,7 +1434,10 @@ const makeSshEnvironmentManager = Effect.fn("ssh/tunnel.SshEnvironmentManager.ma
         remotePort: entry.remotePort,
       });
       const readinessExit = yield* Effect.exit(
-        waitForHttpReady({ baseUrl: entry.httpBaseUrl, timeoutMs: 2_000 }),
+        waitForHttpReady({
+          baseUrl: entry.httpBaseUrl,
+          timeoutMs: SSH_EXISTING_TUNNEL_READY_TIMEOUT_MS,
+        }),
       );
       if (Exit.isSuccess(readinessExit)) {
         yield* Effect.logDebug("ssh.environment.tunnel.reused", {