From dd9d4ebf6dfe37caa8f29d7dc9716d4ed99b9a4c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:06:39 +0000 Subject: [PATCH 01/16] feat(sdk): add SandboxUnreachableError for an unreachable sandbox Raise SandboxUnreachableError (JS) / SandboxUnreachableException (Python) when the proxy reports the sandbox running but envd's port not open (e.g. the sandbox's network is down), or when a request fails at the connection level and the follow-up health probe gets no answer either. Connection establishment failures now run the same health probe as dropped connections. Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 6 + packages/js-sdk/src/envd/api.ts | 125 ++++++++----- packages/js-sdk/src/envd/rpc.ts | 130 ++++++++++++-- packages/js-sdk/src/errors.ts | 44 +++++ packages/js-sdk/src/index.ts | 1 + packages/js-sdk/src/sandbox/index.ts | 21 ++- .../tests/envd/handleEnvdApiError.test.ts | 112 +++++++++++- .../js-sdk/tests/envd/handleRpcError.test.ts | 125 ++++++++++++- packages/python-sdk/e2b/__init__.py | 2 + packages/python-sdk/e2b/envd/api.py | 93 ++++++---- packages/python-sdk/e2b/envd/rpc.py | 43 +++-- packages/python-sdk/e2b/exceptions.py | 36 ++++ .../sandbox_async/filesystem/filesystem.py | 8 +- packages/python-sdk/e2b/sandbox_async/main.py | 7 +- .../e2b/sandbox_sync/filesystem/filesystem.py | 8 +- packages/python-sdk/e2b/sandbox_sync/main.py | 7 +- .../tests/test_envd_api_exception.py | 167 ++++++++++++++++++ .../tests/test_envd_rpc_exception.py | 77 +++++++- 18 files changed, 885 insertions(+), 127 deletions(-) create mode 100644 .changeset/sandbox-unreachable-error.md diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md new file mode 100644 index 0000000000..ecf2c0a2d2 --- /dev/null +++ b/.changeset/sandbox-unreachable-error.md @@ -0,0 +1,6 @@ +--- +'@e2b/python-sdk': patch +'e2b': patch +--- + +Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when the sandbox cannot be reached while it is not confirmed to be stopped: the proxy in front of the sandbox reports it running but envd's port not open (e.g. the sandbox's network is down), or a request to the sandbox fails at the connection level and the follow-up health probe gets no answer either. Previously these surfaced as a sandbox-timeout `TimeoutError`/`TimeoutException`, `isRunning()`/`is_running()` returned `false` for a running sandbox and `commands.run()` reported `SandboxNotFoundError`. Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index 673648c886..b3cdbd9703 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -9,15 +9,20 @@ import { NotFoundError, NotEnoughSpaceError, SandboxNotFoundError, - formatSandboxTimeoutError, + formatSandboxUnavailableError, + isSandboxPortClosedMessage, AuthenticationError, RateLimitError, - TimeoutError, } from '../errors' import { StartResponse, ConnectResponse } from './process/process_pb' import { Code, ConnectError } from '@connectrpc/connect' import { WatchDirResponse } from './filesystem/filesystem_pb' -import { isConnectionTerminatedMessage, SandboxHealthCheck } from './rpc' +import { + isConnectionTerminatedMessage, + resolveTransportFailure, + SandboxHealthCheck, +} from './rpc' +import { isConnectionError } from '../retry' type ApiError = { message?: string } | string @@ -27,7 +32,7 @@ const DEFAULT_ERROR_MAP: Record Error> = { 404: (message) => new NotFoundError(message), 429: (message) => new RateLimitError(`${message}: The requests are being rate limited.`), - 502: formatSandboxTimeoutError, + 502: formatSandboxUnavailableError, 507: (message) => new NotEnoughSpaceError(message), } @@ -37,59 +42,90 @@ const HEALTH_CHECK_TIMEOUT_MS = 5_000 * Probes the sandbox's envd health endpoint. * * @param envdApi - The envd API client of the sandbox. - * @returns `true` if the sandbox is running, `false` if it is not, `undefined` if its state could not be determined. + * @returns `true` if the sandbox is running, `false` if it is not, `undefined` if it answered but its state could not be determined. + * @throws The `fetch` failure when the sandbox could not be reached (no answer within the probe timeout), or a `SandboxUnreachableError` when the proxy reports the sandbox running but envd's port not open. */ export async function checkSandboxHealth( envdApi: EnvdApiClient ): Promise { - try { - const res = await envdApi.api.GET('/health', { - signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS), - }) + const res = await envdApi.api.GET('/health', { + signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS), + }) - if (res.response.status === 502) { - return false + if (res.response.status === 502) { + const message = await getEnvdApiErrorMessage(res) + if (isSandboxPortClosedMessage(message)) { + throw formatSandboxUnavailableError(message) } - if (res.response.ok) { - return true - } - - return undefined - } catch { - return undefined + return false } + if (res.response.ok) { + return true + } + + return undefined } /** - * Handles transport-level fetch failures from envd API calls. When the connection was - * dropped mid-request, probes the sandbox health to tell apart the sandbox being killed - * from a transient network failure (e.g. a load balancer dropping the connection). + * Checks whether a `fetch` rejection is a transport-level failure of the request to + * the sandbox: the connection could not be established or was dropped mid-request + * (the latter surfaces with runtime-specific wording, e.g. undici's `terminated`). + */ +export function isFetchTransportFailure(err: unknown): err is Error { + return ( + err instanceof Error && + (isConnectionTerminatedMessage(err.message) || isConnectionError(err)) + ) +} + +/** + * Handles transport-level fetch failures from envd API calls. When the connection to + * the sandbox could not be established or was dropped mid-request, probes the sandbox + * health to tell apart the sandbox being killed from the sandbox being unreachable or + * a transient network failure (e.g. a load balancer dropping the connection). * * @param err - The caught error, expected to be a fetch transport failure. - * @param checkHealth - Probe returning whether the sandbox is running, or `undefined` when unknown. - * @returns A `TimeoutError` when the connection was terminated mid-request and the sandbox is confirmed gone, or the original error otherwise. + * @param checkHealth - Probe resolving to whether the sandbox is running (`undefined` when unknown) and rejecting when it cannot be reached. + * @returns A `TimeoutError` when the sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, or the original error otherwise. */ export async function handleEnvdApiFetchError( err: unknown, checkHealth?: SandboxHealthCheck ): Promise { - // A connection dropped mid-body surfaces as a fetch failure whose message varies - // by runtime (e.g. undici's 'terminated'); match every known variant. - if (err instanceof Error && isConnectionTerminatedMessage(err.message)) { - const running = checkHealth - ? await checkHealth().catch(() => undefined) - : undefined - - if (running === false) { - return new TimeoutError( - `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.` - ) + if (checkHealth && isFetchTransportFailure(err)) { + const resolved = await resolveTransportFailure(err, checkHealth) + if (resolved) { + return resolved } } return err as Error } +/** + * Extracts the error message of a non-2xx envd API response: the `message` of its + * JSON body, else its text body, else its status text. + */ +export async function getEnvdApiErrorMessage(res: { + error?: ApiError + response: Response +}): Promise { + let message = + (typeof res.error === 'string' ? res.error : res.error?.message) ?? '' + + // openapi-fetch consumes the body when parsing the error, except for + // responses without content + if (!message && !res.response.bodyUsed) { + try { + message = await res.response.text() + } catch { + // ignore unreadable bodies + } + } + + return message || res.response.statusText +} + /** * Handles errors from envd API responses by mapping HTTP status codes to specific error types. * @@ -111,20 +147,7 @@ export async function handleEnvdApiError( return } - let message = - (typeof res.error === 'string' ? res.error : res.error?.message) ?? '' - - // openapi-fetch consumes the body when parsing the error, except for - // responses without content - if (!message && !res.response.bodyUsed) { - try { - message = await res.response.text() - } catch { - // ignore unreadable bodies - } - } - - message = message || res.response.statusText + const message = await getEnvdApiErrorMessage(res) // Check if a custom error mapping is provided for this error code if (errorMap && res.response.status in errorMap) { @@ -150,6 +173,9 @@ export async function handleProcessStartEvent( } catch (err) { if (err instanceof ConnectError) { if (err.code === Code.Unavailable) { + if (isSandboxPortClosedMessage(err.rawMessage)) { + throw formatSandboxUnavailableError(err.rawMessage) + } throw new SandboxNotFoundError( 'Sandbox is probably not running anymore' ) @@ -175,6 +201,9 @@ export async function handleWatchDirStartEvent( } catch (err) { if (err instanceof ConnectError) { if (err.code === Code.Unavailable) { + if (isSandboxPortClosedMessage(err.rawMessage)) { + throw formatSandboxUnavailableError(err.rawMessage) + } throw new SandboxNotFoundError( 'Sandbox is probably not running anymore' ) diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 6c4bb342cc..30711771c2 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -5,18 +5,21 @@ import { compareVersions } from 'compare-versions' import { defaultUsername } from '../connectionConfig' import { AuthenticationError, - formatSandboxTimeoutError, + formatSandboxUnavailableError, InvalidArgumentError, NotFoundError, RateLimitError, SandboxError, + SandboxUnreachableError, TimeoutError, } from '../errors' +import { isConnectionError } from '../retry' import { ENVD_DEFAULT_USER } from './versions' /** - * Result of a sandbox health probe: `true` if the sandbox is running, `false` if it is not, - * `undefined` if its state could not be determined. + * Sandbox health probe: resolves to `true` if the sandbox is running, `false` if it + * is not, `undefined` if it answered but its state could not be determined, and + * rejects when the sandbox could not be reached at all (no answer). */ export type SandboxHealthCheck = () => Promise @@ -68,6 +71,71 @@ function isConnectionTerminatedError(err: unknown): boolean { ) } +/** + * Checks whether the error is a transport-level failure of the request to the sandbox: + * the connection could not be established (connect wraps the `fetch` rejection as + * `Code.Unknown` with the failure as `cause`) or was dropped mid-request. Errors + * envd responded with are never transport failures. + */ +export function isTransportFailure(err: unknown): boolean { + if (isConnectionTerminatedError(err)) { + return true + } + + return ( + err instanceof ConnectError && + err.code === Code.Unknown && + isConnectionError(err.cause) + ) +} + +/** + * Builds the error for a request that failed at the connection level when the + * follow-up health probe got no answer from the sandbox either. A probe answered + * by the proxy with the sandbox running but envd's port not open is already a + * `SandboxUnreachableError` and is kept, with the failed request as its cause. + */ +export function formatSandboxUnreachableError( + err: Error, + probeErr?: unknown +): Error { + if (probeErr instanceof SandboxUnreachableError) { + probeErr.cause = err + return probeErr + } + return new SandboxUnreachableError( + `${err.message}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.getInfo()' and retry the request.`, + { cause: err } + ) +} + +/** + * Runs the health probe for a request that failed at the connection level and + * returns the typed error for the outcome: a `TimeoutError` when the sandbox is + * confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, + * or `undefined` when the sandbox answered (the failure was transient and the + * original error applies). + */ +export async function resolveTransportFailure( + err: Error, + checkHealth: SandboxHealthCheck +): Promise { + let running: boolean | undefined + try { + running = await checkHealth() + } catch (probeErr) { + return formatSandboxUnreachableError(err, probeErr) + } + + if (running === false) { + return new TimeoutError( + `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.` + ) + } + + return undefined +} + const DEFAULT_ERROR_MAP: Partial Error>> = { [Code.InvalidArgument]: (message) => new InvalidArgumentError(message), [Code.Unauthenticated]: (message) => new AuthenticationError(message), @@ -76,7 +144,7 @@ const DEFAULT_ERROR_MAP: Partial Error>> = { new RateLimitError( `${message}: Rate limit exceeded, please try again later.` ), - [Code.Unavailable]: formatSandboxTimeoutError, + [Code.Unavailable]: formatSandboxUnavailableError, [Code.Canceled]: (message) => new TimeoutError( `${message}: This error is likely due to exceeding 'requestTimeoutMs'. You can pass the request timeout value as an option when making the request.` @@ -117,14 +185,16 @@ export function handleRpcError( } /** - * Like {@link handleRpcError}, but when the connection to the sandbox was dropped - * mid-request it probes the sandbox health to tell apart the sandbox being killed - * from a transient network failure (e.g. a load balancer dropping the connection). - * When the probe confirms the sandbox is gone, a `TimeoutError` is returned — - * consistent with how requests to an already-dead sandbox surface. + * Like {@link handleRpcError}, but when the request failed at the connection level + * (the connection to the sandbox could not be established or was dropped + * mid-request) it probes the sandbox health to tell apart the sandbox being killed + * from the sandbox being unreachable or a transient network failure (e.g. a load + * balancer dropping the connection). When the probe confirms the sandbox is gone, a + * `TimeoutError` is returned — consistent with how requests to an already-dead + * sandbox surface; when the probe gets no answer either, a `SandboxUnreachableError`. * * @param err - The caught error, expected to be a `ConnectError` from the gRPC transport. - * @param checkHealth - Probe returning whether the sandbox is running, or `undefined` when unknown. + * @param checkHealth - Probe resolving to whether the sandbox is running (`undefined` when unknown) and rejecting when it cannot be reached. * @param errorMap - Optional map of gRPC `Code` values to error factory functions that override the defaults. * @returns The corresponding `Error` instance. */ @@ -133,13 +203,13 @@ export async function handleRpcErrorWithHealthCheck( checkHealth?: SandboxHealthCheck, errorMap?: Partial Error>> ): Promise { - if (isConnectionTerminatedError(err) && checkHealth) { - const running = await checkHealth().catch(() => undefined) - - if (running === false) { - return new TimeoutError( - `${(err as ConnectError).message}: The sandbox was killed or reached its end of life while the request was in flight.` - ) + if (checkHealth && isTransportFailure(err)) { + const resolved = await resolveTransportFailure( + err as ConnectError, + checkHealth + ) + if (resolved) { + return resolved } } @@ -180,3 +250,29 @@ export function authenticationHeader( return { Authorization: `Basic ${encoded}` } } + +/** + * Rejects a plain (non-Connect-encoded) 502 answered by the proxy in front of envd — + * the sandbox is gone or envd's port is not open — as a `ConnectError` carrying the + * proxy's message. connect reads that message for unary RPCs only; a streaming RPC + * would otherwise surface it as a bare `HTTP 502`. + */ +export async function rejectProxyUnavailableResponse( + res: Response +): Promise { + if (res.status !== 502) { + return res + } + + let message = '' + try { + const body = await res.json() + if (typeof body?.message === 'string') { + message = body.message + } + } catch { + // not a JSON body + } + + throw new ConnectError(message || `HTTP ${res.status}`, Code.Unavailable) +} diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index f148935067..fa5dc6c32e 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -1,3 +1,11 @@ +// Message of the 502 the proxy in front of envd answers with when the sandbox +// is running but nothing listens on envd's port — e.g. its network is down. +const SANDBOX_PORT_CLOSED_MESSAGE = 'port is not open' + +export function isSandboxPortClosedMessage(message: string): boolean { + return message.includes(SANDBOX_PORT_CLOSED_MESSAGE) +} + // This is the message for the sandbox timeout error when the response code is 502/Unavailable export function formatSandboxTimeoutError(message: string) { return new TimeoutError( @@ -5,6 +13,20 @@ export function formatSandboxTimeoutError(message: string) { ) } +/** + * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox is + * gone (killed or timed out) unless the proxy reports it running with envd's port + * not open, in which case the sandbox is unreachable. + */ +export function formatSandboxUnavailableError(message: string): Error { + if (isSandboxPortClosedMessage(message)) { + return new SandboxUnreachableError( + `${message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.getInfo()' and retry the request.` + ) + } + return formatSandboxTimeoutError(message) +} + /** * Base class for all sandbox errors. * @@ -95,6 +117,28 @@ export class SandboxNotFoundError extends NotFoundError { } } +/** + * Thrown when the sandbox could not be reached while not confirmed to be + * stopped: either the proxy in front of the sandbox reports it running but + * envd's port not open (e.g. the sandbox's network is down), or a request + * failed at the connection level (the connection could not be established or + * was dropped mid-request) and a follow-up health probe got no answer from the + * sandbox either. + * + * This usually means the sandbox's network is down, envd inside it is not up + * (yet), or a transient network issue between the client and the sandbox. + * Check the sandbox state with `Sandbox.getInfo()` and retry the request. + */ +export class SandboxUnreachableError extends SandboxError { + constructor(message: string, options?: ErrorOptions) { + super(message) + this.name = 'SandboxUnreachableError' + if (options?.cause !== undefined) { + this.cause = options.cause + } + } +} + /** * Thrown when authentication fails. */ diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index afe6917fcf..d9d26c28d1 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -18,6 +18,7 @@ export { NotFoundError, SandboxError, SandboxNotFoundError, + SandboxUnreachableError, TemplateError, TimeoutError, RateLimitError, diff --git a/packages/js-sdk/src/sandbox/index.ts b/packages/js-sdk/src/sandbox/index.ts index 5f0c6f2f53..38047804a1 100644 --- a/packages/js-sdk/src/sandbox/index.ts +++ b/packages/js-sdk/src/sandbox/index.ts @@ -6,7 +6,12 @@ import { defaultUsername, Username, } from '../connectionConfig' -import { EnvdApiClient, handleEnvdApiError } from '../envd/api' +import { + EnvdApiClient, + getEnvdApiErrorMessage, + handleEnvdApiError, +} from '../envd/api' +import { rejectProxyUnavailableResponse } from '../envd/rpc' import { createEnvdFetch, createEnvdRpcFetch } from '../envd/http2' import { createRpcLogger } from '../logs' import { Commands, Pty } from './commands' @@ -30,7 +35,13 @@ import { } from './sandboxApi' import { getSignature } from './signature' import { compareVersions } from 'compare-versions' -import { InvalidArgumentError, SandboxError, TemplateError } from '../errors' +import { + InvalidArgumentError, + SandboxError, + TemplateError, + formatSandboxUnavailableError, + isSandboxPortClosedMessage, +} from '../errors' import { ENVD_DEBUG_FALLBACK, ENVD_DEFAULT_USER } from '../envd/versions' import { shellQuote } from '../utils' @@ -194,7 +205,7 @@ export class Sandbox extends SandboxApi { redirect: 'follow', } - return envdRpcFetch(url, options) + return envdRpcFetch(url, options).then(rejectProxyUnavailableResponse) }, }) @@ -561,6 +572,10 @@ export class Sandbox extends SandboxApi { }) if (res.response.status == 502) { + const message = await getEnvdApiErrorMessage(res) + if (isSandboxPortClosedMessage(message)) { + throw formatSandboxUnavailableError(message) + } return false } diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts index 4140741946..a749bef146 100644 --- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts +++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts @@ -1,5 +1,9 @@ import { assert, test, describe } from 'vitest' -import { handleEnvdApiError, handleEnvdApiFetchError } from '../../src/envd/api' +import { + checkSandboxHealth, + handleEnvdApiError, + handleEnvdApiFetchError, +} from '../../src/envd/api' import { AuthenticationError, InvalidArgumentError, @@ -7,9 +11,28 @@ import { NotFoundError, RateLimitError, SandboxError, + SandboxUnreachableError, TimeoutError, } from '../../src/errors' +// undici surfaces a refused connection as `fetch failed` with the syscall error as `cause` +function connectRefused(): TypeError { + return Object.assign(new TypeError('fetch failed'), { + cause: Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }), + }) +} + +// Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +const PORT_CLOSED = 'The sandbox is running but port is not open' + +function healthApi(status: number, error?: { message?: string } | string) { + return { + api: { GET: async () => createMockResponse(status, error) }, + } as unknown as Parameters[0] +} + function createMockResponse( status: number, error?: { message?: string } | string @@ -83,6 +106,13 @@ describe('handleEnvdApiError', () => { assert.include(err?.message, 'rate limited') }) + test('returns SandboxUnreachableError for 502 with the sandbox running but its port closed', async () => { + const res = createMockResponse(502, { message: PORT_CLOSED }) + const err = await handleEnvdApiError(res) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err!.message, 'envd inside the sandbox could not be reached') + }) + test('returns TimeoutError for 502', async () => { const res = createMockResponse(502, { message: 'Bad gateway' }) const err = await handleEnvdApiError(res) @@ -151,4 +181,84 @@ describe('handleEnvdApiFetchError', () => { const err = await handleEnvdApiFetchError(original) assert.strictEqual(err, original) }) + + test('returns a SandboxUnreachableError when the health check itself fails', async () => { + const original = new TypeError('terminated') + const err = await handleEnvdApiFetchError(original, async () => { + throw new TypeError('fetch failed') + }) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, 'could not be reached') + assert.strictEqual(err.cause, original) + }) + + test('runs the health check when the connection could not be established', async () => { + let called = false + const original = connectRefused() + const err = await handleEnvdApiFetchError(original, async () => { + called = true + return true + }) + assert.isTrue(called) + assert.strictEqual(err, original) + }) + + test('returns a TimeoutError for a refused connection when the sandbox is gone', async () => { + const err = await handleEnvdApiFetchError( + connectRefused(), + async () => false + ) + assert.instanceOf(err, TimeoutError) + }) + + test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { + const err = await handleEnvdApiFetchError(connectRefused(), async () => { + throw new TypeError('fetch failed') + }) + assert.instanceOf(err, SandboxUnreachableError) + }) + + test('does not run the health check for a non-connection fetch failure', async () => { + const original = new TypeError('fetch failed') + const err = await handleEnvdApiFetchError(original, async () => { + throw new Error('health check should not run') + }) + assert.strictEqual(err, original) + }) +}) + +describe('checkSandboxHealth', () => { + test('reports the sandbox running', async () => { + assert.strictEqual(await checkSandboxHealth(healthApi(204)), true) + }) + + test('reports the sandbox gone for 502', async () => { + const api = healthApi(502, { message: 'The sandbox was not found' }) + assert.strictEqual(await checkSandboxHealth(api), false) + }) + + test('is inconclusive for other statuses', async () => { + assert.strictEqual(await checkSandboxHealth(healthApi(500)), undefined) + }) + + test('throws SandboxUnreachableError for 502 with the sandbox running but its port closed', async () => { + const api = healthApi(502, { message: PORT_CLOSED }) + try { + await checkSandboxHealth(api) + assert.fail('expected the probe to throw') + } catch (err) { + assert.instanceOf(err, SandboxUnreachableError) + assert.include((err as Error).message, PORT_CLOSED) + } + }) + + test('keeps the probe SandboxUnreachableError with the failed request as cause', async () => { + const original = connectRefused() + const err = await handleEnvdApiFetchError(original, () => + checkSandboxHealth(healthApi(502, { message: PORT_CLOSED })) + ) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, PORT_CLOSED) + assert.strictEqual(err.cause, original) + }) }) diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index f90f893e64..caf14667f2 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -3,6 +3,8 @@ import { Code, ConnectError } from '@connectrpc/connect' import { handleRpcError, handleRpcErrorWithHealthCheck, + isTransportFailure, + rejectProxyUnavailableResponse, } from '../../src/envd/rpc' import { AuthenticationError, @@ -10,9 +12,29 @@ import { NotFoundError, RateLimitError, SandboxError, + SandboxUnreachableError, TimeoutError, } from '../../src/errors' +// connect wraps a `fetch` rejection as `Code.Unknown` with the failure as `cause` +function connectRefused(): ConnectError { + const cause = Object.assign(new TypeError('fetch failed'), { + cause: Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }), + }) + return new ConnectError( + cause.message, + Code.Unknown, + undefined, + undefined, + cause + ) +} + +// Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +const PORT_CLOSED = 'The sandbox is running but port is not open' + describe('handleRpcError', () => { test('returns InvalidArgumentError for InvalidArgument', () => { const err = handleRpcError(new ConnectError('bad', Code.InvalidArgument)) @@ -37,6 +59,12 @@ describe('handleRpcError', () => { assert.include(err.message, 'Rate limit') }) + test('returns SandboxUnreachableError for Unavailable with the sandbox running but its port closed', () => { + const err = handleRpcError(new ConnectError(PORT_CLOSED, Code.Unavailable)) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, 'envd inside the sandbox could not be reached') + }) + test('returns TimeoutError for Unavailable', () => { const err = handleRpcError(new ConnectError('gone', Code.Unavailable)) assert.instanceOf(err, TimeoutError) @@ -116,13 +144,71 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.notInclude(err.message, 'killed') }) - test('falls back to the generic mapping when the health check itself fails', async () => { - const err = await handleRpcErrorWithHealthCheck(terminated(), async () => { + test('keeps the probe SandboxUnreachableError with the failed request as cause', async () => { + const original = terminated() + const probeErr = new SandboxUnreachableError(PORT_CLOSED) + const err = await handleRpcErrorWithHealthCheck(original, async () => { + throw probeErr + }) + assert.strictEqual(err, probeErr) + assert.strictEqual(err.cause, original) + }) + + test('returns a SandboxUnreachableError when the health check itself fails', async () => { + const original = terminated() + const err = await handleRpcErrorWithHealthCheck(original, async () => { throw new Error('health check failed') }) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, 'could not be reached') + assert.strictEqual(err.cause, original) + }) + + test('treats a refused connection as a transport failure', () => { + assert.isTrue(isTransportFailure(connectRefused())) + assert.isFalse(isTransportFailure(new ConnectError('nope', Code.NotFound))) + assert.isFalse( + isTransportFailure( + new ConnectError( + 'boom', + Code.Unknown, + undefined, + undefined, + new Error('boom') + ) + ) + ) + }) + + test('runs the health check when the connection could not be established', async () => { + let called = false + const original = connectRefused() + const err = await handleRpcErrorWithHealthCheck(original, async () => { + called = true + return true + }) + assert.isTrue(called) assert.instanceOf(err, SandboxError) assert.notInstanceOf(err, TimeoutError) - assert.notInclude(err.message, 'killed') + assert.notInstanceOf(err, SandboxUnreachableError) + }) + + test('returns a TimeoutError for a refused connection when the sandbox is gone', async () => { + const err = await handleRpcErrorWithHealthCheck( + connectRefused(), + async () => false + ) + assert.instanceOf(err, TimeoutError) + }) + + test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { + const err = await handleRpcErrorWithHealthCheck( + connectRefused(), + async () => { + throw new TypeError('fetch failed') + } + ) + assert.instanceOf(err, SandboxUnreachableError) }) test('does not run the health check for other errors', async () => { @@ -138,3 +224,36 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.isFalse(called) }) }) + +describe('rejectProxyUnavailableResponse', () => { + test('passes other responses through', async () => { + const res = new Response('ok', { status: 200 }) + assert.strictEqual(await rejectProxyUnavailableResponse(res), res) + }) + + test('rejects a proxy 502 as Unavailable with the body message', async () => { + const res = new Response( + JSON.stringify({ message: PORT_CLOSED, port: 49983, code: 502 }), + { status: 502, headers: { 'Content-Type': 'application/json' } } + ) + try { + await rejectProxyUnavailableResponse(res) + assert.fail('expected a rejection') + } catch (err) { + assert.instanceOf(err, ConnectError) + assert.strictEqual((err as ConnectError).code, Code.Unavailable) + assert.strictEqual((err as ConnectError).rawMessage, PORT_CLOSED) + } + }) + + test('falls back to the status for a 502 without a JSON body', async () => { + const res = new Response('bad gateway', { status: 502 }) + try { + await rejectProxyUnavailableResponse(res) + assert.fail('expected a rejection') + } catch (err) { + assert.instanceOf(err, ConnectError) + assert.strictEqual((err as ConnectError).rawMessage, 'HTTP 502') + } + }) +}) diff --git a/packages/python-sdk/e2b/__init__.py b/packages/python-sdk/e2b/__init__.py index f8371b895f..259a1bb12d 100644 --- a/packages/python-sdk/e2b/__init__.py +++ b/packages/python-sdk/e2b/__init__.py @@ -51,6 +51,7 @@ ServiceBusyException, SandboxException, SandboxNotFoundException, + SandboxUnreachableException, TemplateException, TimeoutException, VolumeException, @@ -184,6 +185,7 @@ "NotFoundException", "FileNotFoundException", "SandboxNotFoundException", + "SandboxUnreachableException", "AuthenticationException", "GitAuthException", "GitUpstreamException", diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.py index 811d58d592..d0cf3dc815 100644 --- a/packages/python-sdk/e2b/envd/api.py +++ b/packages/python-sdk/e2b/envd/api.py @@ -3,7 +3,10 @@ from typing import Callable, Optional -from e2b.envd.rpc import format_terminated_exception +from e2b.envd.rpc import ( + format_sandbox_unreachable_exception, + format_terminated_exception, +) from e2b.exceptions import ( SandboxException, NotFoundException, @@ -11,7 +14,8 @@ InvalidArgumentException, NotEnoughSpaceException, RateLimitException, - format_sandbox_timeout_exception, + format_sandbox_unavailable_exception, + is_sandbox_port_closed_message, ) @@ -25,41 +29,63 @@ 429: lambda message: RateLimitException( f"{message}: The requests are being rate limited." ), - 502: format_sandbox_timeout_exception, + 502: format_sandbox_unavailable_exception, 507: NotEnoughSpaceException, } HEALTH_CHECK_TIMEOUT = 5 # seconds +# Raised by the pyqwest httpx adapter when a request fails at the connection +# level: the connection could not be established (``ConnectError`` / +# ``ConnectTimeout``, after the connection layer's own retries) or was dropped +# mid-request (``ReadError`` / ``WriteError`` / ``RemoteProtocolError``, e.g. an +# HTTP/2 stream reset). Read timeouts are excluded: the request reached the +# sandbox and exhausted its budget, which is a definitive result. +ENVD_API_TRANSPORT_ERRORS = ( + httpx.ConnectError, + httpx.ConnectTimeout, + httpx.ReadError, + httpx.WriteError, + httpx.RemoteProtocolError, +) + def check_sandbox_health(envd_api: httpx.Client) -> Optional[bool]: """Probe the sandbox's envd health endpoint. - :return: ``True`` if the sandbox is running, ``False`` if it is not, ``None`` if its state could not be determined. + :return: ``True`` if the sandbox is running, ``False`` if it is not, ``None`` if it answered but its state could not be determined. + :raises httpx.TransportError: When the sandbox could not be reached (no answer within the probe timeout). """ try: r = envd_api.get(ENVD_API_HEALTH_ROUTE, timeout=HEALTH_CHECK_TIMEOUT) - if r.status_code == 502: - return False - if r.is_success: - return True - return None + except httpx.TransportError: + raise except Exception: return None + return _health_status(r) async def acheck_sandbox_health(envd_api: httpx.AsyncClient) -> Optional[bool]: """Async version of :func:`check_sandbox_health`.""" try: r = await envd_api.get(ENVD_API_HEALTH_ROUTE, timeout=HEALTH_CHECK_TIMEOUT) - if r.status_code == 502: - return False - if r.is_success: - return True - return None + except httpx.TransportError: + raise except Exception: return None + return _health_status(r) + + +def _health_status(r: httpx.Response) -> Optional[bool]: + if r.status_code == 502: + message = get_message(r) + if is_sandbox_port_closed_message(message): + raise format_sandbox_unavailable_exception(message) + return False + if r.is_success: + return True + return None def handle_envd_api_transport_exception( @@ -69,12 +95,10 @@ def handle_envd_api_transport_exception( """Handle transport-level errors from envd API requests. :param e: The caught exception, expected to be a transport-level ``httpx`` error. - :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request. - :return: A ``TimeoutException`` when the connection dropped mid-request and the sandbox is confirmed gone, or the original exception unchanged otherwise. + :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a request that failed at the connection level. + :return: A ``TimeoutException`` when the request failed at the connection level and the sandbox is confirmed gone, or the original exception unchanged otherwise. """ - # A remote protocol error (e.g. an HTTP/2 stream reset) means the connection to the - # sandbox was dropped mid-request — either the sandbox died or the network failed - if isinstance(e, httpx.RemoteProtocolError): + if isinstance(e, ENVD_API_TRANSPORT_ERRORS): return format_terminated_exception(e, sandbox_running) return e @@ -84,15 +108,19 @@ def handle_envd_api_transport_exception_with_health( e: Exception, envd_api: httpx.Client, ) -> Exception: - """Like :func:`handle_envd_api_transport_exception`, but when the connection to the - sandbox was dropped mid-request it probes the sandbox health to tell apart the sandbox - being killed from a transient network failure (e.g. a load balancer dropping the connection). + """Like :func:`handle_envd_api_transport_exception`, but when the request failed at + the connection level (the connection to the sandbox could not be established or was + dropped mid-request) it probes the sandbox health to tell apart the sandbox being + killed from the sandbox being unreachable or a transient network failure (e.g. a + load balancer dropping the connection). When the probe gets no answer either, a + ``SandboxUnreachableException`` is returned. """ - sandbox_running = ( - check_sandbox_health(envd_api) - if isinstance(e, httpx.RemoteProtocolError) - else None - ) + if not isinstance(e, ENVD_API_TRANSPORT_ERRORS): + return e + try: + sandbox_running = check_sandbox_health(envd_api) + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_envd_api_transport_exception(e, sandbox_running) @@ -101,11 +129,12 @@ async def ahandle_envd_api_transport_exception_with_health( envd_api: httpx.AsyncClient, ) -> Exception: """Async version of :func:`handle_envd_api_transport_exception_with_health`.""" - sandbox_running = ( - await acheck_sandbox_health(envd_api) - if isinstance(e, httpx.RemoteProtocolError) - else None - ) + if not isinstance(e, ENVD_API_TRANSPORT_ERRORS): + return e + try: + sandbox_running = await acheck_sandbox_health(envd_api) + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_envd_api_transport_exception(e, sandbox_running) diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index 7caca5c70d..f7b0a5ec06 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -9,8 +9,9 @@ SandboxException, InvalidArgumentException, NotFoundException, + SandboxUnreachableException, TimeoutException, - format_sandbox_timeout_exception, + format_sandbox_unavailable_exception, AuthenticationException, RateLimitException, ) @@ -19,7 +20,7 @@ Code.INVALID_ARGUMENT: InvalidArgumentException, Code.UNAUTHENTICATED: AuthenticationException, Code.NOT_FOUND: NotFoundException, - Code.UNAVAILABLE: format_sandbox_timeout_exception, + Code.UNAVAILABLE: format_sandbox_unavailable_exception, Code.RESOURCE_EXHAUSTED: lambda message: RateLimitException( f"{message}: Rate limit exceeded, please try again later." ), @@ -63,7 +64,7 @@ def format_terminated_exception( e: Exception, sandbox_running: Optional[bool], ) -> Exception: - """Handle an exception for a connection to the sandbox dropped mid-request: when a + """Handle an exception for a request that failed at the connection level: when a sandbox health probe confirmed the sandbox is gone (``sandbox_running is False``), return a ``TimeoutException``; otherwise return the original error unchanged.""" if sandbox_running is False: @@ -73,6 +74,24 @@ def format_terminated_exception( return e +def format_sandbox_unreachable_exception( + e: Exception, probe_error: Optional[Exception] = None +) -> Exception: + """Build the exception for a request that failed at the connection level when the + follow-up sandbox health probe got no answer from the sandbox either. A probe + answered by the proxy with the sandbox running but envd's port not open is + already a ``SandboxUnreachableException`` and is kept, with the failed request + as its cause.""" + if isinstance(probe_error, SandboxUnreachableException): + probe_error.__cause__ = e + return probe_error + err = SandboxUnreachableException( + f"{e}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.get_info()' and retry the request." + ) + err.__cause__ = e + return err + + def handle_rpc_exception( e: Exception, error_map: Optional[dict[Code, Callable[[str], Exception]]] = None, @@ -120,16 +139,20 @@ def handle_rpc_exception_with_health( check_health: Optional[Callable[[], Optional[bool]]] = None, error_map: Optional[dict[Code, Callable[[str], Exception]]] = None, ): - """Like :func:`handle_rpc_exception`, but when the connection to the sandbox was - dropped mid-request it probes the sandbox health to tell apart the sandbox being - killed from a transient network failure (e.g. a load balancer dropping the connection). + """Like :func:`handle_rpc_exception`, but when the request failed at the connection + level (the connection to the sandbox could not be established or was dropped + mid-request) it probes the sandbox health to tell apart the sandbox being killed + from the sandbox being unreachable or a transient network failure (e.g. a load + balancer dropping the connection). When the probe confirms the sandbox is gone, a + ``TimeoutException`` is returned; when the probe gets no answer either (the health + check raises), a ``SandboxUnreachableException``. """ sandbox_running = None if check_health is not None and is_transport_failure(e): try: sandbox_running = check_health() - except Exception: - sandbox_running = None + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_rpc_exception(e, error_map, sandbox_running) @@ -143,6 +166,6 @@ async def ahandle_rpc_exception_with_health( if check_health is not None and is_transport_failure(e): try: sandbox_running = await check_health() - except Exception: - sandbox_running = None + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_rpc_exception(e, error_map, sandbox_running) diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index 4cc3abb62e..16e5cb85c9 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -1,5 +1,13 @@ from typing import Optional +# Message of the 502 the proxy in front of envd answers with when the sandbox +# is running but nothing listens on envd's port — e.g. its network is down. +SANDBOX_PORT_CLOSED_MESSAGE = "port is not open" + + +def is_sandbox_port_closed_message(message: str) -> bool: + return SANDBOX_PORT_CLOSED_MESSAGE in message + def format_sandbox_timeout_exception(message: str): return TimeoutException( @@ -7,6 +15,17 @@ def format_sandbox_timeout_exception(message: str): ) +def format_sandbox_unavailable_exception(message: str) -> Exception: + """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox is + gone (killed or timed out) unless the proxy reports it running with envd's port + not open, in which case the sandbox is unreachable.""" + if is_sandbox_port_closed_message(message): + return SandboxUnreachableException( + f"{message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.get_info()' and retry the request." + ) + return format_sandbox_timeout_exception(message) + + def format_request_timeout_error() -> Exception: return TimeoutException( "Request timed out — the 'request_timeout' option can be used to increase this timeout", @@ -88,6 +107,23 @@ class SandboxNotFoundException(NotFoundException): pass +class SandboxUnreachableException(SandboxException): + """ + Raised when the sandbox could not be reached while not confirmed to be + stopped: either the proxy in front of the sandbox reports it running but + envd's port not open (e.g. the sandbox's network is down), or a request + failed at the connection level (the connection could not be established or + was dropped mid-request) and a follow-up health probe got no answer from the + sandbox either. + + This usually means the sandbox's network is down, envd inside it is not up + (yet), or a transient network issue between the client and the sandbox. + Check the sandbox state with ``Sandbox.get_info()`` and retry the request. + """ + + pass + + class AuthenticationException(Exception): """ Raised when authentication fails. diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 76ca51e329..6482eb0e03 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -244,7 +244,7 @@ async def read( ) else: r = await client.send(request, stream=True) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( e, self._envd_api ) @@ -267,7 +267,7 @@ async def read( headers=headers, timeout=timeout, ) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( e, self._envd_api ) @@ -412,7 +412,7 @@ async def _upload_file(file): params=params, timeout=None if is_streamed else upload_timeout, ) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( e, self._envd_api ) @@ -461,7 +461,7 @@ async def _upload_file(file): None if multipart_body_is_streamed(files) else upload_timeout ), ) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( e, self._envd_api ) diff --git a/packages/python-sdk/e2b/sandbox_async/main.py b/packages/python-sdk/e2b/sandbox_async/main.py index 32dd2ba2ed..a143180665 100644 --- a/packages/python-sdk/e2b/sandbox_async/main.py +++ b/packages/python-sdk/e2b/sandbox_async/main.py @@ -13,12 +13,14 @@ from e2b.api.client.types import Unset from e2b.api.client_async import get_envd_api from e2b.connection_config import ApiParams, ConnectionConfig -from e2b.envd.api import ENVD_API_HEALTH_ROUTE, ahandle_envd_api_exception +from e2b.envd.api import ENVD_API_HEALTH_ROUTE, get_message, ahandle_envd_api_exception from e2b.envd.versions import ENVD_DEBUG_FALLBACK from e2b.exceptions import ( SandboxException, TemplateException, format_request_timeout_error, + format_sandbox_unavailable_exception, + is_sandbox_port_closed_message, ) from e2b.sandbox.commands.command_handle import CommandExitException from e2b.sandbox.main import SandboxOpts @@ -156,6 +158,9 @@ async def is_running(self, request_timeout: Optional[float] = None) -> bool: ) if r.status_code == 502: + message = get_message(r) + if is_sandbox_port_closed_message(message): + raise format_sandbox_unavailable_exception(message) return False err = await ahandle_envd_api_exception(r) diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 752d4999a7..568c26134a 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -223,7 +223,7 @@ def read( ) try: r = self._envd_api_streaming.send(request, stream=True) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise handle_envd_api_transport_exception_with_health(e, self._envd_api) err = _handle_filesystem_envd_api_exception(r) @@ -240,7 +240,7 @@ def read( headers=headers, timeout=timeout, ) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise handle_envd_api_transport_exception_with_health(e, self._envd_api) err = _handle_filesystem_envd_api_exception(r) @@ -382,7 +382,7 @@ def write_files( params=params, timeout=None if is_streamed else upload_timeout, ) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise handle_envd_api_transport_exception_with_health( e, self._envd_api ) @@ -425,7 +425,7 @@ def write_files( None if multipart_body_is_streamed(files) else upload_timeout ), ) - except httpx.RemoteProtocolError as e: + except httpx.TransportError as e: raise handle_envd_api_transport_exception_with_health(e, self._envd_api) err = _handle_filesystem_envd_api_exception(r) diff --git a/packages/python-sdk/e2b/sandbox_sync/main.py b/packages/python-sdk/e2b/sandbox_sync/main.py index 22d57c5e6b..970b510b5f 100644 --- a/packages/python-sdk/e2b/sandbox_sync/main.py +++ b/packages/python-sdk/e2b/sandbox_sync/main.py @@ -12,12 +12,14 @@ from e2b.api.client.types import Unset from e2b.api.client_sync import get_envd_api from e2b.connection_config import ApiParams, ConnectionConfig -from e2b.envd.api import ENVD_API_HEALTH_ROUTE, handle_envd_api_exception +from e2b.envd.api import ENVD_API_HEALTH_ROUTE, get_message, handle_envd_api_exception from e2b.envd.versions import ENVD_DEBUG_FALLBACK from e2b.exceptions import ( SandboxException, TemplateException, format_request_timeout_error, + format_sandbox_unavailable_exception, + is_sandbox_port_closed_message, ) from e2b.sandbox.commands.command_handle import CommandExitException from e2b.sandbox.main import SandboxOpts @@ -152,6 +154,9 @@ def is_running(self, request_timeout: Optional[float] = None) -> bool: ) if r.status_code == 502: + message = get_message(r) + if is_sandbox_port_closed_message(message): + raise format_sandbox_unavailable_exception(message) return False err = handle_envd_api_exception(r) diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.py index 7c0694da7c..02a344dd31 100644 --- a/packages/python-sdk/tests/test_envd_api_exception.py +++ b/packages/python-sdk/tests/test_envd_api_exception.py @@ -1,8 +1,12 @@ import httpx from e2b.envd.api import ( + acheck_sandbox_health, + ahandle_envd_api_transport_exception_with_health, + check_sandbox_health, format_envd_api_exception, handle_envd_api_transport_exception, + handle_envd_api_transport_exception_with_health, ) from e2b.exceptions import ( AuthenticationException, @@ -11,6 +15,7 @@ NotFoundException, RateLimitException, SandboxException, + SandboxUnreachableException, TimeoutException, ) @@ -36,6 +41,22 @@ def test_maps_429_to_rate_limit(): assert "rate limited" in str(err) +# Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +PORT_CLOSED = "The sandbox is running but port is not open" + + +def _port_closed(request: httpx.Request): + return httpx.Response( + 502, json={"message": PORT_CLOSED, "port": 49983, "code": 502} + ) + + +def test_maps_502_with_port_closed_to_unreachable(): + err = format_envd_api_exception(502, PORT_CLOSED) + assert isinstance(err, SandboxUnreachableException) + assert "envd inside the sandbox could not be reached" in str(err) + + def test_maps_502_to_timeout(): err = format_envd_api_exception(502, "Bad gateway") assert isinstance(err, TimeoutException) @@ -82,3 +103,149 @@ def test_health_result_running_returns_raw_error(): original = httpx.RemoteProtocolError("peer closed connection") err = handle_envd_api_transport_exception(original, sandbox_running=True) assert err is original + + +def _health_client(handler) -> httpx.Client: + return httpx.Client( + base_url="http://sandbox", transport=httpx.MockTransport(handler) + ) + + +def _ahealth_client(handler) -> httpx.AsyncClient: + return httpx.AsyncClient( + base_url="http://sandbox", transport=httpx.MockTransport(handler) + ) + + +def _refuse(request: httpx.Request): + raise httpx.ConnectError("connection refused", request=request) + + +def test_health_check_reports_running(): + with _health_client(lambda request: httpx.Response(204)) as client: + assert check_sandbox_health(client) is True + + +def test_health_check_reports_sandbox_gone(): + with _health_client(lambda request: httpx.Response(502)) as client: + assert check_sandbox_health(client) is False + + +def test_health_check_raises_unreachable_when_port_closed(): + with _health_client(_port_closed) as client: + try: + check_sandbox_health(client) + except SandboxUnreachableException as e: + assert PORT_CLOSED in str(e) + else: + raise AssertionError("expected the probe to raise") + + +def test_connect_failure_with_port_closed_returns_unreachable(): + original = httpx.ConnectError("connection refused") + with _health_client(_port_closed) as client: + err = handle_envd_api_transport_exception_with_health(original, client) + assert isinstance(err, SandboxUnreachableException) + assert PORT_CLOSED in str(err) + assert err.__cause__ is original + + +def test_health_check_unknown_status_is_inconclusive(): + with _health_client(lambda request: httpx.Response(500)) as client: + assert check_sandbox_health(client) is None + + +def test_health_check_raises_when_sandbox_unreachable(): + with _health_client(_refuse) as client: + try: + check_sandbox_health(client) + except httpx.ConnectError: + pass + else: + raise AssertionError("expected the probe to raise") + + +def test_connect_failure_with_sandbox_killed_returns_timeout(): + with _health_client(lambda request: httpx.Response(502)) as client: + err = handle_envd_api_transport_exception_with_health( + httpx.ConnectError("connection refused"), client + ) + assert isinstance(err, TimeoutException) + + +def test_connect_failure_with_running_sandbox_returns_raw_error(): + original = httpx.ConnectError("connection refused") + with _health_client(lambda request: httpx.Response(204)) as client: + err = handle_envd_api_transport_exception_with_health(original, client) + assert err is original + + +def test_connect_failure_with_unreachable_health_returns_unreachable(): + original = httpx.ConnectError("connection refused") + with _health_client(_refuse) as client: + err = handle_envd_api_transport_exception_with_health(original, client) + assert isinstance(err, SandboxUnreachableException) + assert "could not be reached" in str(err) + assert err.__cause__ is original + + +def test_dropped_connection_with_unreachable_health_returns_unreachable(): + original = httpx.RemoteProtocolError("stream reset") + with _health_client(_refuse) as client: + err = handle_envd_api_transport_exception_with_health(original, client) + assert isinstance(err, SandboxUnreachableException) + + +def test_non_transport_error_skips_health_check(): + def fail(request: httpx.Request): + raise AssertionError("health check should not run") + + original = httpx.ReadTimeout("timed out") + with _health_client(fail) as client: + err = handle_envd_api_transport_exception_with_health(original, client) + assert err is original + + +async def test_async_health_check_raises_when_sandbox_unreachable(): + async with _ahealth_client(_refuse) as client: + try: + await acheck_sandbox_health(client) + except httpx.ConnectError: + pass + else: + raise AssertionError("expected the probe to raise") + + +async def test_async_connect_failure_with_unreachable_health_returns_unreachable(): + original = httpx.ConnectError("connection refused") + async with _ahealth_client(_refuse) as client: + err = await ahandle_envd_api_transport_exception_with_health(original, client) + assert isinstance(err, SandboxUnreachableException) + assert err.__cause__ is original + + +async def test_async_connect_failure_with_sandbox_killed_returns_timeout(): + async with _ahealth_client(lambda request: httpx.Response(502)) as client: + err = await ahandle_envd_api_transport_exception_with_health( + httpx.ConnectError("connection refused"), client + ) + assert isinstance(err, TimeoutException) + + +async def test_async_health_check_raises_unreachable_when_port_closed(): + async with _ahealth_client(_port_closed) as client: + try: + await acheck_sandbox_health(client) + except SandboxUnreachableException as e: + assert PORT_CLOSED in str(e) + else: + raise AssertionError("expected the probe to raise") + + +async def test_async_connect_failure_with_port_closed_returns_unreachable(): + original = httpx.ConnectError("connection refused") + async with _ahealth_client(_port_closed) as client: + err = await ahandle_envd_api_transport_exception_with_health(original, client) + assert isinstance(err, SandboxUnreachableException) + assert PORT_CLOSED in str(err) + assert err.__cause__ is original diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index 5a911ec28c..f15931410c 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -16,6 +16,7 @@ NotFoundException, RateLimitException, SandboxException, + SandboxUnreachableException, TimeoutException, ) @@ -41,6 +42,16 @@ def test_maps_resource_exhausted_to_rate_limit(): assert "Rate limit" in str(err) +# Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +PORT_CLOSED = "The sandbox is running but port is not open" + + +def test_maps_unavailable_with_port_closed_to_unreachable(): + err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, PORT_CLOSED)) + assert isinstance(err, SandboxUnreachableException) + assert "envd inside the sandbox could not be reached" in str(err) + + def test_maps_unavailable_to_timeout(): err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, "gone")) assert isinstance(err, TimeoutException) @@ -76,6 +87,12 @@ def _stream_reset() -> ConnectError: return e +def _connect_refused() -> ConnectError: + err = ConnectError(Code.UNKNOWN, "connection refused") + err.__cause__ = ConnectionError("connection refused") + return err + + def _cancelled() -> ConnectError: # connectrpc converts asyncio cancellation into ConnectError(CANCELED) # with the CancelledError as __cause__. @@ -180,15 +197,60 @@ def test_health_check_unknown_returns_raw_error(): assert err is original -def test_health_check_failure_returns_raw_error(): +def test_health_check_port_closed_keeps_probe_exception(): + probe_error = SandboxUnreachableException(PORT_CLOSED) + + def check(): + raise probe_error + + original = _stream_reset() + err = handle_rpc_exception_with_health(original, check) + assert err is probe_error + assert err.__cause__ is original + + +def test_health_check_failure_returns_unreachable(): def check(): raise RuntimeError("health check failed") original = _stream_reset() err = handle_rpc_exception_with_health(original, check) + assert isinstance(err, SandboxUnreachableException) + assert "could not be reached" in str(err) + assert err.__cause__ is original + + +def test_connect_failure_is_transport_failure(): + assert is_transport_failure(_connect_refused()) + + +def test_connect_failure_runs_health_check(): + called = False + + def check(): + nonlocal called + called = True + return True + + original = _connect_refused() + err = handle_rpc_exception_with_health(original, check) + assert called assert err is original +def test_connect_failure_with_sandbox_killed_returns_timeout(): + err = handle_rpc_exception_with_health(_connect_refused(), lambda: False) + assert isinstance(err, TimeoutException) + + +def test_connect_failure_with_unreachable_health_returns_unreachable(): + def check(): + raise ConnectionError("connection refused") + + err = handle_rpc_exception_with_health(_connect_refused(), check) + assert isinstance(err, SandboxUnreachableException) + + def test_health_check_not_run_for_other_exceptions(): def fail(): raise AssertionError("health check should not run") @@ -208,10 +270,19 @@ async def check(): assert "sandbox was killed or reached its end of life" in str(err) -async def test_async_health_check_failure_returns_raw_error(): +async def test_async_health_check_failure_returns_unreachable(): async def check(): raise RuntimeError("health check failed") original = _stream_reset() err = await ahandle_rpc_exception_with_health(original, check) - assert err is original + assert isinstance(err, SandboxUnreachableException) + assert err.__cause__ is original + + +async def test_async_connect_failure_with_unreachable_health_returns_unreachable(): + async def check(): + raise ConnectionError("connection refused") + + err = await ahandle_rpc_exception_with_health(_connect_refused(), check) + assert isinstance(err, SandboxUnreachableException) From 23130a0429155296bfeec86ed3ba7bbaa5072798 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:21:47 +0000 Subject: [PATCH 02/16] revert proxy 502 'port is not open' mapping; keep sandbox-timeout TimeoutError Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 2 +- packages/js-sdk/src/envd/api.ts | 56 ++++++------------- packages/js-sdk/src/envd/rpc.ts | 47 ++-------------- packages/js-sdk/src/errors.ts | 38 +++---------- packages/js-sdk/src/sandbox/index.ts | 21 +------ .../tests/envd/handleEnvdApiError.test.ts | 34 +---------- .../js-sdk/tests/envd/handleRpcError.test.ts | 53 ------------------ packages/python-sdk/e2b/envd/api.py | 16 ++---- packages/python-sdk/e2b/envd/rpc.py | 24 +++----- packages/python-sdk/e2b/exceptions.py | 35 +++--------- packages/python-sdk/e2b/sandbox_async/main.py | 7 +-- packages/python-sdk/e2b/sandbox_sync/main.py | 7 +-- .../tests/test_envd_api_exception.py | 54 ------------------ .../tests/test_envd_rpc_exception.py | 22 -------- 14 files changed, 58 insertions(+), 358 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index ecf2c0a2d2..ada7cac4f2 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -3,4 +3,4 @@ 'e2b': patch --- -Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when the sandbox cannot be reached while it is not confirmed to be stopped: the proxy in front of the sandbox reports it running but envd's port not open (e.g. the sandbox's network is down), or a request to the sandbox fails at the connection level and the follow-up health probe gets no answer either. Previously these surfaced as a sandbox-timeout `TimeoutError`/`TimeoutException`, `isRunning()`/`is_running()` returned `false` for a running sandbox and `commands.run()` reported `SandboxNotFoundError`. Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. +Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when a request to the sandbox fails at the connection level and the follow-up health probe gets no answer from the sandbox either. Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index b3cdbd9703..17986bdb32 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -9,8 +9,7 @@ import { NotFoundError, NotEnoughSpaceError, SandboxNotFoundError, - formatSandboxUnavailableError, - isSandboxPortClosedMessage, + formatSandboxTimeoutError, AuthenticationError, RateLimitError, } from '../errors' @@ -32,7 +31,7 @@ const DEFAULT_ERROR_MAP: Record Error> = { 404: (message) => new NotFoundError(message), 429: (message) => new RateLimitError(`${message}: The requests are being rate limited.`), - 502: formatSandboxUnavailableError, + 502: formatSandboxTimeoutError, 507: (message) => new NotEnoughSpaceError(message), } @@ -43,7 +42,7 @@ const HEALTH_CHECK_TIMEOUT_MS = 5_000 * * @param envdApi - The envd API client of the sandbox. * @returns `true` if the sandbox is running, `false` if it is not, `undefined` if it answered but its state could not be determined. - * @throws The `fetch` failure when the sandbox could not be reached (no answer within the probe timeout), or a `SandboxUnreachableError` when the proxy reports the sandbox running but envd's port not open. + * @throws The `fetch` failure when the sandbox could not be reached (no answer within the probe timeout). */ export async function checkSandboxHealth( envdApi: EnvdApiClient @@ -53,10 +52,6 @@ export async function checkSandboxHealth( }) if (res.response.status === 502) { - const message = await getEnvdApiErrorMessage(res) - if (isSandboxPortClosedMessage(message)) { - throw formatSandboxUnavailableError(message) - } return false } if (res.response.ok) { @@ -102,30 +97,6 @@ export async function handleEnvdApiFetchError( return err as Error } -/** - * Extracts the error message of a non-2xx envd API response: the `message` of its - * JSON body, else its text body, else its status text. - */ -export async function getEnvdApiErrorMessage(res: { - error?: ApiError - response: Response -}): Promise { - let message = - (typeof res.error === 'string' ? res.error : res.error?.message) ?? '' - - // openapi-fetch consumes the body when parsing the error, except for - // responses without content - if (!message && !res.response.bodyUsed) { - try { - message = await res.response.text() - } catch { - // ignore unreadable bodies - } - } - - return message || res.response.statusText -} - /** * Handles errors from envd API responses by mapping HTTP status codes to specific error types. * @@ -147,7 +118,20 @@ export async function handleEnvdApiError( return } - const message = await getEnvdApiErrorMessage(res) + let message = + (typeof res.error === 'string' ? res.error : res.error?.message) ?? '' + + // openapi-fetch consumes the body when parsing the error, except for + // responses without content + if (!message && !res.response.bodyUsed) { + try { + message = await res.response.text() + } catch { + // ignore unreadable bodies + } + } + + message = message || res.response.statusText // Check if a custom error mapping is provided for this error code if (errorMap && res.response.status in errorMap) { @@ -173,9 +157,6 @@ export async function handleProcessStartEvent( } catch (err) { if (err instanceof ConnectError) { if (err.code === Code.Unavailable) { - if (isSandboxPortClosedMessage(err.rawMessage)) { - throw formatSandboxUnavailableError(err.rawMessage) - } throw new SandboxNotFoundError( 'Sandbox is probably not running anymore' ) @@ -201,9 +182,6 @@ export async function handleWatchDirStartEvent( } catch (err) { if (err instanceof ConnectError) { if (err.code === Code.Unavailable) { - if (isSandboxPortClosedMessage(err.rawMessage)) { - throw formatSandboxUnavailableError(err.rawMessage) - } throw new SandboxNotFoundError( 'Sandbox is probably not running anymore' ) diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 30711771c2..e4477dc778 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -5,7 +5,7 @@ import { compareVersions } from 'compare-versions' import { defaultUsername } from '../connectionConfig' import { AuthenticationError, - formatSandboxUnavailableError, + formatSandboxTimeoutError, InvalidArgumentError, NotFoundError, RateLimitError, @@ -91,18 +91,9 @@ export function isTransportFailure(err: unknown): boolean { /** * Builds the error for a request that failed at the connection level when the - * follow-up health probe got no answer from the sandbox either. A probe answered - * by the proxy with the sandbox running but envd's port not open is already a - * `SandboxUnreachableError` and is kept, with the failed request as its cause. + * follow-up sandbox health probe got no answer from the sandbox either. */ -export function formatSandboxUnreachableError( - err: Error, - probeErr?: unknown -): Error { - if (probeErr instanceof SandboxUnreachableError) { - probeErr.cause = err - return probeErr - } +export function formatSandboxUnreachableError(err: Error): Error { return new SandboxUnreachableError( `${err.message}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.getInfo()' and retry the request.`, { cause: err } @@ -123,8 +114,8 @@ export async function resolveTransportFailure( let running: boolean | undefined try { running = await checkHealth() - } catch (probeErr) { - return formatSandboxUnreachableError(err, probeErr) + } catch { + return formatSandboxUnreachableError(err) } if (running === false) { @@ -144,7 +135,7 @@ const DEFAULT_ERROR_MAP: Partial Error>> = { new RateLimitError( `${message}: Rate limit exceeded, please try again later.` ), - [Code.Unavailable]: formatSandboxUnavailableError, + [Code.Unavailable]: formatSandboxTimeoutError, [Code.Canceled]: (message) => new TimeoutError( `${message}: This error is likely due to exceeding 'requestTimeoutMs'. You can pass the request timeout value as an option when making the request.` @@ -250,29 +241,3 @@ export function authenticationHeader( return { Authorization: `Basic ${encoded}` } } - -/** - * Rejects a plain (non-Connect-encoded) 502 answered by the proxy in front of envd — - * the sandbox is gone or envd's port is not open — as a `ConnectError` carrying the - * proxy's message. connect reads that message for unary RPCs only; a streaming RPC - * would otherwise surface it as a bare `HTTP 502`. - */ -export async function rejectProxyUnavailableResponse( - res: Response -): Promise { - if (res.status !== 502) { - return res - } - - let message = '' - try { - const body = await res.json() - if (typeof body?.message === 'string') { - message = body.message - } - } catch { - // not a JSON body - } - - throw new ConnectError(message || `HTTP ${res.status}`, Code.Unavailable) -} diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index fa5dc6c32e..daeb32cf17 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -1,11 +1,3 @@ -// Message of the 502 the proxy in front of envd answers with when the sandbox -// is running but nothing listens on envd's port — e.g. its network is down. -const SANDBOX_PORT_CLOSED_MESSAGE = 'port is not open' - -export function isSandboxPortClosedMessage(message: string): boolean { - return message.includes(SANDBOX_PORT_CLOSED_MESSAGE) -} - // This is the message for the sandbox timeout error when the response code is 502/Unavailable export function formatSandboxTimeoutError(message: string) { return new TimeoutError( @@ -13,20 +5,6 @@ export function formatSandboxTimeoutError(message: string) { ) } -/** - * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox is - * gone (killed or timed out) unless the proxy reports it running with envd's port - * not open, in which case the sandbox is unreachable. - */ -export function formatSandboxUnavailableError(message: string): Error { - if (isSandboxPortClosedMessage(message)) { - return new SandboxUnreachableError( - `${message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.getInfo()' and retry the request.` - ) - } - return formatSandboxTimeoutError(message) -} - /** * Base class for all sandbox errors. * @@ -118,16 +96,14 @@ export class SandboxNotFoundError extends NotFoundError { } /** - * Thrown when the sandbox could not be reached while not confirmed to be - * stopped: either the proxy in front of the sandbox reports it running but - * envd's port not open (e.g. the sandbox's network is down), or a request - * failed at the connection level (the connection could not be established or - * was dropped mid-request) and a follow-up health probe got no answer from the - * sandbox either. + * Thrown when a request to the sandbox failed at the connection level (the + * connection could not be established or was dropped mid-request) and the + * follow-up health probe got no answer from the sandbox either, so it is not + * confirmed to be stopped. * - * This usually means the sandbox's network is down, envd inside it is not up - * (yet), or a transient network issue between the client and the sandbox. - * Check the sandbox state with `Sandbox.getInfo()` and retry the request. + * This usually means a transient network issue between the client and the + * sandbox, or envd inside the sandbox not being up (yet). Check the sandbox + * state with `Sandbox.getInfo()` and retry the request. */ export class SandboxUnreachableError extends SandboxError { constructor(message: string, options?: ErrorOptions) { diff --git a/packages/js-sdk/src/sandbox/index.ts b/packages/js-sdk/src/sandbox/index.ts index 38047804a1..5f0c6f2f53 100644 --- a/packages/js-sdk/src/sandbox/index.ts +++ b/packages/js-sdk/src/sandbox/index.ts @@ -6,12 +6,7 @@ import { defaultUsername, Username, } from '../connectionConfig' -import { - EnvdApiClient, - getEnvdApiErrorMessage, - handleEnvdApiError, -} from '../envd/api' -import { rejectProxyUnavailableResponse } from '../envd/rpc' +import { EnvdApiClient, handleEnvdApiError } from '../envd/api' import { createEnvdFetch, createEnvdRpcFetch } from '../envd/http2' import { createRpcLogger } from '../logs' import { Commands, Pty } from './commands' @@ -35,13 +30,7 @@ import { } from './sandboxApi' import { getSignature } from './signature' import { compareVersions } from 'compare-versions' -import { - InvalidArgumentError, - SandboxError, - TemplateError, - formatSandboxUnavailableError, - isSandboxPortClosedMessage, -} from '../errors' +import { InvalidArgumentError, SandboxError, TemplateError } from '../errors' import { ENVD_DEBUG_FALLBACK, ENVD_DEFAULT_USER } from '../envd/versions' import { shellQuote } from '../utils' @@ -205,7 +194,7 @@ export class Sandbox extends SandboxApi { redirect: 'follow', } - return envdRpcFetch(url, options).then(rejectProxyUnavailableResponse) + return envdRpcFetch(url, options) }, }) @@ -572,10 +561,6 @@ export class Sandbox extends SandboxApi { }) if (res.response.status == 502) { - const message = await getEnvdApiErrorMessage(res) - if (isSandboxPortClosedMessage(message)) { - throw formatSandboxUnavailableError(message) - } return false } diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts index a749bef146..5b3c701fa8 100644 --- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts +++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts @@ -24,9 +24,6 @@ function connectRefused(): TypeError { }) } -// Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port -const PORT_CLOSED = 'The sandbox is running but port is not open' - function healthApi(status: number, error?: { message?: string } | string) { return { api: { GET: async () => createMockResponse(status, error) }, @@ -106,13 +103,6 @@ describe('handleEnvdApiError', () => { assert.include(err?.message, 'rate limited') }) - test('returns SandboxUnreachableError for 502 with the sandbox running but its port closed', async () => { - const res = createMockResponse(502, { message: PORT_CLOSED }) - const err = await handleEnvdApiError(res) - assert.instanceOf(err, SandboxUnreachableError) - assert.include(err!.message, 'envd inside the sandbox could not be reached') - }) - test('returns TimeoutError for 502', async () => { const res = createMockResponse(502, { message: 'Bad gateway' }) const err = await handleEnvdApiError(res) @@ -233,32 +223,10 @@ describe('checkSandboxHealth', () => { }) test('reports the sandbox gone for 502', async () => { - const api = healthApi(502, { message: 'The sandbox was not found' }) - assert.strictEqual(await checkSandboxHealth(api), false) + assert.strictEqual(await checkSandboxHealth(healthApi(502)), false) }) test('is inconclusive for other statuses', async () => { assert.strictEqual(await checkSandboxHealth(healthApi(500)), undefined) }) - - test('throws SandboxUnreachableError for 502 with the sandbox running but its port closed', async () => { - const api = healthApi(502, { message: PORT_CLOSED }) - try { - await checkSandboxHealth(api) - assert.fail('expected the probe to throw') - } catch (err) { - assert.instanceOf(err, SandboxUnreachableError) - assert.include((err as Error).message, PORT_CLOSED) - } - }) - - test('keeps the probe SandboxUnreachableError with the failed request as cause', async () => { - const original = connectRefused() - const err = await handleEnvdApiFetchError(original, () => - checkSandboxHealth(healthApi(502, { message: PORT_CLOSED })) - ) - assert.instanceOf(err, SandboxUnreachableError) - assert.include(err.message, PORT_CLOSED) - assert.strictEqual(err.cause, original) - }) }) diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index caf14667f2..223f160968 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -4,7 +4,6 @@ import { handleRpcError, handleRpcErrorWithHealthCheck, isTransportFailure, - rejectProxyUnavailableResponse, } from '../../src/envd/rpc' import { AuthenticationError, @@ -32,9 +31,6 @@ function connectRefused(): ConnectError { ) } -// Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port -const PORT_CLOSED = 'The sandbox is running but port is not open' - describe('handleRpcError', () => { test('returns InvalidArgumentError for InvalidArgument', () => { const err = handleRpcError(new ConnectError('bad', Code.InvalidArgument)) @@ -59,12 +55,6 @@ describe('handleRpcError', () => { assert.include(err.message, 'Rate limit') }) - test('returns SandboxUnreachableError for Unavailable with the sandbox running but its port closed', () => { - const err = handleRpcError(new ConnectError(PORT_CLOSED, Code.Unavailable)) - assert.instanceOf(err, SandboxUnreachableError) - assert.include(err.message, 'envd inside the sandbox could not be reached') - }) - test('returns TimeoutError for Unavailable', () => { const err = handleRpcError(new ConnectError('gone', Code.Unavailable)) assert.instanceOf(err, TimeoutError) @@ -144,16 +134,6 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.notInclude(err.message, 'killed') }) - test('keeps the probe SandboxUnreachableError with the failed request as cause', async () => { - const original = terminated() - const probeErr = new SandboxUnreachableError(PORT_CLOSED) - const err = await handleRpcErrorWithHealthCheck(original, async () => { - throw probeErr - }) - assert.strictEqual(err, probeErr) - assert.strictEqual(err.cause, original) - }) - test('returns a SandboxUnreachableError when the health check itself fails', async () => { const original = terminated() const err = await handleRpcErrorWithHealthCheck(original, async () => { @@ -224,36 +204,3 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.isFalse(called) }) }) - -describe('rejectProxyUnavailableResponse', () => { - test('passes other responses through', async () => { - const res = new Response('ok', { status: 200 }) - assert.strictEqual(await rejectProxyUnavailableResponse(res), res) - }) - - test('rejects a proxy 502 as Unavailable with the body message', async () => { - const res = new Response( - JSON.stringify({ message: PORT_CLOSED, port: 49983, code: 502 }), - { status: 502, headers: { 'Content-Type': 'application/json' } } - ) - try { - await rejectProxyUnavailableResponse(res) - assert.fail('expected a rejection') - } catch (err) { - assert.instanceOf(err, ConnectError) - assert.strictEqual((err as ConnectError).code, Code.Unavailable) - assert.strictEqual((err as ConnectError).rawMessage, PORT_CLOSED) - } - }) - - test('falls back to the status for a 502 without a JSON body', async () => { - const res = new Response('bad gateway', { status: 502 }) - try { - await rejectProxyUnavailableResponse(res) - assert.fail('expected a rejection') - } catch (err) { - assert.instanceOf(err, ConnectError) - assert.strictEqual((err as ConnectError).rawMessage, 'HTTP 502') - } - }) -}) diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.py index d0cf3dc815..f6abb6ae93 100644 --- a/packages/python-sdk/e2b/envd/api.py +++ b/packages/python-sdk/e2b/envd/api.py @@ -14,8 +14,7 @@ InvalidArgumentException, NotEnoughSpaceException, RateLimitException, - format_sandbox_unavailable_exception, - is_sandbox_port_closed_message, + format_sandbox_timeout_exception, ) @@ -29,7 +28,7 @@ 429: lambda message: RateLimitException( f"{message}: The requests are being rate limited." ), - 502: format_sandbox_unavailable_exception, + 502: format_sandbox_timeout_exception, 507: NotEnoughSpaceException, } @@ -79,9 +78,6 @@ async def acheck_sandbox_health(envd_api: httpx.AsyncClient) -> Optional[bool]: def _health_status(r: httpx.Response) -> Optional[bool]: if r.status_code == 502: - message = get_message(r) - if is_sandbox_port_closed_message(message): - raise format_sandbox_unavailable_exception(message) return False if r.is_success: return True @@ -119,8 +115,8 @@ def handle_envd_api_transport_exception_with_health( return e try: sandbox_running = check_sandbox_health(envd_api) - except Exception as probe_error: - return format_sandbox_unreachable_exception(e, probe_error) + except Exception: + return format_sandbox_unreachable_exception(e) return handle_envd_api_transport_exception(e, sandbox_running) @@ -133,8 +129,8 @@ async def ahandle_envd_api_transport_exception_with_health( return e try: sandbox_running = await acheck_sandbox_health(envd_api) - except Exception as probe_error: - return format_sandbox_unreachable_exception(e, probe_error) + except Exception: + return format_sandbox_unreachable_exception(e) return handle_envd_api_transport_exception(e, sandbox_running) diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index f7b0a5ec06..66213453c8 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -11,7 +11,7 @@ NotFoundException, SandboxUnreachableException, TimeoutException, - format_sandbox_unavailable_exception, + format_sandbox_timeout_exception, AuthenticationException, RateLimitException, ) @@ -20,7 +20,7 @@ Code.INVALID_ARGUMENT: InvalidArgumentException, Code.UNAUTHENTICATED: AuthenticationException, Code.NOT_FOUND: NotFoundException, - Code.UNAVAILABLE: format_sandbox_unavailable_exception, + Code.UNAVAILABLE: format_sandbox_timeout_exception, Code.RESOURCE_EXHAUSTED: lambda message: RateLimitException( f"{message}: Rate limit exceeded, please try again later." ), @@ -74,17 +74,9 @@ def format_terminated_exception( return e -def format_sandbox_unreachable_exception( - e: Exception, probe_error: Optional[Exception] = None -) -> Exception: +def format_sandbox_unreachable_exception(e: Exception) -> Exception: """Build the exception for a request that failed at the connection level when the - follow-up sandbox health probe got no answer from the sandbox either. A probe - answered by the proxy with the sandbox running but envd's port not open is - already a ``SandboxUnreachableException`` and is kept, with the failed request - as its cause.""" - if isinstance(probe_error, SandboxUnreachableException): - probe_error.__cause__ = e - return probe_error + follow-up sandbox health probe got no answer from the sandbox either.""" err = SandboxUnreachableException( f"{e}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.get_info()' and retry the request." ) @@ -151,8 +143,8 @@ def handle_rpc_exception_with_health( if check_health is not None and is_transport_failure(e): try: sandbox_running = check_health() - except Exception as probe_error: - return format_sandbox_unreachable_exception(e, probe_error) + except Exception: + return format_sandbox_unreachable_exception(e) return handle_rpc_exception(e, error_map, sandbox_running) @@ -166,6 +158,6 @@ async def ahandle_rpc_exception_with_health( if check_health is not None and is_transport_failure(e): try: sandbox_running = await check_health() - except Exception as probe_error: - return format_sandbox_unreachable_exception(e, probe_error) + except Exception: + return format_sandbox_unreachable_exception(e) return handle_rpc_exception(e, error_map, sandbox_running) diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index 16e5cb85c9..303edd0d78 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -1,13 +1,5 @@ from typing import Optional -# Message of the 502 the proxy in front of envd answers with when the sandbox -# is running but nothing listens on envd's port — e.g. its network is down. -SANDBOX_PORT_CLOSED_MESSAGE = "port is not open" - - -def is_sandbox_port_closed_message(message: str) -> bool: - return SANDBOX_PORT_CLOSED_MESSAGE in message - def format_sandbox_timeout_exception(message: str): return TimeoutException( @@ -15,17 +7,6 @@ def format_sandbox_timeout_exception(message: str): ) -def format_sandbox_unavailable_exception(message: str) -> Exception: - """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox is - gone (killed or timed out) unless the proxy reports it running with envd's port - not open, in which case the sandbox is unreachable.""" - if is_sandbox_port_closed_message(message): - return SandboxUnreachableException( - f"{message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.get_info()' and retry the request." - ) - return format_sandbox_timeout_exception(message) - - def format_request_timeout_error() -> Exception: return TimeoutException( "Request timed out — the 'request_timeout' option can be used to increase this timeout", @@ -109,16 +90,14 @@ class SandboxNotFoundException(NotFoundException): class SandboxUnreachableException(SandboxException): """ - Raised when the sandbox could not be reached while not confirmed to be - stopped: either the proxy in front of the sandbox reports it running but - envd's port not open (e.g. the sandbox's network is down), or a request - failed at the connection level (the connection could not be established or - was dropped mid-request) and a follow-up health probe got no answer from the - sandbox either. + Raised when a request to the sandbox failed at the connection level (the + connection could not be established or was dropped mid-request) and the + follow-up health probe got no answer from the sandbox either, so it is not + confirmed to be stopped. - This usually means the sandbox's network is down, envd inside it is not up - (yet), or a transient network issue between the client and the sandbox. - Check the sandbox state with ``Sandbox.get_info()`` and retry the request. + This usually means a transient network issue between the client and the + sandbox, or envd inside the sandbox not being up (yet). Check the sandbox + state with ``Sandbox.get_info()`` and retry the request. """ pass diff --git a/packages/python-sdk/e2b/sandbox_async/main.py b/packages/python-sdk/e2b/sandbox_async/main.py index a143180665..32dd2ba2ed 100644 --- a/packages/python-sdk/e2b/sandbox_async/main.py +++ b/packages/python-sdk/e2b/sandbox_async/main.py @@ -13,14 +13,12 @@ from e2b.api.client.types import Unset from e2b.api.client_async import get_envd_api from e2b.connection_config import ApiParams, ConnectionConfig -from e2b.envd.api import ENVD_API_HEALTH_ROUTE, get_message, ahandle_envd_api_exception +from e2b.envd.api import ENVD_API_HEALTH_ROUTE, ahandle_envd_api_exception from e2b.envd.versions import ENVD_DEBUG_FALLBACK from e2b.exceptions import ( SandboxException, TemplateException, format_request_timeout_error, - format_sandbox_unavailable_exception, - is_sandbox_port_closed_message, ) from e2b.sandbox.commands.command_handle import CommandExitException from e2b.sandbox.main import SandboxOpts @@ -158,9 +156,6 @@ async def is_running(self, request_timeout: Optional[float] = None) -> bool: ) if r.status_code == 502: - message = get_message(r) - if is_sandbox_port_closed_message(message): - raise format_sandbox_unavailable_exception(message) return False err = await ahandle_envd_api_exception(r) diff --git a/packages/python-sdk/e2b/sandbox_sync/main.py b/packages/python-sdk/e2b/sandbox_sync/main.py index 970b510b5f..22d57c5e6b 100644 --- a/packages/python-sdk/e2b/sandbox_sync/main.py +++ b/packages/python-sdk/e2b/sandbox_sync/main.py @@ -12,14 +12,12 @@ from e2b.api.client.types import Unset from e2b.api.client_sync import get_envd_api from e2b.connection_config import ApiParams, ConnectionConfig -from e2b.envd.api import ENVD_API_HEALTH_ROUTE, get_message, handle_envd_api_exception +from e2b.envd.api import ENVD_API_HEALTH_ROUTE, handle_envd_api_exception from e2b.envd.versions import ENVD_DEBUG_FALLBACK from e2b.exceptions import ( SandboxException, TemplateException, format_request_timeout_error, - format_sandbox_unavailable_exception, - is_sandbox_port_closed_message, ) from e2b.sandbox.commands.command_handle import CommandExitException from e2b.sandbox.main import SandboxOpts @@ -154,9 +152,6 @@ def is_running(self, request_timeout: Optional[float] = None) -> bool: ) if r.status_code == 502: - message = get_message(r) - if is_sandbox_port_closed_message(message): - raise format_sandbox_unavailable_exception(message) return False err = handle_envd_api_exception(r) diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.py index 02a344dd31..8dec546425 100644 --- a/packages/python-sdk/tests/test_envd_api_exception.py +++ b/packages/python-sdk/tests/test_envd_api_exception.py @@ -41,22 +41,6 @@ def test_maps_429_to_rate_limit(): assert "rate limited" in str(err) -# Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port -PORT_CLOSED = "The sandbox is running but port is not open" - - -def _port_closed(request: httpx.Request): - return httpx.Response( - 502, json={"message": PORT_CLOSED, "port": 49983, "code": 502} - ) - - -def test_maps_502_with_port_closed_to_unreachable(): - err = format_envd_api_exception(502, PORT_CLOSED) - assert isinstance(err, SandboxUnreachableException) - assert "envd inside the sandbox could not be reached" in str(err) - - def test_maps_502_to_timeout(): err = format_envd_api_exception(502, "Bad gateway") assert isinstance(err, TimeoutException) @@ -131,25 +115,6 @@ def test_health_check_reports_sandbox_gone(): assert check_sandbox_health(client) is False -def test_health_check_raises_unreachable_when_port_closed(): - with _health_client(_port_closed) as client: - try: - check_sandbox_health(client) - except SandboxUnreachableException as e: - assert PORT_CLOSED in str(e) - else: - raise AssertionError("expected the probe to raise") - - -def test_connect_failure_with_port_closed_returns_unreachable(): - original = httpx.ConnectError("connection refused") - with _health_client(_port_closed) as client: - err = handle_envd_api_transport_exception_with_health(original, client) - assert isinstance(err, SandboxUnreachableException) - assert PORT_CLOSED in str(err) - assert err.__cause__ is original - - def test_health_check_unknown_status_is_inconclusive(): with _health_client(lambda request: httpx.Response(500)) as client: assert check_sandbox_health(client) is None @@ -230,22 +195,3 @@ async def test_async_connect_failure_with_sandbox_killed_returns_timeout(): httpx.ConnectError("connection refused"), client ) assert isinstance(err, TimeoutException) - - -async def test_async_health_check_raises_unreachable_when_port_closed(): - async with _ahealth_client(_port_closed) as client: - try: - await acheck_sandbox_health(client) - except SandboxUnreachableException as e: - assert PORT_CLOSED in str(e) - else: - raise AssertionError("expected the probe to raise") - - -async def test_async_connect_failure_with_port_closed_returns_unreachable(): - original = httpx.ConnectError("connection refused") - async with _ahealth_client(_port_closed) as client: - err = await ahandle_envd_api_transport_exception_with_health(original, client) - assert isinstance(err, SandboxUnreachableException) - assert PORT_CLOSED in str(err) - assert err.__cause__ is original diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index f15931410c..b529310c6f 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -42,16 +42,6 @@ def test_maps_resource_exhausted_to_rate_limit(): assert "Rate limit" in str(err) -# Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port -PORT_CLOSED = "The sandbox is running but port is not open" - - -def test_maps_unavailable_with_port_closed_to_unreachable(): - err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, PORT_CLOSED)) - assert isinstance(err, SandboxUnreachableException) - assert "envd inside the sandbox could not be reached" in str(err) - - def test_maps_unavailable_to_timeout(): err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, "gone")) assert isinstance(err, TimeoutException) @@ -197,18 +187,6 @@ def test_health_check_unknown_returns_raw_error(): assert err is original -def test_health_check_port_closed_keeps_probe_exception(): - probe_error = SandboxUnreachableException(PORT_CLOSED) - - def check(): - raise probe_error - - original = _stream_reset() - err = handle_rpc_exception_with_health(original, check) - assert err is probe_error - assert err.__cause__ is original - - def test_health_check_failure_returns_unreachable(): def check(): raise RuntimeError("health check failed") From 9dcc6f88b3a408cd93d0b1247477867f019608c8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:59:51 +0000 Subject: [PATCH 03/16] fix(js-sdk): map Unavailable on command/watch start through the rpc error map instead of SandboxNotFoundError Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 3 ++ packages/js-sdk/src/envd/api.ts | 38 +++---------------- .../js-sdk/tests/envd/handleRpcError.test.ts | 32 ++++++++++++++++ 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index ada7cac4f2..ccb41be720 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -4,3 +4,6 @@ --- Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when a request to the sandbox fails at the connection level and the follow-up health probe gets no answer from the sandbox either. Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. + + +JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` (e.g. its network is down) now raises the sandbox-timeout `TimeoutError` like every other request, instead of `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index 17986bdb32..b8e861fefa 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -8,13 +8,11 @@ import { InvalidArgumentError, NotFoundError, NotEnoughSpaceError, - SandboxNotFoundError, formatSandboxTimeoutError, AuthenticationError, RateLimitError, } from '../errors' import { StartResponse, ConnectResponse } from './process/process_pb' -import { Code, ConnectError } from '@connectrpc/connect' import { WatchDirResponse } from './filesystem/filesystem_pb' import { isConnectionTerminatedMessage, @@ -150,21 +148,9 @@ export async function handleEnvdApiError( export async function handleProcessStartEvent( events: AsyncIterable ) { - let startEvent: StartResponse | ConnectResponse - - try { - startEvent = (await events[Symbol.asyncIterator]().next()).value - } catch (err) { - if (err instanceof ConnectError) { - if (err.code === Code.Unavailable) { - throw new SandboxNotFoundError( - 'Sandbox is probably not running anymore' - ) - } - } - - throw err - } + const startEvent: StartResponse | ConnectResponse = ( + await events[Symbol.asyncIterator]().next() + ).value if (startEvent.event?.event.case !== 'start') { throw new Error('Expected start event') } @@ -175,21 +161,9 @@ export async function handleProcessStartEvent( export async function handleWatchDirStartEvent( events: AsyncIterable ) { - let startEvent: WatchDirResponse - - try { - startEvent = (await events[Symbol.asyncIterator]().next()).value - } catch (err) { - if (err instanceof ConnectError) { - if (err.code === Code.Unavailable) { - throw new SandboxNotFoundError( - 'Sandbox is probably not running anymore' - ) - } - } - - throw err - } + const startEvent: WatchDirResponse = ( + await events[Symbol.asyncIterator]().next() + ).value if (startEvent.event?.case !== 'start') { throw new Error('Expected start event') } diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 223f160968..83ca51a904 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -5,6 +5,10 @@ import { handleRpcErrorWithHealthCheck, isTransportFailure, } from '../../src/envd/rpc' +import { + handleProcessStartEvent, + handleWatchDirStartEvent, +} from '../../src/envd/api' import { AuthenticationError, InvalidArgumentError, @@ -204,3 +208,31 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.isFalse(called) }) }) + +describe('start event handlers', () => { + async function* failing(): AsyncGenerator { + throw new ConnectError('port is not open', Code.Unavailable) + } + + test('handleProcessStartEvent rethrows the ConnectError so the caller maps it', async () => { + await handleProcessStartEvent(failing()).then( + () => assert.fail('expected rejection'), + (err) => { + assert.instanceOf(err, ConnectError) + assert.equal(err.code, Code.Unavailable) + assert.instanceOf(handleRpcError(err), TimeoutError) + } + ) + }) + + test('handleWatchDirStartEvent rethrows the ConnectError so the caller maps it', async () => { + await handleWatchDirStartEvent(failing()).then( + () => assert.fail('expected rejection'), + (err) => { + assert.instanceOf(err, ConnectError) + assert.equal(err.code, Code.Unavailable) + assert.instanceOf(handleRpcError(err), TimeoutError) + } + ) + }) +}) From f30aebdbc0a1932a531b9f3c5a3d39e1fe28912f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:05:22 +0000 Subject: [PATCH 04/16] fix(js-sdk): treat browsers' opaque network TypeErrors as transport failures so they are health-probed Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 2 +- packages/js-sdk/src/envd/api.ts | 4 +-- packages/js-sdk/src/envd/rpc.ts | 9 ++++--- packages/js-sdk/src/retry.ts | 14 +++++++++++ .../tests/envd/handleEnvdApiError.test.ts | 16 ++++++++++++ .../js-sdk/tests/envd/handleRpcError.test.ts | 25 +++++++++++++++++++ packages/js-sdk/tests/retry.test.ts | 21 ++++++++++++++++ 7 files changed, 84 insertions(+), 7 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index ccb41be720..4e792cc552 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -3,7 +3,7 @@ 'e2b': patch --- -Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when a request to the sandbox fails at the connection level and the follow-up health probe gets no answer from the sandbox either. Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. +Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when a request to the sandbox fails at the connection level and the follow-up health probe gets no answer from the sandbox either. Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. In JS this also covers the opaque `TypeError` browsers raise for network failures (`Failed to fetch`, `Load failed`, `NetworkError when attempting to fetch resource.`). JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` (e.g. its network is down) now raises the sandbox-timeout `TimeoutError` like every other request, instead of `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index b8e861fefa..d16976dc5d 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -19,7 +19,7 @@ import { resolveTransportFailure, SandboxHealthCheck, } from './rpc' -import { isConnectionError } from '../retry' +import { isNetworkError } from '../retry' type ApiError = { message?: string } | string @@ -67,7 +67,7 @@ export async function checkSandboxHealth( export function isFetchTransportFailure(err: unknown): err is Error { return ( err instanceof Error && - (isConnectionTerminatedMessage(err.message) || isConnectionError(err)) + (isConnectionTerminatedMessage(err.message) || isNetworkError(err)) ) } diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index e4477dc778..924961fdb0 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -13,7 +13,7 @@ import { SandboxUnreachableError, TimeoutError, } from '../errors' -import { isConnectionError } from '../retry' +import { isNetworkError } from '../retry' import { ENVD_DEFAULT_USER } from './versions' /** @@ -73,8 +73,9 @@ function isConnectionTerminatedError(err: unknown): boolean { /** * Checks whether the error is a transport-level failure of the request to the sandbox: - * the connection could not be established (connect wraps the `fetch` rejection as - * `Code.Unknown` with the failure as `cause`) or was dropped mid-request. Errors + * the connection could not be established or failed with a browser's opaque network + * error (connect wraps the `fetch` rejection as `Code.Unknown` with the failure as + * `cause`) or was dropped mid-request. Errors * envd responded with are never transport failures. */ export function isTransportFailure(err: unknown): boolean { @@ -85,7 +86,7 @@ export function isTransportFailure(err: unknown): boolean { return ( err instanceof ConnectError && err.code === Code.Unknown && - isConnectionError(err.cause) + isNetworkError(err.cause) ) } diff --git a/packages/js-sdk/src/retry.ts b/packages/js-sdk/src/retry.ts index c0eff5f856..903edce8bb 100644 --- a/packages/js-sdk/src/retry.ts +++ b/packages/js-sdk/src/retry.ts @@ -25,6 +25,10 @@ const CONNECTION_ERROR_CODES = new Set([ const CONNECTION_ERROR_SYSCALLS = new Set(['connect', 'getaddrinfo']) // Deno reports hyper's connect-phase failures as `client error (Connect)`. const DENO_CONNECTION_ERROR = /client error \(Connect\)/ +// Browsers reject every network failure with an opaque `TypeError`: +// Chrome `Failed to fetch`, Firefox `NetworkError when attempting to fetch resource.`, Safari `Load failed` +const OPAQUE_NETWORK_ERROR = + /^(Failed to fetch|Load failed|NetworkError when attempting to fetch resource\.?)$/ // GET/PUT/PATCH/DELETE are idempotent by HTTP semantics. A POST is retried // after a 502 or a network error that may have occurred once the request was // written only if it is listed here: a replay is a no-op or fails with a @@ -89,6 +93,16 @@ export function isConnectionError(error: unknown, depth = 0): boolean { return isConnectionError(error.cause, depth + 1) } +/** + * Whether `error` (thrown by `fetch`) is a network failure: a connection error + * (see {@link isConnectionError}) or the opaque `TypeError` browsers raise for any + * network failure, which carries no code or syscall. + */ +export function isNetworkError(error: unknown): boolean { + if (isConnectionError(error)) return true + return error instanceof Error && OPAQUE_NETWORK_ERROR.test(error.message) +} + /** * Whether `request` may be sent again after a 502 or a network error that may * have occurred once the request was written (see `REPLAYABLE_OPERATIONS`). diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts index 5b3c701fa8..684ed84105 100644 --- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts +++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts @@ -124,6 +124,22 @@ describe('handleEnvdApiError', () => { }) describe('handleEnvdApiFetchError', () => { + test.each([ + ['Chrome', 'Failed to fetch'], + ['Firefox', 'NetworkError when attempting to fetch resource.'], + ['Safari', 'Load failed'], + ])( + "returns a SandboxUnreachableError for %s's opaque network error when the probe gets no answer", + async (_, message) => { + const original = new TypeError(message) + const err = await handleEnvdApiFetchError(original, async () => { + throw new TypeError(message) + }) + assert.instanceOf(err, SandboxUnreachableError) + assert.strictEqual(err.cause, original) + } + ) + test('returns the original error for terminated fetch without a health check', async () => { const original = new TypeError('terminated') const err = await handleEnvdApiFetchError(original) diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 83ca51a904..d170fdd882 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -148,6 +148,31 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.strictEqual(err.cause, original) }) + test.each([ + ['Chrome', 'Failed to fetch'], + ['Firefox', 'NetworkError when attempting to fetch resource.'], + ['Safari', 'Load failed'], + ])( + "treats %s's opaque network error as a transport failure", + async (_, message) => { + const cause = new TypeError(message) + const err = new ConnectError( + message, + Code.Unknown, + undefined, + undefined, + cause + ) + assert.isTrue(isTransportFailure(err)) + + const resolved = await handleRpcErrorWithHealthCheck(err, async () => { + throw new TypeError(message) + }) + assert.instanceOf(resolved, SandboxUnreachableError) + assert.strictEqual(resolved.cause, err) + } + ) + test('treats a refused connection as a transport failure', () => { assert.isTrue(isTransportFailure(connectRefused())) assert.isFalse(isTransportFailure(new ConnectError('nope', Code.NotFound))) diff --git a/packages/js-sdk/tests/retry.test.ts b/packages/js-sdk/tests/retry.test.ts index 7bd4f547ee..7f43e9f9e7 100644 --- a/packages/js-sdk/tests/retry.test.ts +++ b/packages/js-sdk/tests/retry.test.ts @@ -7,6 +7,7 @@ import { parseRetryAfter, resolveRetries, withRetry, + isNetworkError, } from '../src/retry' import { EnvdApiClient } from '../src/envd/api' import { InvalidArgumentError } from '../src/errors' @@ -487,6 +488,26 @@ const aborts = { timeout: new DOMException('timed out', 'TimeoutError'), } +describe('isNetworkError', () => { + test.each([ + ...Object.entries(connectFailures), + ['Chrome', opaque.Chrome], + ['Firefox', opaque.Firefox], + ['Safari', opaque.Safari], + ])('recognizes %s', (_, error) => { + expect(isNetworkError(error)).toBe(true) + }) + + test.each([ + ...Object.entries(terminated), + ...Object.entries(aborts), + ['other TypeError', new TypeError('Failed to parse URL')], + ['non-error', 'Failed to fetch'], + ])('rejects %s', (_, error) => { + expect(isNetworkError(error)).toBe(false) + }) +}) + describe('isConnectionError', () => { test.each(Object.entries(connectFailures))('recognizes %s', (_, error) => { expect(isConnectionError(error)).toBe(true) From bac187ccc5461c5f1216a69fbb45f239497ad2d9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:44:06 +0000 Subject: [PATCH 05/16] feat(sdk): SandboxUnreachableError extends TimeoutError; raise it for proxy 502 'port is not open' / Unavailable Re-applies the proxy 502 'sandbox is running but port is not open' mapping as SandboxUnreachableError (JS) / SandboxUnreachableException (Python), now a subclass of the sandbox-timeout TimeoutError / TimeoutException these cases surfaced as before, so existing handling keeps matching. isRunning() / is_running() keep returning false. Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 5 +- packages/js-sdk/src/envd/api.ts | 50 +++++++++++------ packages/js-sdk/src/envd/rpc.ts | 47 +++++++++++++--- packages/js-sdk/src/errors.ts | 42 ++++++++++++--- packages/js-sdk/src/sandbox/index.ts | 3 +- .../tests/envd/handleEnvdApiError.test.ts | 34 +++++++++++- .../js-sdk/tests/envd/handleRpcError.test.ts | 53 ++++++++++++++++++ packages/python-sdk/e2b/envd/api.py | 16 +++--- packages/python-sdk/e2b/envd/rpc.py | 24 ++++++--- packages/python-sdk/e2b/exceptions.py | 39 +++++++++++--- .../tests/test_envd_api_exception.py | 54 +++++++++++++++++++ .../tests/test_envd_rpc_exception.py | 22 ++++++++ 12 files changed, 332 insertions(+), 57 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index 4e792cc552..ce76b2487b 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -3,7 +3,8 @@ 'e2b': patch --- -Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when a request to the sandbox fails at the connection level and the follow-up health probe gets no answer from the sandbox either. Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. In JS this also covers the opaque `TypeError` browsers raise for network failures (`Failed to fetch`, `Load failed`, `NetworkError when attempting to fetch resource.`). +Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when the sandbox cannot be reached while it is not confirmed to be stopped: the proxy in front of the sandbox reports it running but envd's port not open (e.g. the sandbox's network is down), or a request to the sandbox fails at the connection level and the follow-up health probe gets no answer either. It is a subclass of the sandbox-timeout `TimeoutError` / `TimeoutException` these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working. `isRunning()` / `is_running()` still return `false` for such a sandbox. +Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. In JS this also covers the opaque `TypeError` browsers raise for network failures (`Failed to fetch`, `Load failed`, `NetworkError when attempting to fetch resource.`). -JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` (e.g. its network is down) now raises the sandbox-timeout `TimeoutError` like every other request, instead of `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. +JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` goes through the regular RPC error mapping like every other request (`SandboxUnreachableError` when the sandbox is running but envd's port is not open, sandbox-timeout `TimeoutError` otherwise), instead of `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index d16976dc5d..89b9851dc6 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -8,7 +8,8 @@ import { InvalidArgumentError, NotFoundError, NotEnoughSpaceError, - formatSandboxTimeoutError, + formatSandboxUnavailableError, + isSandboxPortClosedMessage, AuthenticationError, RateLimitError, } from '../errors' @@ -29,7 +30,7 @@ const DEFAULT_ERROR_MAP: Record Error> = { 404: (message) => new NotFoundError(message), 429: (message) => new RateLimitError(`${message}: The requests are being rate limited.`), - 502: formatSandboxTimeoutError, + 502: formatSandboxUnavailableError, 507: (message) => new NotEnoughSpaceError(message), } @@ -40,7 +41,7 @@ const HEALTH_CHECK_TIMEOUT_MS = 5_000 * * @param envdApi - The envd API client of the sandbox. * @returns `true` if the sandbox is running, `false` if it is not, `undefined` if it answered but its state could not be determined. - * @throws The `fetch` failure when the sandbox could not be reached (no answer within the probe timeout). + * @throws The `fetch` failure when the sandbox could not be reached (no answer within the probe timeout), or a `SandboxUnreachableError` when the proxy reports the sandbox running but envd's port not open. */ export async function checkSandboxHealth( envdApi: EnvdApiClient @@ -50,6 +51,10 @@ export async function checkSandboxHealth( }) if (res.response.status === 502) { + const message = await getEnvdApiErrorMessage(res) + if (isSandboxPortClosedMessage(message)) { + throw formatSandboxUnavailableError(message) + } return false } if (res.response.ok) { @@ -95,6 +100,30 @@ export async function handleEnvdApiFetchError( return err as Error } +/** + * Extracts the error message of a non-2xx envd API response: the `message` of its + * JSON body, else its text body, else its status text. + */ +export async function getEnvdApiErrorMessage(res: { + error?: ApiError + response: Response +}): Promise { + let message = + (typeof res.error === 'string' ? res.error : res.error?.message) ?? '' + + // openapi-fetch consumes the body when parsing the error, except for + // responses without content + if (!message && !res.response.bodyUsed) { + try { + message = await res.response.text() + } catch { + // ignore unreadable bodies + } + } + + return message || res.response.statusText +} + /** * Handles errors from envd API responses by mapping HTTP status codes to specific error types. * @@ -116,20 +145,7 @@ export async function handleEnvdApiError( return } - let message = - (typeof res.error === 'string' ? res.error : res.error?.message) ?? '' - - // openapi-fetch consumes the body when parsing the error, except for - // responses without content - if (!message && !res.response.bodyUsed) { - try { - message = await res.response.text() - } catch { - // ignore unreadable bodies - } - } - - message = message || res.response.statusText + const message = await getEnvdApiErrorMessage(res) // Check if a custom error mapping is provided for this error code if (errorMap && res.response.status in errorMap) { diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 924961fdb0..22bfa85d9c 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -5,7 +5,7 @@ import { compareVersions } from 'compare-versions' import { defaultUsername } from '../connectionConfig' import { AuthenticationError, - formatSandboxTimeoutError, + formatSandboxUnavailableError, InvalidArgumentError, NotFoundError, RateLimitError, @@ -92,9 +92,18 @@ export function isTransportFailure(err: unknown): boolean { /** * Builds the error for a request that failed at the connection level when the - * follow-up sandbox health probe got no answer from the sandbox either. + * follow-up health probe got no answer from the sandbox either. A probe answered + * by the proxy with the sandbox running but envd's port not open is already a + * `SandboxUnreachableError` and is kept, with the failed request as its cause. */ -export function formatSandboxUnreachableError(err: Error): Error { +export function formatSandboxUnreachableError( + err: Error, + probeErr?: unknown +): Error { + if (probeErr instanceof SandboxUnreachableError) { + probeErr.cause = err + return probeErr + } return new SandboxUnreachableError( `${err.message}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.getInfo()' and retry the request.`, { cause: err } @@ -115,8 +124,8 @@ export async function resolveTransportFailure( let running: boolean | undefined try { running = await checkHealth() - } catch { - return formatSandboxUnreachableError(err) + } catch (probeErr) { + return formatSandboxUnreachableError(err, probeErr) } if (running === false) { @@ -136,7 +145,7 @@ const DEFAULT_ERROR_MAP: Partial Error>> = { new RateLimitError( `${message}: Rate limit exceeded, please try again later.` ), - [Code.Unavailable]: formatSandboxTimeoutError, + [Code.Unavailable]: formatSandboxUnavailableError, [Code.Canceled]: (message) => new TimeoutError( `${message}: This error is likely due to exceeding 'requestTimeoutMs'. You can pass the request timeout value as an option when making the request.` @@ -242,3 +251,29 @@ export function authenticationHeader( return { Authorization: `Basic ${encoded}` } } + +/** + * Rejects a plain (non-Connect-encoded) 502 answered by the proxy in front of envd — + * the sandbox is gone or envd's port is not open — as a `ConnectError` carrying the + * proxy's message. connect reads that message for unary RPCs only; a streaming RPC + * would otherwise surface it as a bare `HTTP 502`. + */ +export async function rejectProxyUnavailableResponse( + res: Response +): Promise { + if (res.status !== 502) { + return res + } + + let message = '' + try { + const body = await res.json() + if (typeof body?.message === 'string') { + message = body.message + } + } catch { + // not a JSON body + } + + throw new ConnectError(message || `HTTP ${res.status}`, Code.Unavailable) +} diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index daeb32cf17..d16a12d455 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -1,3 +1,11 @@ +// Message of the 502 the proxy in front of envd answers with when the sandbox +// is running but nothing listens on envd's port — e.g. its network is down. +const SANDBOX_PORT_CLOSED_MESSAGE = 'port is not open' + +export function isSandboxPortClosedMessage(message: string): boolean { + return message.includes(SANDBOX_PORT_CLOSED_MESSAGE) +} + // This is the message for the sandbox timeout error when the response code is 502/Unavailable export function formatSandboxTimeoutError(message: string) { return new TimeoutError( @@ -5,6 +13,20 @@ export function formatSandboxTimeoutError(message: string) { ) } +/** + * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox is + * gone (killed or timed out) unless the proxy reports it running with envd's port + * not open, in which case the sandbox is unreachable. + */ +export function formatSandboxUnavailableError(message: string): Error { + if (isSandboxPortClosedMessage(message)) { + return new SandboxUnreachableError( + `${message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.getInfo()' and retry the request.` + ) + } + return formatSandboxTimeoutError(message) +} + /** * Base class for all sandbox errors. * @@ -96,16 +118,20 @@ export class SandboxNotFoundError extends NotFoundError { } /** - * Thrown when a request to the sandbox failed at the connection level (the - * connection could not be established or was dropped mid-request) and the - * follow-up health probe got no answer from the sandbox either, so it is not - * confirmed to be stopped. + * Thrown when the sandbox could not be reached while not confirmed to be + * stopped: either the proxy in front of the sandbox reports it running but + * envd's port not open (e.g. the sandbox's network is down), or a request + * failed at the connection level (the connection could not be established or + * was dropped mid-request) and a follow-up health probe got no answer from the + * sandbox either. + * + * This usually means the sandbox's network is down, envd inside it is not up + * (yet), or a transient network issue between the client and the sandbox. + * Check the sandbox state with `Sandbox.getInfo()` and retry the request. * - * This usually means a transient network issue between the client and the - * sandbox, or envd inside the sandbox not being up (yet). Check the sandbox - * state with `Sandbox.getInfo()` and retry the request. + * Subclass of `TimeoutError`, which these cases surfaced as before. */ -export class SandboxUnreachableError extends SandboxError { +export class SandboxUnreachableError extends TimeoutError { constructor(message: string, options?: ErrorOptions) { super(message) this.name = 'SandboxUnreachableError' diff --git a/packages/js-sdk/src/sandbox/index.ts b/packages/js-sdk/src/sandbox/index.ts index 5f0c6f2f53..778c9859fd 100644 --- a/packages/js-sdk/src/sandbox/index.ts +++ b/packages/js-sdk/src/sandbox/index.ts @@ -7,6 +7,7 @@ import { Username, } from '../connectionConfig' import { EnvdApiClient, handleEnvdApiError } from '../envd/api' +import { rejectProxyUnavailableResponse } from '../envd/rpc' import { createEnvdFetch, createEnvdRpcFetch } from '../envd/http2' import { createRpcLogger } from '../logs' import { Commands, Pty } from './commands' @@ -194,7 +195,7 @@ export class Sandbox extends SandboxApi { redirect: 'follow', } - return envdRpcFetch(url, options) + return envdRpcFetch(url, options).then(rejectProxyUnavailableResponse) }, }) diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts index 684ed84105..ada05ae1c5 100644 --- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts +++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts @@ -24,6 +24,9 @@ function connectRefused(): TypeError { }) } +// Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +const PORT_CLOSED = 'The sandbox is running but port is not open' + function healthApi(status: number, error?: { message?: string } | string) { return { api: { GET: async () => createMockResponse(status, error) }, @@ -103,6 +106,13 @@ describe('handleEnvdApiError', () => { assert.include(err?.message, 'rate limited') }) + test('returns SandboxUnreachableError for 502 with the sandbox running but its port closed', async () => { + const res = createMockResponse(502, { message: PORT_CLOSED }) + const err = await handleEnvdApiError(res) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err!.message, 'envd inside the sandbox could not be reached') + }) + test('returns TimeoutError for 502', async () => { const res = createMockResponse(502, { message: 'Bad gateway' }) const err = await handleEnvdApiError(res) @@ -239,10 +249,32 @@ describe('checkSandboxHealth', () => { }) test('reports the sandbox gone for 502', async () => { - assert.strictEqual(await checkSandboxHealth(healthApi(502)), false) + const api = healthApi(502, { message: 'The sandbox was not found' }) + assert.strictEqual(await checkSandboxHealth(api), false) }) test('is inconclusive for other statuses', async () => { assert.strictEqual(await checkSandboxHealth(healthApi(500)), undefined) }) + + test('throws SandboxUnreachableError for 502 with the sandbox running but its port closed', async () => { + const api = healthApi(502, { message: PORT_CLOSED }) + try { + await checkSandboxHealth(api) + assert.fail('expected the probe to throw') + } catch (err) { + assert.instanceOf(err, SandboxUnreachableError) + assert.include((err as Error).message, PORT_CLOSED) + } + }) + + test('keeps the probe SandboxUnreachableError with the failed request as cause', async () => { + const original = connectRefused() + const err = await handleEnvdApiFetchError(original, () => + checkSandboxHealth(healthApi(502, { message: PORT_CLOSED })) + ) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, PORT_CLOSED) + assert.strictEqual(err.cause, original) + }) }) diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index d170fdd882..524ee0ca62 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -4,6 +4,7 @@ import { handleRpcError, handleRpcErrorWithHealthCheck, isTransportFailure, + rejectProxyUnavailableResponse, } from '../../src/envd/rpc' import { handleProcessStartEvent, @@ -35,6 +36,9 @@ function connectRefused(): ConnectError { ) } +// Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +const PORT_CLOSED = 'The sandbox is running but port is not open' + describe('handleRpcError', () => { test('returns InvalidArgumentError for InvalidArgument', () => { const err = handleRpcError(new ConnectError('bad', Code.InvalidArgument)) @@ -59,6 +63,12 @@ describe('handleRpcError', () => { assert.include(err.message, 'Rate limit') }) + test('returns SandboxUnreachableError for Unavailable with the sandbox running but its port closed', () => { + const err = handleRpcError(new ConnectError(PORT_CLOSED, Code.Unavailable)) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, 'envd inside the sandbox could not be reached') + }) + test('returns TimeoutError for Unavailable', () => { const err = handleRpcError(new ConnectError('gone', Code.Unavailable)) assert.instanceOf(err, TimeoutError) @@ -138,6 +148,16 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.notInclude(err.message, 'killed') }) + test('keeps the probe SandboxUnreachableError with the failed request as cause', async () => { + const original = terminated() + const probeErr = new SandboxUnreachableError(PORT_CLOSED) + const err = await handleRpcErrorWithHealthCheck(original, async () => { + throw probeErr + }) + assert.strictEqual(err, probeErr) + assert.strictEqual(err.cause, original) + }) + test('returns a SandboxUnreachableError when the health check itself fails', async () => { const original = terminated() const err = await handleRpcErrorWithHealthCheck(original, async () => { @@ -261,3 +281,36 @@ describe('start event handlers', () => { ) }) }) + +describe('rejectProxyUnavailableResponse', () => { + test('passes other responses through', async () => { + const res = new Response('ok', { status: 200 }) + assert.strictEqual(await rejectProxyUnavailableResponse(res), res) + }) + + test('rejects a proxy 502 as Unavailable with the body message', async () => { + const res = new Response( + JSON.stringify({ message: PORT_CLOSED, port: 49983, code: 502 }), + { status: 502, headers: { 'Content-Type': 'application/json' } } + ) + try { + await rejectProxyUnavailableResponse(res) + assert.fail('expected a rejection') + } catch (err) { + assert.instanceOf(err, ConnectError) + assert.strictEqual((err as ConnectError).code, Code.Unavailable) + assert.strictEqual((err as ConnectError).rawMessage, PORT_CLOSED) + } + }) + + test('falls back to the status for a 502 without a JSON body', async () => { + const res = new Response('bad gateway', { status: 502 }) + try { + await rejectProxyUnavailableResponse(res) + assert.fail('expected a rejection') + } catch (err) { + assert.instanceOf(err, ConnectError) + assert.strictEqual((err as ConnectError).rawMessage, 'HTTP 502') + } + }) +}) diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.py index f6abb6ae93..d0cf3dc815 100644 --- a/packages/python-sdk/e2b/envd/api.py +++ b/packages/python-sdk/e2b/envd/api.py @@ -14,7 +14,8 @@ InvalidArgumentException, NotEnoughSpaceException, RateLimitException, - format_sandbox_timeout_exception, + format_sandbox_unavailable_exception, + is_sandbox_port_closed_message, ) @@ -28,7 +29,7 @@ 429: lambda message: RateLimitException( f"{message}: The requests are being rate limited." ), - 502: format_sandbox_timeout_exception, + 502: format_sandbox_unavailable_exception, 507: NotEnoughSpaceException, } @@ -78,6 +79,9 @@ async def acheck_sandbox_health(envd_api: httpx.AsyncClient) -> Optional[bool]: def _health_status(r: httpx.Response) -> Optional[bool]: if r.status_code == 502: + message = get_message(r) + if is_sandbox_port_closed_message(message): + raise format_sandbox_unavailable_exception(message) return False if r.is_success: return True @@ -115,8 +119,8 @@ def handle_envd_api_transport_exception_with_health( return e try: sandbox_running = check_sandbox_health(envd_api) - except Exception: - return format_sandbox_unreachable_exception(e) + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_envd_api_transport_exception(e, sandbox_running) @@ -129,8 +133,8 @@ async def ahandle_envd_api_transport_exception_with_health( return e try: sandbox_running = await acheck_sandbox_health(envd_api) - except Exception: - return format_sandbox_unreachable_exception(e) + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_envd_api_transport_exception(e, sandbox_running) diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index 66213453c8..f7b0a5ec06 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -11,7 +11,7 @@ NotFoundException, SandboxUnreachableException, TimeoutException, - format_sandbox_timeout_exception, + format_sandbox_unavailable_exception, AuthenticationException, RateLimitException, ) @@ -20,7 +20,7 @@ Code.INVALID_ARGUMENT: InvalidArgumentException, Code.UNAUTHENTICATED: AuthenticationException, Code.NOT_FOUND: NotFoundException, - Code.UNAVAILABLE: format_sandbox_timeout_exception, + Code.UNAVAILABLE: format_sandbox_unavailable_exception, Code.RESOURCE_EXHAUSTED: lambda message: RateLimitException( f"{message}: Rate limit exceeded, please try again later." ), @@ -74,9 +74,17 @@ def format_terminated_exception( return e -def format_sandbox_unreachable_exception(e: Exception) -> Exception: +def format_sandbox_unreachable_exception( + e: Exception, probe_error: Optional[Exception] = None +) -> Exception: """Build the exception for a request that failed at the connection level when the - follow-up sandbox health probe got no answer from the sandbox either.""" + follow-up sandbox health probe got no answer from the sandbox either. A probe + answered by the proxy with the sandbox running but envd's port not open is + already a ``SandboxUnreachableException`` and is kept, with the failed request + as its cause.""" + if isinstance(probe_error, SandboxUnreachableException): + probe_error.__cause__ = e + return probe_error err = SandboxUnreachableException( f"{e}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.get_info()' and retry the request." ) @@ -143,8 +151,8 @@ def handle_rpc_exception_with_health( if check_health is not None and is_transport_failure(e): try: sandbox_running = check_health() - except Exception: - return format_sandbox_unreachable_exception(e) + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_rpc_exception(e, error_map, sandbox_running) @@ -158,6 +166,6 @@ async def ahandle_rpc_exception_with_health( if check_health is not None and is_transport_failure(e): try: sandbox_running = await check_health() - except Exception: - return format_sandbox_unreachable_exception(e) + except Exception as probe_error: + return format_sandbox_unreachable_exception(e, probe_error) return handle_rpc_exception(e, error_map, sandbox_running) diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index 303edd0d78..7ca268c672 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -1,5 +1,13 @@ from typing import Optional +# Message of the 502 the proxy in front of envd answers with when the sandbox +# is running but nothing listens on envd's port — e.g. its network is down. +SANDBOX_PORT_CLOSED_MESSAGE = "port is not open" + + +def is_sandbox_port_closed_message(message: str) -> bool: + return SANDBOX_PORT_CLOSED_MESSAGE in message + def format_sandbox_timeout_exception(message: str): return TimeoutException( @@ -7,6 +15,17 @@ def format_sandbox_timeout_exception(message: str): ) +def format_sandbox_unavailable_exception(message: str) -> Exception: + """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox is + gone (killed or timed out) unless the proxy reports it running with envd's port + not open, in which case the sandbox is unreachable.""" + if is_sandbox_port_closed_message(message): + return SandboxUnreachableException( + f"{message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.get_info()' and retry the request." + ) + return format_sandbox_timeout_exception(message) + + def format_request_timeout_error() -> Exception: return TimeoutException( "Request timed out — the 'request_timeout' option can be used to increase this timeout", @@ -88,16 +107,20 @@ class SandboxNotFoundException(NotFoundException): pass -class SandboxUnreachableException(SandboxException): +class SandboxUnreachableException(TimeoutException): """ - Raised when a request to the sandbox failed at the connection level (the - connection could not be established or was dropped mid-request) and the - follow-up health probe got no answer from the sandbox either, so it is not - confirmed to be stopped. + Raised when the sandbox could not be reached while not confirmed to be + stopped: either the proxy in front of the sandbox reports it running but + envd's port not open (e.g. the sandbox's network is down), or a request + failed at the connection level (the connection could not be established or + was dropped mid-request) and a follow-up health probe got no answer from the + sandbox either. + + This usually means the sandbox's network is down, envd inside it is not up + (yet), or a transient network issue between the client and the sandbox. + Check the sandbox state with ``Sandbox.get_info()`` and retry the request. - This usually means a transient network issue between the client and the - sandbox, or envd inside the sandbox not being up (yet). Check the sandbox - state with ``Sandbox.get_info()`` and retry the request. + Subclass of ``TimeoutException``, which these cases surfaced as before. """ pass diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.py index 8dec546425..02a344dd31 100644 --- a/packages/python-sdk/tests/test_envd_api_exception.py +++ b/packages/python-sdk/tests/test_envd_api_exception.py @@ -41,6 +41,22 @@ def test_maps_429_to_rate_limit(): assert "rate limited" in str(err) +# Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +PORT_CLOSED = "The sandbox is running but port is not open" + + +def _port_closed(request: httpx.Request): + return httpx.Response( + 502, json={"message": PORT_CLOSED, "port": 49983, "code": 502} + ) + + +def test_maps_502_with_port_closed_to_unreachable(): + err = format_envd_api_exception(502, PORT_CLOSED) + assert isinstance(err, SandboxUnreachableException) + assert "envd inside the sandbox could not be reached" in str(err) + + def test_maps_502_to_timeout(): err = format_envd_api_exception(502, "Bad gateway") assert isinstance(err, TimeoutException) @@ -115,6 +131,25 @@ def test_health_check_reports_sandbox_gone(): assert check_sandbox_health(client) is False +def test_health_check_raises_unreachable_when_port_closed(): + with _health_client(_port_closed) as client: + try: + check_sandbox_health(client) + except SandboxUnreachableException as e: + assert PORT_CLOSED in str(e) + else: + raise AssertionError("expected the probe to raise") + + +def test_connect_failure_with_port_closed_returns_unreachable(): + original = httpx.ConnectError("connection refused") + with _health_client(_port_closed) as client: + err = handle_envd_api_transport_exception_with_health(original, client) + assert isinstance(err, SandboxUnreachableException) + assert PORT_CLOSED in str(err) + assert err.__cause__ is original + + def test_health_check_unknown_status_is_inconclusive(): with _health_client(lambda request: httpx.Response(500)) as client: assert check_sandbox_health(client) is None @@ -195,3 +230,22 @@ async def test_async_connect_failure_with_sandbox_killed_returns_timeout(): httpx.ConnectError("connection refused"), client ) assert isinstance(err, TimeoutException) + + +async def test_async_health_check_raises_unreachable_when_port_closed(): + async with _ahealth_client(_port_closed) as client: + try: + await acheck_sandbox_health(client) + except SandboxUnreachableException as e: + assert PORT_CLOSED in str(e) + else: + raise AssertionError("expected the probe to raise") + + +async def test_async_connect_failure_with_port_closed_returns_unreachable(): + original = httpx.ConnectError("connection refused") + async with _ahealth_client(_port_closed) as client: + err = await ahandle_envd_api_transport_exception_with_health(original, client) + assert isinstance(err, SandboxUnreachableException) + assert PORT_CLOSED in str(err) + assert err.__cause__ is original diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index b529310c6f..f15931410c 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -42,6 +42,16 @@ def test_maps_resource_exhausted_to_rate_limit(): assert "Rate limit" in str(err) +# Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port +PORT_CLOSED = "The sandbox is running but port is not open" + + +def test_maps_unavailable_with_port_closed_to_unreachable(): + err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, PORT_CLOSED)) + assert isinstance(err, SandboxUnreachableException) + assert "envd inside the sandbox could not be reached" in str(err) + + def test_maps_unavailable_to_timeout(): err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, "gone")) assert isinstance(err, TimeoutException) @@ -187,6 +197,18 @@ def test_health_check_unknown_returns_raw_error(): assert err is original +def test_health_check_port_closed_keeps_probe_exception(): + probe_error = SandboxUnreachableException(PORT_CLOSED) + + def check(): + raise probe_error + + original = _stream_reset() + err = handle_rpc_exception_with_health(original, check) + assert err is probe_error + assert err.__cause__ is original + + def test_health_check_failure_returns_unreachable(): def check(): raise RuntimeError("health check failed") From d9612f9fd05064e37e0c6aac97bec6c2bf3351c5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:53:27 +0000 Subject: [PATCH 06/16] feat(sdk): raise SandboxNotFoundError/SandboxNotFoundException for a 502 'The sandbox was not found' everywhere Every envd RPC and HTTP request (and a dropped connection whose health probe finds the sandbox gone) now raises SandboxNotFoundError / SandboxNotFoundException instead of the sandbox-timeout TimeoutError when the proxy reports the sandbox was not found. Port-not-open stays SandboxUnreachableError, other 502s the timeout error. Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 4 ++- packages/js-sdk/src/envd/api.ts | 2 +- packages/js-sdk/src/envd/rpc.ts | 3 ++- packages/js-sdk/src/errors.ts | 24 +++++++++++++++--- .../tests/envd/handleEnvdApiError.test.ts | 20 +++++++++++---- .../js-sdk/tests/envd/handleRpcError.test.ts | 19 ++++++++++---- packages/python-sdk/e2b/envd/rpc.py | 5 ++-- packages/python-sdk/e2b/exceptions.py | 25 ++++++++++++++++--- .../tests/test_envd_api_exception.py | 17 +++++++++---- .../tests/test_envd_rpc_exception.py | 17 ++++++++++--- 10 files changed, 106 insertions(+), 30 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index ce76b2487b..e6317e68ec 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -5,6 +5,8 @@ Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when the sandbox cannot be reached while it is not confirmed to be stopped: the proxy in front of the sandbox reports it running but envd's port not open (e.g. the sandbox's network is down), or a request to the sandbox fails at the connection level and the follow-up health probe gets no answer either. It is a subclass of the sandbox-timeout `TimeoutError` / `TimeoutException` these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working. `isRunning()` / `is_running()` still return `false` for such a sandbox. +A `502 "The sandbox was not found"` from the proxy in front of the sandbox (the sandbox was killed or reached its end of life) now raises `SandboxNotFoundError` / `SandboxNotFoundException` from every request, instead of the sandbox-timeout `TimeoutError` / `TimeoutException` — a request whose connection dropped because the sandbox was killed raises it too. Code catching `TimeoutError` for a killed sandbox must catch `SandboxNotFoundError` (a `NotFoundError`) instead. Other 502s keep raising the sandbox-timeout error. + Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. In JS this also covers the opaque `TypeError` browsers raise for network failures (`Failed to fetch`, `Load failed`, `NetworkError when attempting to fetch resource.`). -JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` goes through the regular RPC error mapping like every other request (`SandboxUnreachableError` when the sandbox is running but envd's port is not open, sandbox-timeout `TimeoutError` otherwise), instead of `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. +JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` goes through the regular RPC error mapping like every other request (`SandboxNotFoundError` when the sandbox was not found, `SandboxUnreachableError` when it is running but envd's port is not open, sandbox-timeout `TimeoutError` otherwise), instead of always `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index 89b9851dc6..43a9dba825 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -84,7 +84,7 @@ export function isFetchTransportFailure(err: unknown): err is Error { * * @param err - The caught error, expected to be a fetch transport failure. * @param checkHealth - Probe resolving to whether the sandbox is running (`undefined` when unknown) and rejecting when it cannot be reached. - * @returns A `TimeoutError` when the sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, or the original error otherwise. + * @returns A `SandboxNotFoundError` when the sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, or the original error otherwise. */ export async function handleEnvdApiFetchError( err: unknown, diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 22bfa85d9c..2f238e0af2 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -10,6 +10,7 @@ import { NotFoundError, RateLimitError, SandboxError, + SandboxNotFoundError, SandboxUnreachableError, TimeoutError, } from '../errors' @@ -129,7 +130,7 @@ export async function resolveTransportFailure( } if (running === false) { - return new TimeoutError( + return new SandboxNotFoundError( `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.` ) } diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index d16a12d455..259e290127 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -6,6 +6,20 @@ export function isSandboxPortClosedMessage(message: string): boolean { return message.includes(SANDBOX_PORT_CLOSED_MESSAGE) } +// Message of the 502 the proxy in front of envd answers with when the sandbox +// does not exist anymore — it was killed or reached its end of life. +const SANDBOX_NOT_FOUND_MESSAGE = 'was not found' + +export function isSandboxNotFoundMessage(message: string): boolean { + return message.includes(SANDBOX_NOT_FOUND_MESSAGE) +} + +export function formatSandboxNotFoundError(message: string) { + return new SandboxNotFoundError( + `${message}: The sandbox was killed or reached its end of life. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` + ) +} + // This is the message for the sandbox timeout error when the response code is 502/Unavailable export function formatSandboxTimeoutError(message: string) { return new TimeoutError( @@ -14,11 +28,15 @@ export function formatSandboxTimeoutError(message: string) { } /** - * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox is - * gone (killed or timed out) unless the proxy reports it running with envd's port - * not open, in which case the sandbox is unreachable. + * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox was + * not found (killed or timed out) → `SandboxNotFoundError`; the sandbox is running + * but envd's port is not open (e.g. its network is down) → `SandboxUnreachableError`; + * anything else → the sandbox-timeout `TimeoutError`. */ export function formatSandboxUnavailableError(message: string): Error { + if (isSandboxNotFoundMessage(message)) { + return formatSandboxNotFoundError(message) + } if (isSandboxPortClosedMessage(message)) { return new SandboxUnreachableError( `${message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.getInfo()' and retry the request.` diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts index ada05ae1c5..56bb3e24fe 100644 --- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts +++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts @@ -12,6 +12,7 @@ import { RateLimitError, SandboxError, SandboxUnreachableError, + SandboxNotFoundError, TimeoutError, } from '../../src/errors' @@ -113,6 +114,15 @@ describe('handleEnvdApiError', () => { assert.include(err!.message, 'envd inside the sandbox could not be reached') }) + test('returns SandboxNotFoundError for 502 with the sandbox not found', async () => { + const res = createMockResponse(502, { + message: 'The sandbox was not found', + }) + const err = await handleEnvdApiError(res) + assert.instanceOf(err, SandboxNotFoundError) + assert.include(err!.message, 'The sandbox was not found') + }) + test('returns TimeoutError for 502', async () => { const res = createMockResponse(502, { message: 'Bad gateway' }) const err = await handleEnvdApiError(res) @@ -156,12 +166,12 @@ describe('handleEnvdApiFetchError', () => { assert.strictEqual(err, original) }) - test('returns a TimeoutError when the health check says the sandbox is not running', async () => { + test('returns a SandboxNotFoundError when the health check says the sandbox is not running', async () => { const err = await handleEnvdApiFetchError( new TypeError('terminated'), async () => false ) - assert.instanceOf(err, TimeoutError) + assert.instanceOf(err, SandboxNotFoundError) assert.include(err.message, 'sandbox was killed or reached its end of life') }) @@ -178,7 +188,7 @@ describe('handleEnvdApiFetchError', () => { for (const [runtime, error] of Object.entries(runtimeTerminatedErrors)) { test(`treats the ${runtime} dropped-connection error as terminated`, async () => { const err = await handleEnvdApiFetchError(error, async () => false) - assert.instanceOf(err, TimeoutError) + assert.instanceOf(err, SandboxNotFoundError) assert.include( err.message, 'sandbox was killed or reached its end of life' @@ -219,12 +229,12 @@ describe('handleEnvdApiFetchError', () => { assert.strictEqual(err, original) }) - test('returns a TimeoutError for a refused connection when the sandbox is gone', async () => { + test('returns a SandboxNotFoundError for a refused connection when the sandbox is gone', async () => { const err = await handleEnvdApiFetchError( connectRefused(), async () => false ) - assert.instanceOf(err, TimeoutError) + assert.instanceOf(err, SandboxNotFoundError) }) test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 524ee0ca62..5e26e82320 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -17,6 +17,7 @@ import { RateLimitError, SandboxError, SandboxUnreachableError, + SandboxNotFoundError, TimeoutError, } from '../../src/errors' @@ -69,6 +70,14 @@ describe('handleRpcError', () => { assert.include(err.message, 'envd inside the sandbox could not be reached') }) + test('returns SandboxNotFoundError for Unavailable with the sandbox not found', () => { + const err = handleRpcError( + new ConnectError('The sandbox was not found', Code.Unavailable) + ) + assert.instanceOf(err, SandboxNotFoundError) + assert.include(err.message, 'The sandbox was not found') + }) + test('returns TimeoutError for Unavailable', () => { const err = handleRpcError(new ConnectError('gone', Code.Unavailable)) assert.instanceOf(err, TimeoutError) @@ -105,12 +114,12 @@ describe('handleRpcErrorWithHealthCheck', () => { Browser: 'network error', } - test('returns a TimeoutError when the health check says the sandbox is not running', async () => { + test('returns a SandboxNotFoundError when the health check says the sandbox is not running', async () => { const err = await handleRpcErrorWithHealthCheck( terminated(), async () => false ) - assert.instanceOf(err, TimeoutError) + assert.instanceOf(err, SandboxNotFoundError) assert.include(err.message, 'sandbox was killed or reached its end of life') }) @@ -120,7 +129,7 @@ describe('handleRpcErrorWithHealthCheck', () => { new ConnectError(message, Code.Unknown), async () => false ) - assert.instanceOf(err, TimeoutError) + assert.instanceOf(err, SandboxNotFoundError) assert.include( err.message, 'sandbox was killed or reached its end of life' @@ -222,12 +231,12 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.notInstanceOf(err, SandboxUnreachableError) }) - test('returns a TimeoutError for a refused connection when the sandbox is gone', async () => { + test('returns a SandboxNotFoundError for a refused connection when the sandbox is gone', async () => { const err = await handleRpcErrorWithHealthCheck( connectRefused(), async () => false ) - assert.instanceOf(err, TimeoutError) + assert.instanceOf(err, SandboxNotFoundError) }) test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index f7b0a5ec06..51d636a07e 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -9,6 +9,7 @@ SandboxException, InvalidArgumentException, NotFoundException, + SandboxNotFoundException, SandboxUnreachableException, TimeoutException, format_sandbox_unavailable_exception, @@ -66,9 +67,9 @@ def format_terminated_exception( ) -> Exception: """Handle an exception for a request that failed at the connection level: when a sandbox health probe confirmed the sandbox is gone (``sandbox_running is False``), - return a ``TimeoutException``; otherwise return the original error unchanged.""" + return a ``SandboxNotFoundException``; otherwise return the original error unchanged.""" if sandbox_running is False: - return TimeoutException( + return SandboxNotFoundException( f"{e}: The sandbox was killed or reached its end of life while the request was in flight." ) return e diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index 7ca268c672..4e9a1da633 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -9,6 +9,21 @@ def is_sandbox_port_closed_message(message: str) -> bool: return SANDBOX_PORT_CLOSED_MESSAGE in message +# Message of the 502 the proxy in front of envd answers with when the sandbox +# does not exist anymore — it was killed or reached its end of life. +SANDBOX_NOT_FOUND_MESSAGE = "was not found" + + +def is_sandbox_not_found_message(message: str) -> bool: + return SANDBOX_NOT_FOUND_MESSAGE in message + + +def format_sandbox_not_found_exception(message: str): + return SandboxNotFoundException( + f"{message}: The sandbox was killed or reached its end of life. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." + ) + + def format_sandbox_timeout_exception(message: str): return TimeoutException( f"{message}: This error is likely due to sandbox timeout. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." @@ -16,9 +31,13 @@ def format_sandbox_timeout_exception(message: str): def format_sandbox_unavailable_exception(message: str) -> Exception: - """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox is - gone (killed or timed out) unless the proxy reports it running with envd's port - not open, in which case the sandbox is unreachable.""" + """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox was + not found (killed or timed out) -> ``SandboxNotFoundException``; the sandbox is + running but envd's port is not open (e.g. its network is down) -> + ``SandboxUnreachableException``; anything else -> the sandbox-timeout + ``TimeoutException``.""" + if is_sandbox_not_found_message(message): + return format_sandbox_not_found_exception(message) if is_sandbox_port_closed_message(message): return SandboxUnreachableException( f"{message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.get_info()' and retry the request." diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.py index 02a344dd31..bd058bbacf 100644 --- a/packages/python-sdk/tests/test_envd_api_exception.py +++ b/packages/python-sdk/tests/test_envd_api_exception.py @@ -16,6 +16,7 @@ RateLimitException, SandboxException, SandboxUnreachableException, + SandboxNotFoundException, TimeoutException, ) @@ -91,11 +92,17 @@ def test_returns_original_when_not_transport_error(): assert err is original +def test_502_sandbox_not_found_returns_not_found(): + err = format_envd_api_exception(502, "The sandbox was not found") + assert isinstance(err, SandboxNotFoundException) + assert "The sandbox was not found" in str(err) + + def test_health_result_confirms_sandbox_killed(): err = handle_envd_api_transport_exception( httpx.RemoteProtocolError("peer closed connection"), sandbox_running=False ) - assert isinstance(err, TimeoutException) + assert isinstance(err, SandboxNotFoundException) assert "sandbox was killed or reached its end of life" in str(err) @@ -165,12 +172,12 @@ def test_health_check_raises_when_sandbox_unreachable(): raise AssertionError("expected the probe to raise") -def test_connect_failure_with_sandbox_killed_returns_timeout(): +def test_connect_failure_with_sandbox_killed_returns_not_found(): with _health_client(lambda request: httpx.Response(502)) as client: err = handle_envd_api_transport_exception_with_health( httpx.ConnectError("connection refused"), client ) - assert isinstance(err, TimeoutException) + assert isinstance(err, SandboxNotFoundException) def test_connect_failure_with_running_sandbox_returns_raw_error(): @@ -224,12 +231,12 @@ async def test_async_connect_failure_with_unreachable_health_returns_unreachable assert err.__cause__ is original -async def test_async_connect_failure_with_sandbox_killed_returns_timeout(): +async def test_async_connect_failure_with_sandbox_killed_returns_not_found(): async with _ahealth_client(lambda request: httpx.Response(502)) as client: err = await ahandle_envd_api_transport_exception_with_health( httpx.ConnectError("connection refused"), client ) - assert isinstance(err, TimeoutException) + assert isinstance(err, SandboxNotFoundException) async def test_async_health_check_raises_unreachable_when_port_closed(): diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index f15931410c..4fb5197c0f 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -17,6 +17,7 @@ RateLimitException, SandboxException, SandboxUnreachableException, + SandboxNotFoundException, TimeoutException, ) @@ -179,9 +180,17 @@ def test_returns_raw_transport_failure_without_health_result(): assert err is original +def test_unavailable_sandbox_not_found_returns_not_found(): + err = handle_rpc_exception( + ConnectError(Code.UNAVAILABLE, "The sandbox was not found") + ) + assert isinstance(err, SandboxNotFoundException) + assert "The sandbox was not found" in str(err) + + def test_health_check_confirms_sandbox_killed(): err = handle_rpc_exception_with_health(_stream_reset(), lambda: False) - assert isinstance(err, TimeoutException) + assert isinstance(err, SandboxNotFoundException) assert "sandbox was killed or reached its end of life" in str(err) @@ -238,9 +247,9 @@ def check(): assert err is original -def test_connect_failure_with_sandbox_killed_returns_timeout(): +def test_connect_failure_with_sandbox_killed_returns_not_found(): err = handle_rpc_exception_with_health(_connect_refused(), lambda: False) - assert isinstance(err, TimeoutException) + assert isinstance(err, SandboxNotFoundException) def test_connect_failure_with_unreachable_health_returns_unreachable(): @@ -266,7 +275,7 @@ async def check(): return False err = await ahandle_rpc_exception_with_health(_stream_reset(), check) - assert isinstance(err, TimeoutException) + assert isinstance(err, SandboxNotFoundException) assert "sandbox was killed or reached its end of life" in str(err) From db210d2f06b2169384dec61553e714579deb7fdd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:56:23 +0000 Subject: [PATCH 07/16] feat(sdk): probe sandbox health on an Unavailable with an unknown message so a stream ended by a sandbox kill raises SandboxNotFound Co-Authored-By: mish@e2b.dev --- packages/js-sdk/src/envd/rpc.ts | 18 +++++++- .../js-sdk/tests/envd/handleRpcError.test.ts | 34 ++++++++++++++ packages/python-sdk/e2b/envd/rpc.py | 26 ++++++++++- .../tests/test_envd_rpc_exception.py | 44 +++++++++++++++++++ 4 files changed, 119 insertions(+), 3 deletions(-) diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 2f238e0af2..12bd7b180f 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -11,6 +11,8 @@ import { RateLimitError, SandboxError, SandboxNotFoundError, + isSandboxPortClosedMessage, + isSandboxNotFoundMessage, SandboxUnreachableError, TimeoutError, } from '../errors' @@ -200,12 +202,26 @@ export function handleRpcError( * @param errorMap - Optional map of gRPC `Code` values to error factory functions that override the defaults. * @returns The corresponding `Error` instance. */ +/** + * An `Unavailable` whose message is neither the proxy's "sandbox was not found" nor + * "port is not open" — e.g. envd ending a stream while the sandbox is being killed — + * does not tell whether the sandbox is gone; the health probe does. + */ +function isAmbiguousUnavailable(err: unknown): boolean { + return ( + err instanceof ConnectError && + err.code === Code.Unavailable && + !isSandboxNotFoundMessage(err.rawMessage) && + !isSandboxPortClosedMessage(err.rawMessage) + ) +} + export async function handleRpcErrorWithHealthCheck( err: unknown, checkHealth?: SandboxHealthCheck, errorMap?: Partial Error>> ): Promise { - if (checkHealth && isTransportFailure(err)) { + if (checkHealth && (isTransportFailure(err) || isAmbiguousUnavailable(err))) { const resolved = await resolveTransportFailure( err as ConnectError, checkHealth diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 5e26e82320..70b3cf3915 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -114,6 +114,40 @@ describe('handleRpcErrorWithHealthCheck', () => { Browser: 'network error', } + test('probes health on an Unavailable with an unknown message and returns SandboxNotFoundError when the sandbox is gone', async () => { + const err = await handleRpcErrorWithHealthCheck( + new ConnectError( + 'the connection to sandbox ended before the stream completed', + Code.Unavailable + ), + async () => false + ) + assert.instanceOf(err, SandboxNotFoundError) + assert.include(err.message, 'ended before the stream completed') + }) + + test('keeps the TimeoutError for an Unavailable with an unknown message when the sandbox is running', async () => { + const err = await handleRpcErrorWithHealthCheck( + new ConnectError('stream ended', Code.Unavailable), + async () => true + ) + assert.instanceOf(err, TimeoutError) + assert.notInstanceOf(err, SandboxNotFoundError) + }) + + test('does not probe health on an Unavailable with the sandbox not found', async () => { + let probed = false + const err = await handleRpcErrorWithHealthCheck( + new ConnectError('The sandbox was not found', Code.Unavailable), + async () => { + probed = true + return true + } + ) + assert.isFalse(probed) + assert.instanceOf(err, SandboxNotFoundError) + }) + test('returns a SandboxNotFoundError when the health check says the sandbox is not running', async () => { const err = await handleRpcErrorWithHealthCheck( terminated(), diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index 51d636a07e..8f791194b2 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -13,6 +13,8 @@ SandboxUnreachableException, TimeoutException, format_sandbox_unavailable_exception, + is_sandbox_port_closed_message, + is_sandbox_not_found_message, AuthenticationException, RateLimitException, ) @@ -135,6 +137,18 @@ def handle_rpc_exception( return e +def is_ambiguous_unavailable(e: Exception) -> bool: + """An ``UNAVAILABLE`` whose message is neither the proxy's "sandbox was not found" + nor "port is not open" — e.g. envd ending a stream while the sandbox is being + killed — does not tell whether the sandbox is gone; the health probe does.""" + return ( + isinstance(e, ConnectError) + and e.code == Code.UNAVAILABLE + and not is_sandbox_not_found_message(e.message) + and not is_sandbox_port_closed_message(e.message) + ) + + def handle_rpc_exception_with_health( e: Exception, check_health: Optional[Callable[[], Optional[bool]]] = None, @@ -149,11 +163,15 @@ def handle_rpc_exception_with_health( check raises), a ``SandboxUnreachableException``. """ sandbox_running = None - if check_health is not None and is_transport_failure(e): + if check_health is not None and ( + is_transport_failure(e) or is_ambiguous_unavailable(e) + ): try: sandbox_running = check_health() except Exception as probe_error: return format_sandbox_unreachable_exception(e, probe_error) + if sandbox_running is False and is_ambiguous_unavailable(e): + return format_terminated_exception(e, sandbox_running) return handle_rpc_exception(e, error_map, sandbox_running) @@ -164,9 +182,13 @@ async def ahandle_rpc_exception_with_health( ): """Async version of :func:`handle_rpc_exception_with_health`.""" sandbox_running = None - if check_health is not None and is_transport_failure(e): + if check_health is not None and ( + is_transport_failure(e) or is_ambiguous_unavailable(e) + ): try: sandbox_running = await check_health() except Exception as probe_error: return format_sandbox_unreachable_exception(e, probe_error) + if sandbox_running is False and is_ambiguous_unavailable(e): + return format_terminated_exception(e, sandbox_running) return handle_rpc_exception(e, error_map, sandbox_running) diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index 4fb5197c0f..9ef012006f 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -188,6 +188,50 @@ def test_unavailable_sandbox_not_found_returns_not_found(): assert "The sandbox was not found" in str(err) +def test_ambiguous_unavailable_probes_health_and_returns_not_found(): + err = handle_rpc_exception_with_health( + ConnectError( + Code.UNAVAILABLE, + "the connection to sandbox ended before the stream completed", + ), + lambda: False, + ) + assert isinstance(err, SandboxNotFoundException) + assert "ended before the stream completed" in str(err) + + +def test_ambiguous_unavailable_keeps_timeout_when_running(): + err = handle_rpc_exception_with_health( + ConnectError(Code.UNAVAILABLE, "stream ended"), lambda: True + ) + assert isinstance(err, TimeoutException) + assert not isinstance(err, SandboxNotFoundException) + + +def test_unavailable_sandbox_not_found_does_not_probe(): + probed = [] + + def check(): + probed.append(True) + return True + + err = handle_rpc_exception_with_health( + ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), check + ) + assert not probed + assert isinstance(err, SandboxNotFoundException) + + +async def test_async_ambiguous_unavailable_probes_health_and_returns_not_found(): + async def check(): + return False + + err = await ahandle_rpc_exception_with_health( + ConnectError(Code.UNAVAILABLE, "stream ended"), check + ) + assert isinstance(err, SandboxNotFoundException) + + def test_health_check_confirms_sandbox_killed(): err = handle_rpc_exception_with_health(_stream_reset(), lambda: False) assert isinstance(err, SandboxNotFoundException) From 5080b27f8a4172b6fcaf0f5ab09d9f17dcb4eefa Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:04:20 +0000 Subject: [PATCH 08/16] feat(sdk): raise SandboxUnreachableError for every 502/Unavailable from the proxy; SandboxNotFound only when starting a command/PTY/watch on a sandbox that was not found Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 4 +- packages/js-sdk/src/envd/api.ts | 2 +- packages/js-sdk/src/envd/rpc.ts | 35 ++++------ packages/js-sdk/src/errors.ts | 50 ++++++------- packages/js-sdk/src/sandbox/commands/index.ts | 7 +- packages/js-sdk/src/sandbox/commands/pty.ts | 13 +++- .../js-sdk/src/sandbox/filesystem/index.ts | 2 + .../tests/envd/handleEnvdApiError.test.ts | 17 +++-- .../js-sdk/tests/envd/handleRpcError.test.ts | 54 ++++++++------ packages/python-sdk/e2b/envd/rpc.py | 41 ++++------- packages/python-sdk/e2b/exceptions.py | 49 +++++++------ .../e2b/sandbox_async/commands/command.py | 6 +- .../e2b/sandbox_async/commands/pty.py | 10 ++- .../sandbox_async/filesystem/filesystem.py | 3 +- .../e2b/sandbox_sync/commands/command.py | 6 +- .../e2b/sandbox_sync/commands/pty.py | 10 ++- .../e2b/sandbox_sync/filesystem/filesystem.py | 3 +- .../tests/test_envd_api_exception.py | 17 +++-- .../tests/test_envd_rpc_exception.py | 70 +++++++++++-------- 19 files changed, 213 insertions(+), 186 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index e6317e68ec..7e576ea373 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -5,8 +5,8 @@ Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when the sandbox cannot be reached while it is not confirmed to be stopped: the proxy in front of the sandbox reports it running but envd's port not open (e.g. the sandbox's network is down), or a request to the sandbox fails at the connection level and the follow-up health probe gets no answer either. It is a subclass of the sandbox-timeout `TimeoutError` / `TimeoutException` these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working. `isRunning()` / `is_running()` still return `false` for such a sandbox. -A `502 "The sandbox was not found"` from the proxy in front of the sandbox (the sandbox was killed or reached its end of life) now raises `SandboxNotFoundError` / `SandboxNotFoundException` from every request, instead of the sandbox-timeout `TimeoutError` / `TimeoutException` — a request whose connection dropped because the sandbox was killed raises it too. Code catching `TimeoutError` for a killed sandbox must catch `SandboxNotFoundError` (a `NotFoundError`) instead. Other 502s keep raising the sandbox-timeout error. +Every `502` / `Unavailable` answered by the proxy in front of the sandbox — the sandbox was not found (killed or reached its timeout) or it is running but envd's port is not open (e.g. `ip link set eth0 down` inside it) — now raises `SandboxUnreachableError` / `SandboxUnreachableException` from every envd request, as does a request whose connection dropped because the sandbox was killed. The new error extends `TimeoutError` / `TimeoutException`, so existing timeout handling keeps working. Starting a command / PTY / directory watch on a sandbox the proxy reports as not found keeps raising `SandboxNotFoundError` in JS and now raises `SandboxNotFoundException` in Python (instead of `TimeoutException`). Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. In JS this also covers the opaque `TypeError` browsers raise for network failures (`Failed to fetch`, `Load failed`, `NetworkError when attempting to fetch resource.`). -JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` goes through the regular RPC error mapping like every other request (`SandboxNotFoundError` when the sandbox was not found, `SandboxUnreachableError` when it is running but envd's port is not open, sandbox-timeout `TimeoutError` otherwise), instead of always `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. +JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` goes through the regular RPC error mapping like every other request (`SandboxNotFoundError` with the proxy's message when the sandbox was not found, `SandboxUnreachableError` otherwise), instead of always `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index 43a9dba825..5cf7f6ca0e 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -84,7 +84,7 @@ export function isFetchTransportFailure(err: unknown): err is Error { * * @param err - The caught error, expected to be a fetch transport failure. * @param checkHealth - Probe resolving to whether the sandbox is running (`undefined` when unknown) and rejecting when it cannot be reached. - * @returns A `SandboxNotFoundError` when the sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, or the original error otherwise. + * @returns A `SandboxUnreachableError` when the sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, or the original error otherwise. */ export async function handleEnvdApiFetchError( err: unknown, diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 12bd7b180f..0b6d2910a2 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -5,14 +5,12 @@ import { compareVersions } from 'compare-versions' import { defaultUsername } from '../connectionConfig' import { AuthenticationError, + formatSandboxStartUnavailableError, formatSandboxUnavailableError, InvalidArgumentError, NotFoundError, RateLimitError, SandboxError, - SandboxNotFoundError, - isSandboxPortClosedMessage, - isSandboxNotFoundMessage, SandboxUnreachableError, TimeoutError, } from '../errors' @@ -132,14 +130,25 @@ export async function resolveTransportFailure( } if (running === false) { - return new SandboxNotFoundError( - `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.` + return new SandboxUnreachableError( + `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.`, + { cause: err } ) } return undefined } +/** + * Error map for starting a command/PTY/watch: a 502 "sandbox was not found" from the + * proxy is a `SandboxNotFoundError`, every other `Unavailable` a `SandboxUnreachableError`. + */ +export const START_RPC_ERROR_MAP: Partial< + Record Error> +> = { + [Code.Unavailable]: formatSandboxStartUnavailableError, +} + const DEFAULT_ERROR_MAP: Partial Error>> = { [Code.InvalidArgument]: (message) => new InvalidArgumentError(message), [Code.Unauthenticated]: (message) => new AuthenticationError(message), @@ -202,26 +211,12 @@ export function handleRpcError( * @param errorMap - Optional map of gRPC `Code` values to error factory functions that override the defaults. * @returns The corresponding `Error` instance. */ -/** - * An `Unavailable` whose message is neither the proxy's "sandbox was not found" nor - * "port is not open" — e.g. envd ending a stream while the sandbox is being killed — - * does not tell whether the sandbox is gone; the health probe does. - */ -function isAmbiguousUnavailable(err: unknown): boolean { - return ( - err instanceof ConnectError && - err.code === Code.Unavailable && - !isSandboxNotFoundMessage(err.rawMessage) && - !isSandboxPortClosedMessage(err.rawMessage) - ) -} - export async function handleRpcErrorWithHealthCheck( err: unknown, checkHealth?: SandboxHealthCheck, errorMap?: Partial Error>> ): Promise { - if (checkHealth && (isTransportFailure(err) || isAmbiguousUnavailable(err))) { + if (checkHealth && isTransportFailure(err)) { const resolved = await resolveTransportFailure( err as ConnectError, checkHealth diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index 259e290127..b14d3d46dd 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -1,48 +1,48 @@ // Message of the 502 the proxy in front of envd answers with when the sandbox // is running but nothing listens on envd's port — e.g. its network is down. -const SANDBOX_PORT_CLOSED_MESSAGE = 'port is not open' - -export function isSandboxPortClosedMessage(message: string): boolean { - return message.includes(SANDBOX_PORT_CLOSED_MESSAGE) +// This is the message for the sandbox timeout error when the response code is 502/Unavailable +export function formatSandboxTimeoutError(message: string) { + return new TimeoutError( + `${message}: This error is likely due to sandbox timeout. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` + ) } -// Message of the 502 the proxy in front of envd answers with when the sandbox -// does not exist anymore — it was killed or reached its end of life. +// Messages of the 502 the proxy in front of envd answers with when the sandbox does +// not exist anymore (killed or reached its timeout) or when it is running but envd's +// port is not open (e.g. its network is down). const SANDBOX_NOT_FOUND_MESSAGE = 'was not found' +const SANDBOX_PORT_CLOSED_MESSAGE = 'port is not open' export function isSandboxNotFoundMessage(message: string): boolean { return message.includes(SANDBOX_NOT_FOUND_MESSAGE) } -export function formatSandboxNotFoundError(message: string) { - return new SandboxNotFoundError( - `${message}: The sandbox was killed or reached its end of life. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` - ) +export function isSandboxPortClosedMessage(message: string): boolean { + return message.includes(SANDBOX_PORT_CLOSED_MESSAGE) } -// This is the message for the sandbox timeout error when the response code is 502/Unavailable -export function formatSandboxTimeoutError(message: string) { - return new TimeoutError( - `${message}: This error is likely due to sandbox timeout. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` +/** + * Maps a 502/Unavailable answered by the proxy in front of envd — the sandbox was + * not found (killed or reached its timeout), or it is running but envd's port is not + * open (e.g. its network is down) — to a `SandboxUnreachableError`. + */ +export function formatSandboxUnavailableError(message: string): Error { + return new SandboxUnreachableError( + `${message}: The sandbox could not be reached — it was killed, reached its timeout, or envd inside it is not reachable (e.g. its network is down). Check the sandbox state with 'Sandbox.getInfo()'; you can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` ) } /** - * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox was - * not found (killed or timed out) → `SandboxNotFoundError`; the sandbox is running - * but envd's port is not open (e.g. its network is down) → `SandboxUnreachableError`; - * anything else → the sandbox-timeout `TimeoutError`. + * Like `formatSandboxUnavailableError`, for starting a command/PTY/watch: when the + * proxy reports the sandbox was not found, a `SandboxNotFoundError` is returned. */ -export function formatSandboxUnavailableError(message: string): Error { +export function formatSandboxStartUnavailableError(message: string): Error { if (isSandboxNotFoundMessage(message)) { - return formatSandboxNotFoundError(message) - } - if (isSandboxPortClosedMessage(message)) { - return new SandboxUnreachableError( - `${message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.getInfo()' and retry the request.` + return new SandboxNotFoundError( + `${message}: The sandbox was killed or reached its timeout. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` ) } - return formatSandboxTimeoutError(message) + return formatSandboxUnavailableError(message) } /** diff --git a/packages/js-sdk/src/sandbox/commands/index.ts b/packages/js-sdk/src/sandbox/commands/index.ts index a9905d084d..a9569d0b17 100644 --- a/packages/js-sdk/src/sandbox/commands/index.ts +++ b/packages/js-sdk/src/sandbox/commands/index.ts @@ -29,6 +29,7 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, + START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE } from '../../envd/versions' @@ -475,7 +476,11 @@ export class Commands { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) + throw await handleRpcErrorWithHealthCheck( + err, + this.checkHealth, + START_RPC_ERROR_MAP + ) } } } diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.ts index 29928a92f3..c65bca09d1 100644 --- a/packages/js-sdk/src/sandbox/commands/pty.ts +++ b/packages/js-sdk/src/sandbox/commands/pty.ts @@ -22,6 +22,7 @@ import { CommandHandle } from './commandHandle' import { authenticationHeader, handleRpcErrorWithHealthCheck, + START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { @@ -161,7 +162,11 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) + throw await handleRpcErrorWithHealthCheck( + err, + this.checkHealth, + START_RPC_ERROR_MAP + ) } } @@ -218,7 +223,11 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) + throw await handleRpcErrorWithHealthCheck( + err, + this.checkHealth, + START_RPC_ERROR_MAP + ) } } diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index f3bae63862..3ff22aa867 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -25,6 +25,7 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, + START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' @@ -62,6 +63,7 @@ const FILESYSTEM_HTTP_ERROR_MAP: Record Error> = { const FILESYSTEM_RPC_ERROR_MAP: Partial< Record Error> > = { + ...START_RPC_ERROR_MAP, [Code.NotFound]: (message: string) => new FileNotFoundError(message), } diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts index 56bb3e24fe..f79d8355d6 100644 --- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts +++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts @@ -12,7 +12,6 @@ import { RateLimitError, SandboxError, SandboxUnreachableError, - SandboxNotFoundError, TimeoutError, } from '../../src/errors' @@ -111,15 +110,15 @@ describe('handleEnvdApiError', () => { const res = createMockResponse(502, { message: PORT_CLOSED }) const err = await handleEnvdApiError(res) assert.instanceOf(err, SandboxUnreachableError) - assert.include(err!.message, 'envd inside the sandbox could not be reached') + assert.include(err!.message, 'The sandbox could not be reached') }) - test('returns SandboxNotFoundError for 502 with the sandbox not found', async () => { + test('returns SandboxUnreachableError for 502 for the sandbox not found', async () => { const res = createMockResponse(502, { message: 'The sandbox was not found', }) const err = await handleEnvdApiError(res) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) assert.include(err!.message, 'The sandbox was not found') }) @@ -166,12 +165,12 @@ describe('handleEnvdApiFetchError', () => { assert.strictEqual(err, original) }) - test('returns a SandboxNotFoundError when the health check says the sandbox is not running', async () => { + test('returns a SandboxUnreachableError when the health check says the sandbox is not running', async () => { const err = await handleEnvdApiFetchError( new TypeError('terminated'), async () => false ) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) assert.include(err.message, 'sandbox was killed or reached its end of life') }) @@ -188,7 +187,7 @@ describe('handleEnvdApiFetchError', () => { for (const [runtime, error] of Object.entries(runtimeTerminatedErrors)) { test(`treats the ${runtime} dropped-connection error as terminated`, async () => { const err = await handleEnvdApiFetchError(error, async () => false) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) assert.include( err.message, 'sandbox was killed or reached its end of life' @@ -229,12 +228,12 @@ describe('handleEnvdApiFetchError', () => { assert.strictEqual(err, original) }) - test('returns a SandboxNotFoundError for a refused connection when the sandbox is gone', async () => { + test('returns a SandboxUnreachableError for a refused connection when the sandbox is gone', async () => { const err = await handleEnvdApiFetchError( connectRefused(), async () => false ) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) }) test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 70b3cf3915..05442ea40e 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -3,6 +3,7 @@ import { Code, ConnectError } from '@connectrpc/connect' import { handleRpcError, handleRpcErrorWithHealthCheck, + START_RPC_ERROR_MAP, isTransportFailure, rejectProxyUnavailableResponse, } from '../../src/envd/rpc' @@ -16,8 +17,8 @@ import { NotFoundError, RateLimitError, SandboxError, - SandboxUnreachableError, SandboxNotFoundError, + SandboxUnreachableError, TimeoutError, } from '../../src/errors' @@ -67,17 +68,35 @@ describe('handleRpcError', () => { test('returns SandboxUnreachableError for Unavailable with the sandbox running but its port closed', () => { const err = handleRpcError(new ConnectError(PORT_CLOSED, Code.Unavailable)) assert.instanceOf(err, SandboxUnreachableError) - assert.include(err.message, 'envd inside the sandbox could not be reached') + assert.include(err.message, 'The sandbox could not be reached') }) - test('returns SandboxNotFoundError for Unavailable with the sandbox not found', () => { + test('START_RPC_ERROR_MAP maps Unavailable with the sandbox not found to SandboxNotFoundError', () => { const err = handleRpcError( - new ConnectError('The sandbox was not found', Code.Unavailable) + new ConnectError('The sandbox was not found', Code.Unavailable), + START_RPC_ERROR_MAP ) assert.instanceOf(err, SandboxNotFoundError) assert.include(err.message, 'The sandbox was not found') }) + test('START_RPC_ERROR_MAP maps Unavailable with the port closed to SandboxUnreachableError', () => { + const err = handleRpcError( + new ConnectError(PORT_CLOSED, Code.Unavailable), + START_RPC_ERROR_MAP + ) + assert.instanceOf(err, SandboxUnreachableError) + assert.notInstanceOf(err, SandboxNotFoundError) + }) + + test('returns SandboxUnreachableError for Unavailable for the sandbox not found', () => { + const err = handleRpcError( + new ConnectError('The sandbox was not found', Code.Unavailable) + ) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, 'The sandbox was not found') + }) + test('returns TimeoutError for Unavailable', () => { const err = handleRpcError(new ConnectError('gone', Code.Unavailable)) assert.instanceOf(err, TimeoutError) @@ -114,7 +133,7 @@ describe('handleRpcErrorWithHealthCheck', () => { Browser: 'network error', } - test('probes health on an Unavailable with an unknown message and returns SandboxNotFoundError when the sandbox is gone', async () => { + test('probes health on an Unavailable with an unknown message and returns SandboxUnreachableError when the sandbox is gone', async () => { const err = await handleRpcErrorWithHealthCheck( new ConnectError( 'the connection to sandbox ended before the stream completed', @@ -122,20 +141,11 @@ describe('handleRpcErrorWithHealthCheck', () => { ), async () => false ) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) assert.include(err.message, 'ended before the stream completed') }) - test('keeps the TimeoutError for an Unavailable with an unknown message when the sandbox is running', async () => { - const err = await handleRpcErrorWithHealthCheck( - new ConnectError('stream ended', Code.Unavailable), - async () => true - ) - assert.instanceOf(err, TimeoutError) - assert.notInstanceOf(err, SandboxNotFoundError) - }) - - test('does not probe health on an Unavailable with the sandbox not found', async () => { + test('does not probe health on an Unavailable for the sandbox not found', async () => { let probed = false const err = await handleRpcErrorWithHealthCheck( new ConnectError('The sandbox was not found', Code.Unavailable), @@ -145,15 +155,15 @@ describe('handleRpcErrorWithHealthCheck', () => { } ) assert.isFalse(probed) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) }) - test('returns a SandboxNotFoundError when the health check says the sandbox is not running', async () => { + test('returns a SandboxUnreachableError when the health check says the sandbox is not running', async () => { const err = await handleRpcErrorWithHealthCheck( terminated(), async () => false ) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) assert.include(err.message, 'sandbox was killed or reached its end of life') }) @@ -163,7 +173,7 @@ describe('handleRpcErrorWithHealthCheck', () => { new ConnectError(message, Code.Unknown), async () => false ) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) assert.include( err.message, 'sandbox was killed or reached its end of life' @@ -265,12 +275,12 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.notInstanceOf(err, SandboxUnreachableError) }) - test('returns a SandboxNotFoundError for a refused connection when the sandbox is gone', async () => { + test('returns a SandboxUnreachableError for a refused connection when the sandbox is gone', async () => { const err = await handleRpcErrorWithHealthCheck( connectRefused(), async () => false ) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxUnreachableError) }) test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index 8f791194b2..5aec9f7930 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -9,16 +9,21 @@ SandboxException, InvalidArgumentException, NotFoundException, - SandboxNotFoundException, SandboxUnreachableException, TimeoutException, + format_sandbox_start_unavailable_exception, format_sandbox_unavailable_exception, - is_sandbox_port_closed_message, - is_sandbox_not_found_message, AuthenticationException, RateLimitException, ) +# Error map for starting a command/PTY/watch: a 502 "sandbox was not found" from the +# proxy is a ``SandboxNotFoundException``, every other ``UNAVAILABLE`` a +# ``SandboxUnreachableException``. +START_RPC_ERROR_MAP: dict[Code, Callable[[str], Exception]] = { + Code.UNAVAILABLE: format_sandbox_start_unavailable_exception, +} + _DEFAULT_RPC_ERROR_MAP: dict[Code, Callable[[str], Exception]] = { Code.INVALID_ARGUMENT: InvalidArgumentException, Code.UNAUTHENTICATED: AuthenticationException, @@ -69,9 +74,9 @@ def format_terminated_exception( ) -> Exception: """Handle an exception for a request that failed at the connection level: when a sandbox health probe confirmed the sandbox is gone (``sandbox_running is False``), - return a ``SandboxNotFoundException``; otherwise return the original error unchanged.""" + return a ``SandboxUnreachableException``; otherwise return the original error unchanged.""" if sandbox_running is False: - return SandboxNotFoundException( + return SandboxUnreachableException( f"{e}: The sandbox was killed or reached its end of life while the request was in flight." ) return e @@ -137,18 +142,6 @@ def handle_rpc_exception( return e -def is_ambiguous_unavailable(e: Exception) -> bool: - """An ``UNAVAILABLE`` whose message is neither the proxy's "sandbox was not found" - nor "port is not open" — e.g. envd ending a stream while the sandbox is being - killed — does not tell whether the sandbox is gone; the health probe does.""" - return ( - isinstance(e, ConnectError) - and e.code == Code.UNAVAILABLE - and not is_sandbox_not_found_message(e.message) - and not is_sandbox_port_closed_message(e.message) - ) - - def handle_rpc_exception_with_health( e: Exception, check_health: Optional[Callable[[], Optional[bool]]] = None, @@ -159,19 +152,15 @@ def handle_rpc_exception_with_health( mid-request) it probes the sandbox health to tell apart the sandbox being killed from the sandbox being unreachable or a transient network failure (e.g. a load balancer dropping the connection). When the probe confirms the sandbox is gone, a - ``TimeoutException`` is returned; when the probe gets no answer either (the health + ``SandboxUnreachableException`` is returned; when the probe gets no answer either (the health check raises), a ``SandboxUnreachableException``. """ sandbox_running = None - if check_health is not None and ( - is_transport_failure(e) or is_ambiguous_unavailable(e) - ): + if check_health is not None and is_transport_failure(e): try: sandbox_running = check_health() except Exception as probe_error: return format_sandbox_unreachable_exception(e, probe_error) - if sandbox_running is False and is_ambiguous_unavailable(e): - return format_terminated_exception(e, sandbox_running) return handle_rpc_exception(e, error_map, sandbox_running) @@ -182,13 +171,9 @@ async def ahandle_rpc_exception_with_health( ): """Async version of :func:`handle_rpc_exception_with_health`.""" sandbox_running = None - if check_health is not None and ( - is_transport_failure(e) or is_ambiguous_unavailable(e) - ): + if check_health is not None and is_transport_failure(e): try: sandbox_running = await check_health() except Exception as probe_error: return format_sandbox_unreachable_exception(e, probe_error) - if sandbox_running is False and is_ambiguous_unavailable(e): - return format_terminated_exception(e, sandbox_running) return handle_rpc_exception(e, error_map, sandbox_running) diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index 4e9a1da633..053553ff9e 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -1,48 +1,47 @@ from typing import Optional + # Message of the 502 the proxy in front of envd answers with when the sandbox # is running but nothing listens on envd's port — e.g. its network is down. -SANDBOX_PORT_CLOSED_MESSAGE = "port is not open" - - -def is_sandbox_port_closed_message(message: str) -> bool: - return SANDBOX_PORT_CLOSED_MESSAGE in message +def format_sandbox_timeout_exception(message: str): + return TimeoutException( + f"{message}: This error is likely due to sandbox timeout. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." + ) -# Message of the 502 the proxy in front of envd answers with when the sandbox -# does not exist anymore — it was killed or reached its end of life. +# Messages of the 502 the proxy in front of envd answers with when the sandbox does +# not exist anymore (killed or reached its timeout) or when it is running but envd's +# port is not open (e.g. its network is down). SANDBOX_NOT_FOUND_MESSAGE = "was not found" +SANDBOX_PORT_CLOSED_MESSAGE = "port is not open" def is_sandbox_not_found_message(message: str) -> bool: return SANDBOX_NOT_FOUND_MESSAGE in message -def format_sandbox_not_found_exception(message: str): - return SandboxNotFoundException( - f"{message}: The sandbox was killed or reached its end of life. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." - ) +def is_sandbox_port_closed_message(message: str) -> bool: + return SANDBOX_PORT_CLOSED_MESSAGE in message -def format_sandbox_timeout_exception(message: str): - return TimeoutException( - f"{message}: This error is likely due to sandbox timeout. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." +def format_sandbox_unavailable_exception(message: str) -> Exception: + """Map a 502/UNAVAILABLE answered by the proxy in front of envd -- the sandbox was + not found (killed or reached its timeout), or it is running but envd's port is + not open (e.g. its network is down) -- to a ``SandboxUnreachableException``.""" + return SandboxUnreachableException( + f"{message}: The sandbox could not be reached -- it was killed, reached its timeout, or envd inside it is not reachable (e.g. its network is down). Check the sandbox state with 'Sandbox.get_info()'; you can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." ) -def format_sandbox_unavailable_exception(message: str) -> Exception: - """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox was - not found (killed or timed out) -> ``SandboxNotFoundException``; the sandbox is - running but envd's port is not open (e.g. its network is down) -> - ``SandboxUnreachableException``; anything else -> the sandbox-timeout - ``TimeoutException``.""" +def format_sandbox_start_unavailable_exception(message: str) -> Exception: + """Like :func:`format_sandbox_unavailable_exception`, for starting a command/PTY/ + watch: when the proxy reports the sandbox was not found, a + ``SandboxNotFoundException`` is returned.""" if is_sandbox_not_found_message(message): - return format_sandbox_not_found_exception(message) - if is_sandbox_port_closed_message(message): - return SandboxUnreachableException( - f"{message}: envd inside the sandbox could not be reached although the sandbox is running — e.g. its network is down. Check the sandbox state with 'Sandbox.get_info()' and retry the request." + return SandboxNotFoundException( + f"{message}: The sandbox was killed or reached its timeout. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." ) - return format_sandbox_timeout_exception(message) + return format_sandbox_unavailable_exception(message) def format_request_timeout_error() -> Exception: diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command.py b/packages/python-sdk/e2b/sandbox_async/commands/command.py index f2ad186066..2d4aea112e 100644 --- a/packages/python-sdk/e2b/sandbox_async/commands/command.py +++ b/packages/python-sdk/e2b/sandbox_async/commands/command.py @@ -14,7 +14,7 @@ from e2b.envd.process import process_connect, process_pb from e2b.envd.api import acheck_sandbox_health -from e2b.envd.rpc import ahandle_rpc_exception_with_health +from e2b.envd.rpc import START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -390,4 +390,6 @@ async def _create_handle( await events.aclose() except Exception: pass - raise await ahandle_rpc_exception_with_health(e, self._check_health) + raise await ahandle_rpc_exception_with_health( + e, self._check_health, START_RPC_ERROR_MAP + ) diff --git a/packages/python-sdk/e2b/sandbox_async/commands/pty.py b/packages/python-sdk/e2b/sandbox_async/commands/pty.py index dda9d2e6f3..bb00aa0f05 100644 --- a/packages/python-sdk/e2b/sandbox_async/commands/pty.py +++ b/packages/python-sdk/e2b/sandbox_async/commands/pty.py @@ -15,7 +15,7 @@ KEEPALIVE_PING_INTERVAL_SEC, ) from e2b.envd.api import acheck_sandbox_health -from e2b.envd.rpc import ahandle_rpc_exception_with_health +from e2b.envd.rpc import START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -178,7 +178,9 @@ async def create( await events.aclose() except Exception: pass - raise await ahandle_rpc_exception_with_health(e, self._check_health) + raise await ahandle_rpc_exception_with_health( + e, self._check_health, START_RPC_ERROR_MAP + ) async def connect( self, @@ -227,7 +229,9 @@ async def connect( await events.aclose() except Exception: pass - raise await ahandle_rpc_exception_with_health(e, self._check_health) + raise await ahandle_rpc_exception_with_health( + e, self._check_health, START_RPC_ERROR_MAP + ) async def resize( self, diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 6482eb0e03..0b9963f427 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -24,7 +24,7 @@ from protobuf import Oneof from e2b.envd.filesystem import filesystem_connect, filesystem_pb -from e2b.envd.rpc import ahandle_rpc_exception_with_health +from e2b.envd.rpc import START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health from e2b.envd.utils import authentication_header, timeout_to_ms from e2b.envd.client_async import as_stream, create_rpc_client, first_event from e2b.envd.versions import ( @@ -59,6 +59,7 @@ from e2b.sandbox_async.utils import OutputHandler _FILESYSTEM_RPC_ERROR_MAP = { + **START_RPC_ERROR_MAP, Code.NOT_FOUND: FileNotFoundException, } diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command.py b/packages/python-sdk/e2b/sandbox_sync/commands/command.py index d929301e4e..321f9a43ab 100644 --- a/packages/python-sdk/e2b/sandbox_sync/commands/command.py +++ b/packages/python-sdk/e2b/sandbox_sync/commands/command.py @@ -24,7 +24,7 @@ from e2b.envd.process import process_connect, process_pb from e2b.envd.api import check_sandbox_health -from e2b.envd.rpc import handle_rpc_exception_with_health +from e2b.envd.rpc import START_RPC_ERROR_MAP, handle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -385,4 +385,6 @@ def _create_handle( events.close() except Exception: pass - raise handle_rpc_exception_with_health(e, self._check_health) + raise handle_rpc_exception_with_health( + e, self._check_health, START_RPC_ERROR_MAP + ) diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py index 9465007759..c30edf1fb3 100644 --- a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py +++ b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py @@ -15,7 +15,7 @@ KEEPALIVE_PING_INTERVAL_SEC, ) from e2b.envd.api import check_sandbox_health -from e2b.envd.rpc import handle_rpc_exception_with_health +from e2b.envd.rpc import START_RPC_ERROR_MAP, handle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -169,7 +169,9 @@ def create( events.close() except Exception: pass - raise handle_rpc_exception_with_health(e, self._check_health) + raise handle_rpc_exception_with_health( + e, self._check_health, START_RPC_ERROR_MAP + ) def connect( self, @@ -213,7 +215,9 @@ def connect( events.close() except Exception: pass - raise handle_rpc_exception_with_health(e, self._check_health) + raise handle_rpc_exception_with_health( + e, self._check_health, START_RPC_ERROR_MAP + ) def resize( self, diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 568c26134a..3b3f0d2a1d 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -21,7 +21,7 @@ handle_envd_api_transport_exception_with_health, ) from e2b.envd.filesystem import filesystem_connect, filesystem_pb -from e2b.envd.rpc import handle_rpc_exception_with_health +from e2b.envd.rpc import START_RPC_ERROR_MAP, handle_rpc_exception_with_health from e2b.envd.utils import authentication_header, timeout_to_ms from e2b.envd.client_sync import create_rpc_client from e2b.envd.versions import ( @@ -55,6 +55,7 @@ _FILESYSTEM_RPC_ERROR_MAP = { + **START_RPC_ERROR_MAP, Code.NOT_FOUND: FileNotFoundException, } diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.py index bd058bbacf..19b4e28d05 100644 --- a/packages/python-sdk/tests/test_envd_api_exception.py +++ b/packages/python-sdk/tests/test_envd_api_exception.py @@ -16,7 +16,6 @@ RateLimitException, SandboxException, SandboxUnreachableException, - SandboxNotFoundException, TimeoutException, ) @@ -55,7 +54,7 @@ def _port_closed(request: httpx.Request): def test_maps_502_with_port_closed_to_unreachable(): err = format_envd_api_exception(502, PORT_CLOSED) assert isinstance(err, SandboxUnreachableException) - assert "envd inside the sandbox could not be reached" in str(err) + assert "The sandbox could not be reached" in str(err) def test_maps_502_to_timeout(): @@ -92,9 +91,9 @@ def test_returns_original_when_not_transport_error(): assert err is original -def test_502_sandbox_not_found_returns_not_found(): +def test_502_sandbox_not_found_returns_unreachable(): err = format_envd_api_exception(502, "The sandbox was not found") - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) assert "The sandbox was not found" in str(err) @@ -102,7 +101,7 @@ def test_health_result_confirms_sandbox_killed(): err = handle_envd_api_transport_exception( httpx.RemoteProtocolError("peer closed connection"), sandbox_running=False ) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) assert "sandbox was killed or reached its end of life" in str(err) @@ -172,12 +171,12 @@ def test_health_check_raises_when_sandbox_unreachable(): raise AssertionError("expected the probe to raise") -def test_connect_failure_with_sandbox_killed_returns_not_found(): +def test_connect_failure_with_sandbox_killed_returns_unreachable(): with _health_client(lambda request: httpx.Response(502)) as client: err = handle_envd_api_transport_exception_with_health( httpx.ConnectError("connection refused"), client ) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) def test_connect_failure_with_running_sandbox_returns_raw_error(): @@ -231,12 +230,12 @@ async def test_async_connect_failure_with_unreachable_health_returns_unreachable assert err.__cause__ is original -async def test_async_connect_failure_with_sandbox_killed_returns_not_found(): +async def test_async_connect_failure_with_sandbox_killed_returns_unreachable(): async with _ahealth_client(lambda request: httpx.Response(502)) as client: err = await ahandle_envd_api_transport_exception_with_health( httpx.ConnectError("connection refused"), client ) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) async def test_async_health_check_raises_unreachable_when_port_closed(): diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index 9ef012006f..5a056507e9 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -5,6 +5,7 @@ from pyqwest import StreamError, StreamErrorCode from e2b.envd.rpc import ( + START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health, handle_rpc_exception, handle_rpc_exception_with_health, @@ -16,8 +17,8 @@ NotFoundException, RateLimitException, SandboxException, - SandboxUnreachableException, SandboxNotFoundException, + SandboxUnreachableException, TimeoutException, ) @@ -50,7 +51,7 @@ def test_maps_resource_exhausted_to_rate_limit(): def test_maps_unavailable_with_port_closed_to_unreachable(): err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, PORT_CLOSED)) assert isinstance(err, SandboxUnreachableException) - assert "envd inside the sandbox could not be reached" in str(err) + assert "The sandbox could not be reached" in str(err) def test_maps_unavailable_to_timeout(): @@ -180,34 +181,14 @@ def test_returns_raw_transport_failure_without_health_result(): assert err is original -def test_unavailable_sandbox_not_found_returns_not_found(): +def test_unavailable_sandbox_not_found_returns_unreachable(): err = handle_rpc_exception( ConnectError(Code.UNAVAILABLE, "The sandbox was not found") ) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) assert "The sandbox was not found" in str(err) -def test_ambiguous_unavailable_probes_health_and_returns_not_found(): - err = handle_rpc_exception_with_health( - ConnectError( - Code.UNAVAILABLE, - "the connection to sandbox ended before the stream completed", - ), - lambda: False, - ) - assert isinstance(err, SandboxNotFoundException) - assert "ended before the stream completed" in str(err) - - -def test_ambiguous_unavailable_keeps_timeout_when_running(): - err = handle_rpc_exception_with_health( - ConnectError(Code.UNAVAILABLE, "stream ended"), lambda: True - ) - assert isinstance(err, TimeoutException) - assert not isinstance(err, SandboxNotFoundException) - - def test_unavailable_sandbox_not_found_does_not_probe(): probed = [] @@ -219,22 +200,51 @@ def check(): ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), check ) assert not probed - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) -async def test_async_ambiguous_unavailable_probes_health_and_returns_not_found(): +async def test_async_ambiguous_unavailable_probes_health_and_returns_unreachable(): async def check(): return False err = await ahandle_rpc_exception_with_health( ConnectError(Code.UNAVAILABLE, "stream ended"), check ) + assert isinstance(err, SandboxUnreachableException) + + +def test_start_error_map_sandbox_not_found_returns_not_found(): + err = handle_rpc_exception( + ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), + START_RPC_ERROR_MAP, + ) + assert isinstance(err, SandboxNotFoundException) + assert "The sandbox was not found" in str(err) + + +def test_start_error_map_port_closed_returns_unreachable(): + err = handle_rpc_exception( + ConnectError(Code.UNAVAILABLE, PORT_CLOSED), START_RPC_ERROR_MAP + ) + assert isinstance(err, SandboxUnreachableException) + assert not isinstance(err, SandboxNotFoundException) + + +async def test_async_start_error_map_sandbox_not_found_returns_not_found(): + async def check(): + return True + + err = await ahandle_rpc_exception_with_health( + ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), + check, + START_RPC_ERROR_MAP, + ) assert isinstance(err, SandboxNotFoundException) def test_health_check_confirms_sandbox_killed(): err = handle_rpc_exception_with_health(_stream_reset(), lambda: False) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) assert "sandbox was killed or reached its end of life" in str(err) @@ -291,9 +301,9 @@ def check(): assert err is original -def test_connect_failure_with_sandbox_killed_returns_not_found(): +def test_connect_failure_with_sandbox_killed_returns_unreachable(): err = handle_rpc_exception_with_health(_connect_refused(), lambda: False) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) def test_connect_failure_with_unreachable_health_returns_unreachable(): @@ -319,7 +329,7 @@ async def check(): return False err = await ahandle_rpc_exception_with_health(_stream_reset(), check) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxUnreachableException) assert "sandbox was killed or reached its end of life" in str(err) From ca30ac8498afc47541ada626bd1fa8cf2b847273 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:15:42 +0000 Subject: [PATCH 09/16] fix(js-sdk): let Connect-encoded 502 errors through rejectProxyUnavailableResponse so the transport decodes their declared code Co-Authored-By: mish@e2b.dev --- packages/js-sdk/src/envd/rpc.ts | 8 +++++++- packages/js-sdk/tests/envd/handleRpcError.test.ts | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 0b6d2910a2..4a2cded127 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -279,7 +279,13 @@ export async function rejectProxyUnavailableResponse( let message = '' try { - const body = await res.json() + const text = await res.text() + const body = JSON.parse(text) + if (typeof body?.code === 'string') { + // Connect-encoded error (string `code`, e.g. "permission_denied") — let the + // transport decode it with its declared code. + return new Response(text, res) + } if (typeof body?.message === 'string') { message = body.message } diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 05442ea40e..082668306c 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -356,6 +356,21 @@ describe('rejectProxyUnavailableResponse', () => { } }) + test('passes a Connect-encoded 502 error through for the transport to decode', async () => { + const body = JSON.stringify({ + code: 'permission_denied', + message: 'access denied', + }) + const res = new Response(body, { + status: 502, + headers: { 'Content-Type': 'application/json' }, + }) + const out = await rejectProxyUnavailableResponse(res) + assert.strictEqual(out.status, 502) + assert.strictEqual(out.headers.get('Content-Type'), 'application/json') + assert.deepEqual(await out.json(), JSON.parse(body)) + }) + test('falls back to the status for a 502 without a JSON body', async () => { const res = new Response('bad gateway', { status: 502 }) try { From 25d3dc5a843a9bd052832e5d6ad8e521e7068e57 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:26:56 +0000 Subject: [PATCH 10/16] fix(js-sdk): only forward 502 bodies whose string code is a Connect code; others stay Unavailable with the proxy message Co-Authored-By: mish@e2b.dev --- packages/js-sdk/src/envd/rpc.ts | 25 ++++++++++++++++--- .../js-sdk/tests/envd/handleRpcError.test.ts | 21 ++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 4a2cded127..c0d648478a 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -264,6 +264,25 @@ export function authenticationHeader( return { Authorization: `Basic ${encoded}` } } +const CONNECT_CODE_NAMES = new Set([ + 'canceled', + 'unknown', + 'invalid_argument', + 'deadline_exceeded', + 'not_found', + 'already_exists', + 'permission_denied', + 'resource_exhausted', + 'failed_precondition', + 'aborted', + 'out_of_range', + 'unimplemented', + 'internal', + 'unavailable', + 'data_loss', + 'unauthenticated', +]) + /** * Rejects a plain (non-Connect-encoded) 502 answered by the proxy in front of envd — * the sandbox is gone or envd's port is not open — as a `ConnectError` carrying the @@ -281,9 +300,9 @@ export async function rejectProxyUnavailableResponse( try { const text = await res.text() const body = JSON.parse(text) - if (typeof body?.code === 'string') { - // Connect-encoded error (string `code`, e.g. "permission_denied") — let the - // transport decode it with its declared code. + if (typeof body?.code === 'string' && CONNECT_CODE_NAMES.has(body.code)) { + // Connect-encoded error (declared string `code`, e.g. "permission_denied") — + // let the transport decode it with that code. return new Response(text, res) } if (typeof body?.message === 'string') { diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 082668306c..1543459906 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -371,6 +371,27 @@ describe('rejectProxyUnavailableResponse', () => { assert.deepEqual(await out.json(), JSON.parse(body)) }) + test('rejects a 502 whose string code is not a Connect code as Unavailable with the body message', async () => { + const res = new Response( + JSON.stringify({ + code: 'BAD_GATEWAY', + message: 'The sandbox was not found', + }), + { status: 502, headers: { 'Content-Type': 'application/json' } } + ) + try { + await rejectProxyUnavailableResponse(res) + assert.fail('expected a rejection') + } catch (err) { + assert.instanceOf(err, ConnectError) + assert.strictEqual((err as ConnectError).code, Code.Unavailable) + assert.strictEqual( + (err as ConnectError).rawMessage, + 'The sandbox was not found' + ) + } + }) + test('falls back to the status for a 502 without a JSON body', async () => { const res = new Response('bad gateway', { status: 502 }) try { From 4bd05eec1940e28da8675fd2cf1d1597ec2bf5de Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:35:27 +0000 Subject: [PATCH 11/16] fix(sdk): scope the start error map to watchDir; filesystem RPCs keep SandboxUnreachable for a not-found 502 Co-Authored-By: mish@e2b.dev --- .../js-sdk/src/sandbox/filesystem/index.ts | 23 ++++++++++++------- .../js-sdk/tests/envd/handleRpcError.test.ts | 9 ++++++-- .../sandbox_async/filesystem/filesystem.py | 18 +++++++++++---- .../e2b/sandbox_sync/filesystem/filesystem.py | 20 ++++++++++++---- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 3ff22aa867..6e3dac0d6f 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -63,19 +63,22 @@ const FILESYSTEM_HTTP_ERROR_MAP: Record Error> = { const FILESYSTEM_RPC_ERROR_MAP: Partial< Record Error> > = { - ...START_RPC_ERROR_MAP, [Code.NotFound]: (message: string) => new FileNotFoundError(message), } +const WATCH_DIR_START_RPC_ERROR_MAP: Partial< + Record Error> +> = { + ...FILESYSTEM_RPC_ERROR_MAP, + ...START_RPC_ERROR_MAP, +} + async function handleFilesystemRpcError( err: unknown, - checkHealth?: SandboxHealthCheck + checkHealth?: SandboxHealthCheck, + errorMap = FILESYSTEM_RPC_ERROR_MAP ): Promise { - return handleRpcErrorWithHealthCheck( - err, - checkHealth, - FILESYSTEM_RPC_ERROR_MAP - ) + return handleRpcErrorWithHealthCheck(err, checkHealth, errorMap) } function handleFilesystemEnvdApiError(res: { @@ -1113,7 +1116,11 @@ export class Filesystem { ) } catch (err) { cleanup() - throw await handleFilesystemRpcError(err, this.checkHealth) + throw await handleFilesystemRpcError( + err, + this.checkHealth, + WATCH_DIR_START_RPC_ERROR_MAP + ) } } } diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 1543459906..02c9844427 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -133,14 +133,19 @@ describe('handleRpcErrorWithHealthCheck', () => { Browser: 'network error', } - test('probes health on an Unavailable with an unknown message and returns SandboxUnreachableError when the sandbox is gone', async () => { + test('returns SandboxUnreachableError for an Unavailable stream termination without probing health', async () => { + let probed = false const err = await handleRpcErrorWithHealthCheck( new ConnectError( 'the connection to sandbox ended before the stream completed', Code.Unavailable ), - async () => false + async () => { + probed = true + return false + } ) + assert.isFalse(probed) assert.instanceOf(err, SandboxUnreachableError) assert.include(err.message, 'ended before the stream completed') }) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 0b9963f427..ef55da1873 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -1,5 +1,5 @@ import asyncio -from typing import IO, Dict, List, Literal, Optional, Union, overload +from typing import IO, Callable, Dict, List, Literal, Optional, Union, overload import httpx @@ -59,20 +59,26 @@ from e2b.sandbox_async.utils import OutputHandler _FILESYSTEM_RPC_ERROR_MAP = { - **START_RPC_ERROR_MAP, Code.NOT_FOUND: FileNotFoundException, } +_WATCH_DIR_START_RPC_ERROR_MAP = { + **_FILESYSTEM_RPC_ERROR_MAP, + **START_RPC_ERROR_MAP, +} + _FILESYSTEM_HTTP_ERROR_MAP = { 404: FileNotFoundException, } async def _ahandle_filesystem_rpc_exception( - e: Exception, envd_api: httpx.AsyncClient + e: Exception, + envd_api: httpx.AsyncClient, + error_map: dict[Code, Callable[[str], Exception]] = _FILESYSTEM_RPC_ERROR_MAP, ) -> Exception: return await ahandle_rpc_exception_with_health( - e, lambda: acheck_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP + e, lambda: acheck_sandbox_health(envd_api), error_map ) @@ -757,4 +763,6 @@ async def watch_dir( await events.aclose() except Exception: pass - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception( + e, self._envd_api, _WATCH_DIR_START_RPC_ERROR_MAP + ) diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 3b3f0d2a1d..454551408c 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,4 +1,4 @@ -from typing import IO, Dict, List, Literal, Optional, Union, overload +from typing import IO, Callable, Dict, List, Literal, Optional, Union, overload import httpx from connectrpc.code import Code @@ -55,18 +55,26 @@ _FILESYSTEM_RPC_ERROR_MAP = { - **START_RPC_ERROR_MAP, Code.NOT_FOUND: FileNotFoundException, } +_WATCH_DIR_START_RPC_ERROR_MAP = { + **_FILESYSTEM_RPC_ERROR_MAP, + **START_RPC_ERROR_MAP, +} + _FILESYSTEM_HTTP_ERROR_MAP = { 404: FileNotFoundException, } -def _handle_filesystem_rpc_exception(e: Exception, envd_api: httpx.Client) -> Exception: +def _handle_filesystem_rpc_exception( + e: Exception, + envd_api: httpx.Client, + error_map: dict[Code, Callable[[str], Exception]] = _FILESYSTEM_RPC_ERROR_MAP, +) -> Exception: return handle_rpc_exception_with_health( - e, lambda: check_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP + e, lambda: check_sandbox_health(envd_api), error_map ) @@ -686,7 +694,9 @@ def watch_dir( }, ) except Exception as e: - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception( + e, self._envd_api, _WATCH_DIR_START_RPC_ERROR_MAP + ) return WatchHandle( self._rpc, From 8a0c90b2c5edb7d502ade0afd691e529c125e5f2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:10:43 +0000 Subject: [PATCH 12/16] feat(sdk): add SandboxNotRunningError/Exception for a proxy or probe 'sandbox was not found' A 502/Unavailable 'The sandbox was not found' from any envd call, or from the health probe after a connection-level failure, is now SandboxNotRunningError (TimeoutError subclass) instead of SandboxNotFoundError on the start paths and SandboxUnreachableError elsewhere; the start-only error map is gone. Port not open / other 502 / probe without an answer stay SandboxUnreachableError. Also: keep the plain-text body of a 502 as the message in JS, use connect's codeFromString instead of a hand-maintained code set, tolerate non-object JSON 502 bodies in the Python health probe, report the probe's failure in the unreachable error and keep the failed request as cause, drop the dead formatSandboxTimeoutError, fix docstrings and the changeset. Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 9 ++- packages/js-sdk/src/envd/api.ts | 14 ++-- packages/js-sdk/src/envd/rpc.ts | 77 ++++++++----------- packages/js-sdk/src/errors.ts | 65 ++++++++-------- packages/js-sdk/src/index.ts | 1 + packages/js-sdk/src/sandbox/commands/index.ts | 7 +- packages/js-sdk/src/sandbox/commands/pty.ts | 13 +--- .../js-sdk/src/sandbox/filesystem/index.ts | 14 +--- .../tests/envd/handleEnvdApiError.test.ts | 49 +++++++++--- .../js-sdk/tests/envd/handleRpcError.test.ts | 73 ++++++++++-------- packages/python-sdk/e2b/__init__.py | 2 + packages/python-sdk/e2b/envd/api.py | 27 ++++--- packages/python-sdk/e2b/envd/rpc.py | 30 ++++---- packages/python-sdk/e2b/exceptions.py | 56 +++++++------- .../e2b/sandbox_async/commands/command.py | 6 +- .../e2b/sandbox_async/commands/pty.py | 10 +-- .../sandbox_async/filesystem/filesystem.py | 11 +-- .../e2b/sandbox_sync/commands/command.py | 6 +- .../e2b/sandbox_sync/commands/pty.py | 10 +-- .../e2b/sandbox_sync/filesystem/filesystem.py | 11 +-- .../tests/test_envd_api_exception.py | 58 +++++++++++--- .../tests/test_envd_rpc_exception.py | 60 +++++++-------- 22 files changed, 299 insertions(+), 310 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index 7e576ea373..f18d425ba5 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -3,10 +3,11 @@ 'e2b': patch --- -Add `SandboxUnreachableError` (JS) / `SandboxUnreachableException` (Python), raised when the sandbox cannot be reached while it is not confirmed to be stopped: the proxy in front of the sandbox reports it running but envd's port not open (e.g. the sandbox's network is down), or a request to the sandbox fails at the connection level and the follow-up health probe gets no answer either. It is a subclass of the sandbox-timeout `TimeoutError` / `TimeoutException` these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working. `isRunning()` / `is_running()` still return `false` for such a sandbox. +Classify requests that fail because the sandbox cannot be reached. Two new error classes, both subclasses of the sandbox-timeout `TimeoutError` (JS) / `TimeoutException` (Python) these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working: -Every `502` / `Unavailable` answered by the proxy in front of the sandbox — the sandbox was not found (killed or reached its timeout) or it is running but envd's port is not open (e.g. `ip link set eth0 down` inside it) — now raises `SandboxUnreachableError` / `SandboxUnreachableException` from every envd request, as does a request whose connection dropped because the sandbox was killed. The new error extends `TimeoutError` / `TimeoutException`, so existing timeout handling keeps working. Starting a command / PTY / directory watch on a sandbox the proxy reports as not found keeps raising `SandboxNotFoundError` in JS and now raises `SandboxNotFoundException` in Python (instead of `TimeoutException`). +- `SandboxNotRunningError` / `SandboxNotRunningException` — the proxy in front of the sandbox answered `502 "The sandbox was not found"` (killed or reached its timeout), from any envd request (RPC or files API) or from the health probe run after a request failed at the connection level (e.g. the sandbox killed while a command was running). +- `SandboxUnreachableError` / `SandboxUnreachableException` — the sandbox is not confirmed stopped but cannot be reached: the proxy reports it running but envd's port not open (e.g. `ip link set eth0 down` inside the sandbox), any other `502`, or a request failed at the connection level and the follow-up health probe got no answer either. The failed request is the error's `cause` / `__cause__`. -Requests whose connection to the sandbox could not be established now run the same health probe as requests whose connection was dropped mid-request, so a killed sandbox surfaces as a `TimeoutError` instead of a raw connection error. In JS this also covers the opaque `TypeError` browsers raise for network failures (`Failed to fetch`, `Load failed`, `NetworkError when attempting to fetch resource.`). +Requests whose connection could not be established (`ECONNREFUSED`, DNS, unreachable host, browsers' `Failed to fetch`, `httpx.ConnectError`, …) now run the same health probe as requests whose connection dropped mid-request; when the probe says the sandbox is running (or is inconclusive) the original transport error is raised unchanged. `isRunning()` / `is_running()` still return `false` for any `502`. -JS: starting a command, PTY or directory watch on a sandbox whose envd answers `Unavailable` goes through the regular RPC error mapping like every other request (`SandboxNotFoundError` with the proxy's message when the sandbox was not found, `SandboxUnreachableError` otherwise), instead of always `SandboxNotFoundError: Sandbox is probably not running anymore`. This matches the Python SDK and gives the CLI (`sandbox exec` / `connect`) a correct error. +JS: starting a command, PTY or directory watch on a sandbox that is gone raises `SandboxNotRunningError` instead of `SandboxNotFoundError: Sandbox is probably not running anymore`, like every other request (Python: `SandboxNotRunningException` instead of `TimeoutException`). The CLI's `sandbox exec` / `sandbox connect` print that error instead of a stack trace. Plain-text `502` bodies keep the proxy's message in JS, matching Python. diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts index 5cf7f6ca0e..f92223c940 100644 --- a/packages/js-sdk/src/envd/api.ts +++ b/packages/js-sdk/src/envd/api.ts @@ -9,7 +9,7 @@ import { NotFoundError, NotEnoughSpaceError, formatSandboxUnavailableError, - isSandboxPortClosedMessage, + SandboxNotRunningError, AuthenticationError, RateLimitError, } from '../errors' @@ -41,7 +41,7 @@ const HEALTH_CHECK_TIMEOUT_MS = 5_000 * * @param envdApi - The envd API client of the sandbox. * @returns `true` if the sandbox is running, `false` if it is not, `undefined` if it answered but its state could not be determined. - * @throws The `fetch` failure when the sandbox could not be reached (no answer within the probe timeout), or a `SandboxUnreachableError` when the proxy reports the sandbox running but envd's port not open. + * @throws The `fetch` failure when the sandbox could not be reached (no answer within the probe timeout), or the `SandboxUnreachableError` a request would get when the proxy reports the sandbox running but envd's port not open. */ export async function checkSandboxHealth( envdApi: EnvdApiClient @@ -51,11 +51,11 @@ export async function checkSandboxHealth( }) if (res.response.status === 502) { - const message = await getEnvdApiErrorMessage(res) - if (isSandboxPortClosedMessage(message)) { - throw formatSandboxUnavailableError(message) + const err = formatSandboxUnavailableError(await getEnvdApiErrorMessage(res)) + if (err instanceof SandboxNotRunningError) { + return false } - return false + throw err } if (res.response.ok) { return true @@ -84,7 +84,7 @@ export function isFetchTransportFailure(err: unknown): err is Error { * * @param err - The caught error, expected to be a fetch transport failure. * @param checkHealth - Probe resolving to whether the sandbox is running (`undefined` when unknown) and rejecting when it cannot be reached. - * @returns A `SandboxUnreachableError` when the sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, or the original error otherwise. + * @returns A `SandboxNotRunningError` when the sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, or the original error otherwise. */ export async function handleEnvdApiFetchError( err: unknown, diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index c0d648478a..0c638bb4a4 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -1,16 +1,17 @@ import { Code, ConnectError } from '@connectrpc/connect' +import { codeFromString } from '@connectrpc/connect/protocol-connect' import { runtime } from '../utils' import { compareVersions } from 'compare-versions' import { defaultUsername } from '../connectionConfig' import { AuthenticationError, - formatSandboxStartUnavailableError, formatSandboxUnavailableError, InvalidArgumentError, NotFoundError, RateLimitError, SandboxError, + SandboxNotRunningError, SandboxUnreachableError, TimeoutError, } from '../errors' @@ -93,30 +94,33 @@ export function isTransportFailure(err: unknown): boolean { /** * Builds the error for a request that failed at the connection level when the - * follow-up health probe got no answer from the sandbox either. A probe answered - * by the proxy with the sandbox running but envd's port not open is already a - * `SandboxUnreachableError` and is kept, with the failed request as its cause. + * follow-up health probe failed too. A probe answered by the proxy with the sandbox + * running but envd's port not open is already a `SandboxUnreachableError` and is + * kept, with the failed request as its cause; otherwise the probe's failure is + * reported alongside the request's, which becomes the cause. */ export function formatSandboxUnreachableError( err: Error, - probeErr?: unknown + probeErr: unknown ): Error { if (probeErr instanceof SandboxUnreachableError) { probeErr.cause = err return probeErr } + const probeMessage = + probeErr instanceof Error ? probeErr.message : String(probeErr) return new SandboxUnreachableError( - `${err.message}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.getInfo()' and retry the request.`, + `${err.message}: The sandbox could not be reached and its health probe failed too (${probeMessage}). It was not confirmed to be stopped — this is likely a network issue or envd inside the sandbox not being up yet; check the sandbox state with 'Sandbox.getInfo()'.`, { cause: err } ) } /** * Runs the health probe for a request that failed at the connection level and - * returns the typed error for the outcome: a `TimeoutError` when the sandbox is - * confirmed gone, a `SandboxUnreachableError` when the probe got no answer either, - * or `undefined` when the sandbox answered (the failure was transient and the - * original error applies). + * returns the typed error for the outcome: a `SandboxNotRunningError` when the + * sandbox is confirmed gone, a `SandboxUnreachableError` when the probe got no + * answer either, or `undefined` when the sandbox answered (the failure was + * transient and the original error applies). */ export async function resolveTransportFailure( err: Error, @@ -130,7 +134,7 @@ export async function resolveTransportFailure( } if (running === false) { - return new SandboxUnreachableError( + return new SandboxNotRunningError( `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.`, { cause: err } ) @@ -139,16 +143,6 @@ export async function resolveTransportFailure( return undefined } -/** - * Error map for starting a command/PTY/watch: a 502 "sandbox was not found" from the - * proxy is a `SandboxNotFoundError`, every other `Unavailable` a `SandboxUnreachableError`. - */ -export const START_RPC_ERROR_MAP: Partial< - Record Error> -> = { - [Code.Unavailable]: formatSandboxStartUnavailableError, -} - const DEFAULT_ERROR_MAP: Partial Error>> = { [Code.InvalidArgument]: (message) => new InvalidArgumentError(message), [Code.Unauthenticated]: (message) => new AuthenticationError(message), @@ -203,8 +197,9 @@ export function handleRpcError( * mid-request) it probes the sandbox health to tell apart the sandbox being killed * from the sandbox being unreachable or a transient network failure (e.g. a load * balancer dropping the connection). When the probe confirms the sandbox is gone, a - * `TimeoutError` is returned — consistent with how requests to an already-dead - * sandbox surface; when the probe gets no answer either, a `SandboxUnreachableError`. + * `SandboxNotRunningError` is returned — the same error a request answered by the + * proxy with the sandbox not found gets; when the probe gets no answer either, a + * `SandboxUnreachableError`. * * @param err - The caught error, expected to be a `ConnectError` from the gRPC transport. * @param checkHealth - Probe resolving to whether the sandbox is running (`undefined` when unknown) and rejecting when it cannot be reached. @@ -264,25 +259,6 @@ export function authenticationHeader( return { Authorization: `Basic ${encoded}` } } -const CONNECT_CODE_NAMES = new Set([ - 'canceled', - 'unknown', - 'invalid_argument', - 'deadline_exceeded', - 'not_found', - 'already_exists', - 'permission_denied', - 'resource_exhausted', - 'failed_precondition', - 'aborted', - 'out_of_range', - 'unimplemented', - 'internal', - 'unavailable', - 'data_loss', - 'unauthenticated', -]) - /** * Rejects a plain (non-Connect-encoded) 502 answered by the proxy in front of envd — * the sandbox is gone or envd's port is not open — as a `ConnectError` carrying the @@ -298,18 +274,25 @@ export async function rejectProxyUnavailableResponse( let message = '' try { - const text = await res.text() - const body = JSON.parse(text) - if (typeof body?.code === 'string' && CONNECT_CODE_NAMES.has(body.code)) { + message = await res.text() + } catch { + // unreadable body + } + try { + const body = JSON.parse(message) + if ( + typeof body?.code === 'string' && + codeFromString(body.code) !== undefined + ) { // Connect-encoded error (declared string `code`, e.g. "permission_denied") — // let the transport decode it with that code. - return new Response(text, res) + return new Response(message, res) } if (typeof body?.message === 'string') { message = body.message } } catch { - // not a JSON body + // not a JSON body: keep the proxy's plain-text message } throw new ConnectError(message || `HTTP ${res.status}`, Code.Unavailable) diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index b14d3d46dd..2845891cd4 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -1,48 +1,27 @@ -// Message of the 502 the proxy in front of envd answers with when the sandbox -// is running but nothing listens on envd's port — e.g. its network is down. -// This is the message for the sandbox timeout error when the response code is 502/Unavailable -export function formatSandboxTimeoutError(message: string) { - return new TimeoutError( - `${message}: This error is likely due to sandbox timeout. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` - ) -} - // Messages of the 502 the proxy in front of envd answers with when the sandbox does // not exist anymore (killed or reached its timeout) or when it is running but envd's // port is not open (e.g. its network is down). const SANDBOX_NOT_FOUND_MESSAGE = 'was not found' -const SANDBOX_PORT_CLOSED_MESSAGE = 'port is not open' export function isSandboxNotFoundMessage(message: string): boolean { return message.includes(SANDBOX_NOT_FOUND_MESSAGE) } -export function isSandboxPortClosedMessage(message: string): boolean { - return message.includes(SANDBOX_PORT_CLOSED_MESSAGE) -} - /** - * Maps a 502/Unavailable answered by the proxy in front of envd — the sandbox was - * not found (killed or reached its timeout), or it is running but envd's port is not - * open (e.g. its network is down) — to a `SandboxUnreachableError`. + * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox was not + * found (killed or reached its timeout) is a `SandboxNotRunningError`; the sandbox + * running but envd's port not open (e.g. its network is down), or any other message, + * a `SandboxUnreachableError`. */ export function formatSandboxUnavailableError(message: string): Error { - return new SandboxUnreachableError( - `${message}: The sandbox could not be reached — it was killed, reached its timeout, or envd inside it is not reachable (e.g. its network is down). Check the sandbox state with 'Sandbox.getInfo()'; you can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` - ) -} - -/** - * Like `formatSandboxUnavailableError`, for starting a command/PTY/watch: when the - * proxy reports the sandbox was not found, a `SandboxNotFoundError` is returned. - */ -export function formatSandboxStartUnavailableError(message: string): Error { if (isSandboxNotFoundMessage(message)) { - return new SandboxNotFoundError( + return new SandboxNotRunningError( `${message}: The sandbox was killed or reached its timeout. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` ) } - return formatSandboxUnavailableError(message) + return new SandboxUnreachableError( + `${message}: The sandbox is running but envd inside it could not be reached (e.g. its network is down). Check the sandbox state with 'Sandbox.getInfo()'.` + ) } /** @@ -136,16 +115,34 @@ export class SandboxNotFoundError extends NotFoundError { } /** - * Thrown when the sandbox could not be reached while not confirmed to be + * Thrown when the sandbox is not running anymore: the proxy in front of it + * answered a request (or the health probe run after a request failed at the + * connection level) with the sandbox not found — it was killed or reached its + * timeout. Retrying the request will not help; create or resume a sandbox. + * + * Subclass of `TimeoutError`, which this case surfaced as before. + */ +export class SandboxNotRunningError extends TimeoutError { + constructor(message: string, options?: ErrorOptions) { + super(message) + this.name = 'SandboxNotRunningError' + if (options?.cause !== undefined) { + this.cause = options.cause + } + } +} + +/** + * Thrown when the sandbox could not be reached while it is not confirmed to be * stopped: either the proxy in front of the sandbox reports it running but * envd's port not open (e.g. the sandbox's network is down), or a request * failed at the connection level (the connection could not be established or - * was dropped mid-request) and a follow-up health probe got no answer from the - * sandbox either. + * was dropped mid-request) and the follow-up health probe got no answer from the + * sandbox either. A sandbox confirmed stopped is a `SandboxNotRunningError`. * * This usually means the sandbox's network is down, envd inside it is not up - * (yet), or a transient network issue between the client and the sandbox. - * Check the sandbox state with `Sandbox.getInfo()` and retry the request. + * (yet), or a network issue between the client and the sandbox. Check the + * sandbox state with `Sandbox.getInfo()`. * * Subclass of `TimeoutError`, which these cases surfaced as before. */ diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index d9d26c28d1..1799643be3 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -18,6 +18,7 @@ export { NotFoundError, SandboxError, SandboxNotFoundError, + SandboxNotRunningError, SandboxUnreachableError, TemplateError, TimeoutError, diff --git a/packages/js-sdk/src/sandbox/commands/index.ts b/packages/js-sdk/src/sandbox/commands/index.ts index a9569d0b17..a9905d084d 100644 --- a/packages/js-sdk/src/sandbox/commands/index.ts +++ b/packages/js-sdk/src/sandbox/commands/index.ts @@ -29,7 +29,6 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, - START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE } from '../../envd/versions' @@ -476,11 +475,7 @@ export class Commands { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck( - err, - this.checkHealth, - START_RPC_ERROR_MAP - ) + throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) } } } diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.ts index c65bca09d1..29928a92f3 100644 --- a/packages/js-sdk/src/sandbox/commands/pty.ts +++ b/packages/js-sdk/src/sandbox/commands/pty.ts @@ -22,7 +22,6 @@ import { CommandHandle } from './commandHandle' import { authenticationHeader, handleRpcErrorWithHealthCheck, - START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { @@ -162,11 +161,7 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck( - err, - this.checkHealth, - START_RPC_ERROR_MAP - ) + throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) } } @@ -223,11 +218,7 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck( - err, - this.checkHealth, - START_RPC_ERROR_MAP - ) + throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) } } diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 6e3dac0d6f..712cea23fd 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -25,7 +25,6 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, - START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' @@ -66,13 +65,6 @@ const FILESYSTEM_RPC_ERROR_MAP: Partial< [Code.NotFound]: (message: string) => new FileNotFoundError(message), } -const WATCH_DIR_START_RPC_ERROR_MAP: Partial< - Record Error> -> = { - ...FILESYSTEM_RPC_ERROR_MAP, - ...START_RPC_ERROR_MAP, -} - async function handleFilesystemRpcError( err: unknown, checkHealth?: SandboxHealthCheck, @@ -1116,11 +1108,7 @@ export class Filesystem { ) } catch (err) { cleanup() - throw await handleFilesystemRpcError( - err, - this.checkHealth, - WATCH_DIR_START_RPC_ERROR_MAP - ) + throw await handleFilesystemRpcError(err, this.checkHealth) } } } diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts index f79d8355d6..10bfa0a7ea 100644 --- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts +++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts @@ -11,6 +11,7 @@ import { NotFoundError, RateLimitError, SandboxError, + SandboxNotRunningError, SandboxUnreachableError, TimeoutError, } from '../../src/errors' @@ -110,18 +111,26 @@ describe('handleEnvdApiError', () => { const res = createMockResponse(502, { message: PORT_CLOSED }) const err = await handleEnvdApiError(res) assert.instanceOf(err, SandboxUnreachableError) - assert.include(err!.message, 'The sandbox could not be reached') + assert.include(err!.message, 'could not be reached') }) - test('returns SandboxUnreachableError for 502 for the sandbox not found', async () => { + test('returns SandboxNotRunningError for 502 for the sandbox not found', async () => { const res = createMockResponse(502, { message: 'The sandbox was not found', }) const err = await handleEnvdApiError(res) - assert.instanceOf(err, SandboxUnreachableError) + assert.instanceOf(err, SandboxNotRunningError) + assert.instanceOf(err, TimeoutError) + assert.notInstanceOf(err, SandboxUnreachableError) assert.include(err!.message, 'The sandbox was not found') }) + test('returns SandboxNotRunningError for a plain-text 502 for the sandbox not found', async () => { + const res = createMockResponse(502, 'The sandbox was not found') + const err = await handleEnvdApiError(res) + assert.instanceOf(err, SandboxNotRunningError) + }) + test('returns TimeoutError for 502', async () => { const res = createMockResponse(502, { message: 'Bad gateway' }) const err = await handleEnvdApiError(res) @@ -165,13 +174,13 @@ describe('handleEnvdApiFetchError', () => { assert.strictEqual(err, original) }) - test('returns a SandboxUnreachableError when the health check says the sandbox is not running', async () => { - const err = await handleEnvdApiFetchError( - new TypeError('terminated'), - async () => false - ) - assert.instanceOf(err, SandboxUnreachableError) + test('returns a SandboxNotRunningError when the health check says the sandbox is not running', async () => { + const original = new TypeError('terminated') + const err = await handleEnvdApiFetchError(original, async () => false) + assert.instanceOf(err, SandboxNotRunningError) + assert.notInstanceOf(err, SandboxUnreachableError) assert.include(err.message, 'sandbox was killed or reached its end of life') + assert.strictEqual(err.cause, original) }) // Each JS runtime surfaces a dropped connection with different wording, and not @@ -187,7 +196,7 @@ describe('handleEnvdApiFetchError', () => { for (const [runtime, error] of Object.entries(runtimeTerminatedErrors)) { test(`treats the ${runtime} dropped-connection error as terminated`, async () => { const err = await handleEnvdApiFetchError(error, async () => false) - assert.instanceOf(err, SandboxUnreachableError) + assert.instanceOf(err, SandboxNotRunningError) assert.include( err.message, 'sandbox was killed or reached its end of life' @@ -214,6 +223,7 @@ describe('handleEnvdApiFetchError', () => { }) assert.instanceOf(err, SandboxUnreachableError) assert.include(err.message, 'could not be reached') + assert.include(err.message, 'fetch failed') assert.strictEqual(err.cause, original) }) @@ -228,12 +238,12 @@ describe('handleEnvdApiFetchError', () => { assert.strictEqual(err, original) }) - test('returns a SandboxUnreachableError for a refused connection when the sandbox is gone', async () => { + test('returns a SandboxNotRunningError for a refused connection when the sandbox is gone', async () => { const err = await handleEnvdApiFetchError( connectRefused(), async () => false ) - assert.instanceOf(err, SandboxUnreachableError) + assert.instanceOf(err, SandboxNotRunningError) }) test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { @@ -262,10 +272,25 @@ describe('checkSandboxHealth', () => { assert.strictEqual(await checkSandboxHealth(api), false) }) + test('reports the sandbox gone for a plain-text 502', async () => { + const api = healthApi(502, 'The sandbox was not found') + assert.strictEqual(await checkSandboxHealth(api), false) + }) + test('is inconclusive for other statuses', async () => { assert.strictEqual(await checkSandboxHealth(healthApi(500)), undefined) }) + test('throws SandboxUnreachableError for a 502 with an unrecognized message', async () => { + try { + await checkSandboxHealth(healthApi(502, { message: 'Bad gateway' })) + assert.fail('expected the probe to throw') + } catch (err) { + assert.instanceOf(err, SandboxUnreachableError) + assert.notInstanceOf(err, SandboxNotRunningError) + } + }) + test('throws SandboxUnreachableError for 502 with the sandbox running but its port closed', async () => { const api = healthApi(502, { message: PORT_CLOSED }) try { diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 02c9844427..28f24f653f 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -3,7 +3,6 @@ import { Code, ConnectError } from '@connectrpc/connect' import { handleRpcError, handleRpcErrorWithHealthCheck, - START_RPC_ERROR_MAP, isTransportFailure, rejectProxyUnavailableResponse, } from '../../src/envd/rpc' @@ -17,7 +16,7 @@ import { NotFoundError, RateLimitError, SandboxError, - SandboxNotFoundError, + SandboxNotRunningError, SandboxUnreachableError, TimeoutError, } from '../../src/errors' @@ -68,33 +67,23 @@ describe('handleRpcError', () => { test('returns SandboxUnreachableError for Unavailable with the sandbox running but its port closed', () => { const err = handleRpcError(new ConnectError(PORT_CLOSED, Code.Unavailable)) assert.instanceOf(err, SandboxUnreachableError) - assert.include(err.message, 'The sandbox could not be reached') + assert.include(err.message, 'could not be reached') }) - test('START_RPC_ERROR_MAP maps Unavailable with the sandbox not found to SandboxNotFoundError', () => { + test('returns SandboxNotRunningError for Unavailable with the sandbox not found', () => { const err = handleRpcError( - new ConnectError('The sandbox was not found', Code.Unavailable), - START_RPC_ERROR_MAP + new ConnectError('The sandbox was not found', Code.Unavailable) ) - assert.instanceOf(err, SandboxNotFoundError) + assert.instanceOf(err, SandboxNotRunningError) + assert.instanceOf(err, TimeoutError) + assert.notInstanceOf(err, SandboxUnreachableError) assert.include(err.message, 'The sandbox was not found') }) - test('START_RPC_ERROR_MAP maps Unavailable with the port closed to SandboxUnreachableError', () => { - const err = handleRpcError( - new ConnectError(PORT_CLOSED, Code.Unavailable), - START_RPC_ERROR_MAP - ) - assert.instanceOf(err, SandboxUnreachableError) - assert.notInstanceOf(err, SandboxNotFoundError) - }) - - test('returns SandboxUnreachableError for Unavailable for the sandbox not found', () => { - const err = handleRpcError( - new ConnectError('The sandbox was not found', Code.Unavailable) - ) + test('returns SandboxUnreachableError, not SandboxNotRunningError, for Unavailable with the port closed', () => { + const err = handleRpcError(new ConnectError(PORT_CLOSED, Code.Unavailable)) assert.instanceOf(err, SandboxUnreachableError) - assert.include(err.message, 'The sandbox was not found') + assert.notInstanceOf(err, SandboxNotRunningError) }) test('returns TimeoutError for Unavailable', () => { @@ -160,16 +149,17 @@ describe('handleRpcErrorWithHealthCheck', () => { } ) assert.isFalse(probed) - assert.instanceOf(err, SandboxUnreachableError) + assert.instanceOf(err, SandboxNotRunningError) }) - test('returns a SandboxUnreachableError when the health check says the sandbox is not running', async () => { - const err = await handleRpcErrorWithHealthCheck( - terminated(), - async () => false - ) - assert.instanceOf(err, SandboxUnreachableError) + test('returns a SandboxNotRunningError when the health check says the sandbox is not running', async () => { + const original = terminated() + const err = await handleRpcErrorWithHealthCheck(original, async () => false) + assert.instanceOf(err, SandboxNotRunningError) + assert.instanceOf(err, TimeoutError) + assert.notInstanceOf(err, SandboxUnreachableError) assert.include(err.message, 'sandbox was killed or reached its end of life') + assert.strictEqual(err.cause, original) }) for (const [runtime, message] of Object.entries(runtimeTerminatedMessages)) { @@ -178,7 +168,7 @@ describe('handleRpcErrorWithHealthCheck', () => { new ConnectError(message, Code.Unknown), async () => false ) - assert.instanceOf(err, SandboxUnreachableError) + assert.instanceOf(err, SandboxNotRunningError) assert.include( err.message, 'sandbox was killed or reached its end of life' @@ -223,6 +213,7 @@ describe('handleRpcErrorWithHealthCheck', () => { }) assert.instanceOf(err, SandboxUnreachableError) assert.include(err.message, 'could not be reached') + assert.include(err.message, 'health check failed') assert.strictEqual(err.cause, original) }) @@ -280,12 +271,12 @@ describe('handleRpcErrorWithHealthCheck', () => { assert.notInstanceOf(err, SandboxUnreachableError) }) - test('returns a SandboxUnreachableError for a refused connection when the sandbox is gone', async () => { + test('returns a SandboxNotRunningError for a refused connection when the sandbox is gone', async () => { const err = await handleRpcErrorWithHealthCheck( connectRefused(), async () => false ) - assert.instanceOf(err, SandboxUnreachableError) + assert.instanceOf(err, SandboxNotRunningError) }) test('returns a SandboxUnreachableError for a refused connection when the probe gets no answer', async () => { @@ -397,8 +388,24 @@ describe('rejectProxyUnavailableResponse', () => { } }) - test('falls back to the status for a 502 without a JSON body', async () => { - const res = new Response('bad gateway', { status: 502 }) + test('keeps the plain-text body of a 502 as the message', async () => { + const res = new Response('The sandbox was not found', { status: 502 }) + try { + await rejectProxyUnavailableResponse(res) + assert.fail('expected a rejection') + } catch (err) { + assert.instanceOf(err, ConnectError) + assert.strictEqual((err as ConnectError).code, Code.Unavailable) + assert.strictEqual( + (err as ConnectError).rawMessage, + 'The sandbox was not found' + ) + assert.instanceOf(handleRpcError(err), SandboxNotRunningError) + } + }) + + test('falls back to the status for a 502 without a body', async () => { + const res = new Response(null, { status: 502 }) try { await rejectProxyUnavailableResponse(res) assert.fail('expected a rejection') diff --git a/packages/python-sdk/e2b/__init__.py b/packages/python-sdk/e2b/__init__.py index 259a1bb12d..93c67b65bf 100644 --- a/packages/python-sdk/e2b/__init__.py +++ b/packages/python-sdk/e2b/__init__.py @@ -51,6 +51,7 @@ ServiceBusyException, SandboxException, SandboxNotFoundException, + SandboxNotRunningException, SandboxUnreachableException, TemplateException, TimeoutException, @@ -185,6 +186,7 @@ "NotFoundException", "FileNotFoundException", "SandboxNotFoundException", + "SandboxNotRunningException", "SandboxUnreachableException", "AuthenticationException", "GitAuthException", diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.py index d0cf3dc815..4ffc69ebb6 100644 --- a/packages/python-sdk/e2b/envd/api.py +++ b/packages/python-sdk/e2b/envd/api.py @@ -1,5 +1,4 @@ import httpx -import json from typing import Callable, Optional @@ -14,8 +13,8 @@ InvalidArgumentException, NotEnoughSpaceException, RateLimitException, + SandboxNotRunningException, format_sandbox_unavailable_exception, - is_sandbox_port_closed_message, ) @@ -56,6 +55,7 @@ def check_sandbox_health(envd_api: httpx.Client) -> Optional[bool]: :return: ``True`` if the sandbox is running, ``False`` if it is not, ``None`` if it answered but its state could not be determined. :raises httpx.TransportError: When the sandbox could not be reached (no answer within the probe timeout). + :raises SandboxUnreachableException: When the proxy reports the sandbox running but envd's port not open -- the exception a request would get. """ try: r = envd_api.get(ENVD_API_HEALTH_ROUTE, timeout=HEALTH_CHECK_TIMEOUT) @@ -79,10 +79,10 @@ async def acheck_sandbox_health(envd_api: httpx.AsyncClient) -> Optional[bool]: def _health_status(r: httpx.Response) -> Optional[bool]: if r.status_code == 502: - message = get_message(r) - if is_sandbox_port_closed_message(message): - raise format_sandbox_unavailable_exception(message) - return False + err = format_sandbox_unavailable_exception(get_message(r)) + if isinstance(err, SandboxNotRunningException): + return False + raise err if r.is_success: return True return None @@ -96,7 +96,7 @@ def handle_envd_api_transport_exception( :param e: The caught exception, expected to be a transport-level ``httpx`` error. :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a request that failed at the connection level. - :return: A ``TimeoutException`` when the request failed at the connection level and the sandbox is confirmed gone, or the original exception unchanged otherwise. + :return: A ``SandboxNotRunningException`` when the request failed at the connection level and the sandbox is confirmed gone, or the original exception unchanged otherwise. """ if isinstance(e, ENVD_API_TRANSPORT_ERRORS): return format_terminated_exception(e, sandbox_running) @@ -139,12 +139,15 @@ async def ahandle_envd_api_transport_exception_with_health( def get_message(e: httpx.Response) -> str: + """The error message of a response: the string ``message`` of its JSON object + body, else its text.""" try: - message = e.json().get("message", e.text) - except json.JSONDecodeError: - message = e.text - - return message + body = e.json() + except ValueError: + return e.text + if isinstance(body, dict) and isinstance(body.get("message"), str): + return body["message"] + return e.text def handle_envd_api_exception( diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index 5aec9f7930..0f8428c015 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -9,21 +9,14 @@ SandboxException, InvalidArgumentException, NotFoundException, + SandboxNotRunningException, SandboxUnreachableException, TimeoutException, - format_sandbox_start_unavailable_exception, format_sandbox_unavailable_exception, AuthenticationException, RateLimitException, ) -# Error map for starting a command/PTY/watch: a 502 "sandbox was not found" from the -# proxy is a ``SandboxNotFoundException``, every other ``UNAVAILABLE`` a -# ``SandboxUnreachableException``. -START_RPC_ERROR_MAP: dict[Code, Callable[[str], Exception]] = { - Code.UNAVAILABLE: format_sandbox_start_unavailable_exception, -} - _DEFAULT_RPC_ERROR_MAP: dict[Code, Callable[[str], Exception]] = { Code.INVALID_ARGUMENT: InvalidArgumentException, Code.UNAUTHENTICATED: AuthenticationException, @@ -74,27 +67,30 @@ def format_terminated_exception( ) -> Exception: """Handle an exception for a request that failed at the connection level: when a sandbox health probe confirmed the sandbox is gone (``sandbox_running is False``), - return a ``SandboxUnreachableException``; otherwise return the original error unchanged.""" + return a ``SandboxNotRunningException``; otherwise return the original error unchanged.""" if sandbox_running is False: - return SandboxUnreachableException( + err = SandboxNotRunningException( f"{e}: The sandbox was killed or reached its end of life while the request was in flight." ) + err.__cause__ = e + return err return e def format_sandbox_unreachable_exception( - e: Exception, probe_error: Optional[Exception] = None + e: Exception, probe_error: Exception ) -> Exception: """Build the exception for a request that failed at the connection level when the - follow-up sandbox health probe got no answer from the sandbox either. A probe - answered by the proxy with the sandbox running but envd's port not open is - already a ``SandboxUnreachableException`` and is kept, with the failed request - as its cause.""" + follow-up sandbox health probe failed too. A probe answered by the proxy with the + sandbox running but envd's port not open is already a + ``SandboxUnreachableException`` and is kept, with the failed request as its + cause; otherwise the probe's failure is reported alongside the request's, which + becomes the cause.""" if isinstance(probe_error, SandboxUnreachableException): probe_error.__cause__ = e return probe_error err = SandboxUnreachableException( - f"{e}: The sandbox could not be reached. It was not confirmed to be stopped, so this is likely a transient network issue or envd inside the sandbox not being up yet — check the sandbox state with 'Sandbox.get_info()' and retry the request." + f"{e}: The sandbox could not be reached and its health probe failed too ({probe_error}). It was not confirmed to be stopped — this is likely a network issue or envd inside the sandbox not being up yet; check the sandbox state with 'Sandbox.get_info()'." ) err.__cause__ = e return err @@ -110,7 +106,7 @@ def handle_rpc_exception( :param e: The caught exception, expected to be a ``ConnectError``. :param error_map: Optional map of gRPC codes to exception factories that override the defaults. :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request. - :return: The corresponding exception. A connection dropped mid-request with the sandbox confirmed gone becomes a ``TimeoutException``; non-``ConnectError`` errors are otherwise returned as-is. + :return: The corresponding exception. A connection dropped mid-request with the sandbox confirmed gone becomes a ``SandboxNotRunningException``; non-``ConnectError`` errors are otherwise returned as-is. """ if isinstance(e, ConnectError): # connectrpc converts asyncio cancellation into a ConnectError with diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index 053553ff9e..61ded79eab 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -3,45 +3,28 @@ # Message of the 502 the proxy in front of envd answers with when the sandbox # is running but nothing listens on envd's port — e.g. its network is down. -def format_sandbox_timeout_exception(message: str): - return TimeoutException( - f"{message}: This error is likely due to sandbox timeout. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." - ) - - # Messages of the 502 the proxy in front of envd answers with when the sandbox does # not exist anymore (killed or reached its timeout) or when it is running but envd's # port is not open (e.g. its network is down). SANDBOX_NOT_FOUND_MESSAGE = "was not found" -SANDBOX_PORT_CLOSED_MESSAGE = "port is not open" def is_sandbox_not_found_message(message: str) -> bool: return SANDBOX_NOT_FOUND_MESSAGE in message -def is_sandbox_port_closed_message(message: str) -> bool: - return SANDBOX_PORT_CLOSED_MESSAGE in message - - def format_sandbox_unavailable_exception(message: str) -> Exception: - """Map a 502/UNAVAILABLE answered by the proxy in front of envd -- the sandbox was - not found (killed or reached its timeout), or it is running but envd's port is - not open (e.g. its network is down) -- to a ``SandboxUnreachableException``.""" - return SandboxUnreachableException( - f"{message}: The sandbox could not be reached -- it was killed, reached its timeout, or envd inside it is not reachable (e.g. its network is down). Check the sandbox state with 'Sandbox.get_info()'; you can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." - ) - - -def format_sandbox_start_unavailable_exception(message: str) -> Exception: - """Like :func:`format_sandbox_unavailable_exception`, for starting a command/PTY/ - watch: when the proxy reports the sandbox was not found, a - ``SandboxNotFoundException`` is returned.""" + """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox was + not found (killed or reached its timeout) is a ``SandboxNotRunningException``; + the sandbox running but envd's port not open (e.g. its network is down), or any + other message, a ``SandboxUnreachableException``.""" if is_sandbox_not_found_message(message): - return SandboxNotFoundException( + return SandboxNotRunningException( f"{message}: The sandbox was killed or reached its timeout. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." ) - return format_sandbox_unavailable_exception(message) + return SandboxUnreachableException( + f"{message}: The sandbox is running but envd inside it could not be reached (e.g. its network is down). Check the sandbox state with 'Sandbox.get_info()'." + ) def format_request_timeout_error() -> Exception: @@ -125,18 +108,31 @@ class SandboxNotFoundException(NotFoundException): pass +class SandboxNotRunningException(TimeoutException): + """ + Raised when the sandbox is not running anymore: the proxy in front of it + answered a request (or the health probe run after a request failed at the + connection level) with the sandbox not found -- it was killed or reached its + timeout. Retrying the request will not help; create or resume a sandbox. + + Subclass of ``TimeoutException``, which this case surfaced as before. + """ + + pass + + class SandboxUnreachableException(TimeoutException): """ - Raised when the sandbox could not be reached while not confirmed to be + Raised when the sandbox could not be reached while it is not confirmed to be stopped: either the proxy in front of the sandbox reports it running but envd's port not open (e.g. the sandbox's network is down), or a request failed at the connection level (the connection could not be established or - was dropped mid-request) and a follow-up health probe got no answer from the - sandbox either. + was dropped mid-request) and the follow-up health probe got no answer from the + sandbox either. A sandbox confirmed stopped is a ``SandboxNotRunningException``. This usually means the sandbox's network is down, envd inside it is not up - (yet), or a transient network issue between the client and the sandbox. - Check the sandbox state with ``Sandbox.get_info()`` and retry the request. + (yet), or a network issue between the client and the sandbox. Check the + sandbox state with ``Sandbox.get_info()``. Subclass of ``TimeoutException``, which these cases surfaced as before. """ diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command.py b/packages/python-sdk/e2b/sandbox_async/commands/command.py index 2d4aea112e..f2ad186066 100644 --- a/packages/python-sdk/e2b/sandbox_async/commands/command.py +++ b/packages/python-sdk/e2b/sandbox_async/commands/command.py @@ -14,7 +14,7 @@ from e2b.envd.process import process_connect, process_pb from e2b.envd.api import acheck_sandbox_health -from e2b.envd.rpc import START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health +from e2b.envd.rpc import ahandle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -390,6 +390,4 @@ async def _create_handle( await events.aclose() except Exception: pass - raise await ahandle_rpc_exception_with_health( - e, self._check_health, START_RPC_ERROR_MAP - ) + raise await ahandle_rpc_exception_with_health(e, self._check_health) diff --git a/packages/python-sdk/e2b/sandbox_async/commands/pty.py b/packages/python-sdk/e2b/sandbox_async/commands/pty.py index bb00aa0f05..dda9d2e6f3 100644 --- a/packages/python-sdk/e2b/sandbox_async/commands/pty.py +++ b/packages/python-sdk/e2b/sandbox_async/commands/pty.py @@ -15,7 +15,7 @@ KEEPALIVE_PING_INTERVAL_SEC, ) from e2b.envd.api import acheck_sandbox_health -from e2b.envd.rpc import START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health +from e2b.envd.rpc import ahandle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -178,9 +178,7 @@ async def create( await events.aclose() except Exception: pass - raise await ahandle_rpc_exception_with_health( - e, self._check_health, START_RPC_ERROR_MAP - ) + raise await ahandle_rpc_exception_with_health(e, self._check_health) async def connect( self, @@ -229,9 +227,7 @@ async def connect( await events.aclose() except Exception: pass - raise await ahandle_rpc_exception_with_health( - e, self._check_health, START_RPC_ERROR_MAP - ) + raise await ahandle_rpc_exception_with_health(e, self._check_health) async def resize( self, diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index ef55da1873..f4a8a4cfd7 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -24,7 +24,7 @@ from protobuf import Oneof from e2b.envd.filesystem import filesystem_connect, filesystem_pb -from e2b.envd.rpc import START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health +from e2b.envd.rpc import ahandle_rpc_exception_with_health from e2b.envd.utils import authentication_header, timeout_to_ms from e2b.envd.client_async import as_stream, create_rpc_client, first_event from e2b.envd.versions import ( @@ -62,11 +62,6 @@ Code.NOT_FOUND: FileNotFoundException, } -_WATCH_DIR_START_RPC_ERROR_MAP = { - **_FILESYSTEM_RPC_ERROR_MAP, - **START_RPC_ERROR_MAP, -} - _FILESYSTEM_HTTP_ERROR_MAP = { 404: FileNotFoundException, } @@ -763,6 +758,4 @@ async def watch_dir( await events.aclose() except Exception: pass - raise await _ahandle_filesystem_rpc_exception( - e, self._envd_api, _WATCH_DIR_START_RPC_ERROR_MAP - ) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command.py b/packages/python-sdk/e2b/sandbox_sync/commands/command.py index 321f9a43ab..d929301e4e 100644 --- a/packages/python-sdk/e2b/sandbox_sync/commands/command.py +++ b/packages/python-sdk/e2b/sandbox_sync/commands/command.py @@ -24,7 +24,7 @@ from e2b.envd.process import process_connect, process_pb from e2b.envd.api import check_sandbox_health -from e2b.envd.rpc import START_RPC_ERROR_MAP, handle_rpc_exception_with_health +from e2b.envd.rpc import handle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -385,6 +385,4 @@ def _create_handle( events.close() except Exception: pass - raise handle_rpc_exception_with_health( - e, self._check_health, START_RPC_ERROR_MAP - ) + raise handle_rpc_exception_with_health(e, self._check_health) diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py index c30edf1fb3..9465007759 100644 --- a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py +++ b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py @@ -15,7 +15,7 @@ KEEPALIVE_PING_INTERVAL_SEC, ) from e2b.envd.api import check_sandbox_health -from e2b.envd.rpc import START_RPC_ERROR_MAP, handle_rpc_exception_with_health +from e2b.envd.rpc import handle_rpc_exception_with_health from e2b.envd.utils import ( authentication_header, extract_start_pid, @@ -169,9 +169,7 @@ def create( events.close() except Exception: pass - raise handle_rpc_exception_with_health( - e, self._check_health, START_RPC_ERROR_MAP - ) + raise handle_rpc_exception_with_health(e, self._check_health) def connect( self, @@ -215,9 +213,7 @@ def connect( events.close() except Exception: pass - raise handle_rpc_exception_with_health( - e, self._check_health, START_RPC_ERROR_MAP - ) + raise handle_rpc_exception_with_health(e, self._check_health) def resize( self, diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 454551408c..4ba007e230 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -21,7 +21,7 @@ handle_envd_api_transport_exception_with_health, ) from e2b.envd.filesystem import filesystem_connect, filesystem_pb -from e2b.envd.rpc import START_RPC_ERROR_MAP, handle_rpc_exception_with_health +from e2b.envd.rpc import handle_rpc_exception_with_health from e2b.envd.utils import authentication_header, timeout_to_ms from e2b.envd.client_sync import create_rpc_client from e2b.envd.versions import ( @@ -58,11 +58,6 @@ Code.NOT_FOUND: FileNotFoundException, } -_WATCH_DIR_START_RPC_ERROR_MAP = { - **_FILESYSTEM_RPC_ERROR_MAP, - **START_RPC_ERROR_MAP, -} - _FILESYSTEM_HTTP_ERROR_MAP = { 404: FileNotFoundException, } @@ -694,9 +689,7 @@ def watch_dir( }, ) except Exception as e: - raise _handle_filesystem_rpc_exception( - e, self._envd_api, _WATCH_DIR_START_RPC_ERROR_MAP - ) + raise _handle_filesystem_rpc_exception(e, self._envd_api) return WatchHandle( self._rpc, diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.py index 19b4e28d05..59da690806 100644 --- a/packages/python-sdk/tests/test_envd_api_exception.py +++ b/packages/python-sdk/tests/test_envd_api_exception.py @@ -15,6 +15,7 @@ NotFoundException, RateLimitException, SandboxException, + SandboxNotRunningException, SandboxUnreachableException, TimeoutException, ) @@ -54,7 +55,7 @@ def _port_closed(request: httpx.Request): def test_maps_502_with_port_closed_to_unreachable(): err = format_envd_api_exception(502, PORT_CLOSED) assert isinstance(err, SandboxUnreachableException) - assert "The sandbox could not be reached" in str(err) + assert "could not be reached" in str(err) def test_maps_502_to_timeout(): @@ -91,9 +92,11 @@ def test_returns_original_when_not_transport_error(): assert err is original -def test_502_sandbox_not_found_returns_unreachable(): +def test_502_sandbox_not_found_returns_not_running(): err = format_envd_api_exception(502, "The sandbox was not found") - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) + assert isinstance(err, TimeoutException) + assert not isinstance(err, SandboxUnreachableException) assert "The sandbox was not found" in str(err) @@ -101,7 +104,7 @@ def test_health_result_confirms_sandbox_killed(): err = handle_envd_api_transport_exception( httpx.RemoteProtocolError("peer closed connection"), sandbox_running=False ) - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) assert "sandbox was killed or reached its end of life" in str(err) @@ -127,13 +130,47 @@ def _refuse(request: httpx.Request): raise httpx.ConnectError("connection refused", request=request) +def _not_found(request: httpx.Request): + return httpx.Response(502, json={"message": "The sandbox was not found"}) + + def test_health_check_reports_running(): with _health_client(lambda request: httpx.Response(204)) as client: assert check_sandbox_health(client) is True def test_health_check_reports_sandbox_gone(): + with _health_client(_not_found) as client: + assert check_sandbox_health(client) is False + + +def test_health_check_raises_unreachable_for_other_502(): with _health_client(lambda request: httpx.Response(502)) as client: + try: + check_sandbox_health(client) + except SandboxUnreachableException: + pass + else: + raise AssertionError("expected the probe to raise") + + +def test_health_check_tolerates_non_object_json_502_bodies(): + for body in (["Bad gateway"], 1, {"message": 5}): + with _health_client( + lambda request, body=body: httpx.Response(502, json=body) + ) as client: + try: + check_sandbox_health(client) + except SandboxUnreachableException: + pass + else: + raise AssertionError("expected the probe to raise") + + +def test_health_check_reports_sandbox_gone_for_plain_text_body(): + with _health_client( + lambda request: httpx.Response(502, text="The sandbox was not found") + ) as client: assert check_sandbox_health(client) is False @@ -171,12 +208,12 @@ def test_health_check_raises_when_sandbox_unreachable(): raise AssertionError("expected the probe to raise") -def test_connect_failure_with_sandbox_killed_returns_unreachable(): - with _health_client(lambda request: httpx.Response(502)) as client: +def test_connect_failure_with_sandbox_killed_returns_not_running(): + with _health_client(_not_found) as client: err = handle_envd_api_transport_exception_with_health( httpx.ConnectError("connection refused"), client ) - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) def test_connect_failure_with_running_sandbox_returns_raw_error(): @@ -192,6 +229,7 @@ def test_connect_failure_with_unreachable_health_returns_unreachable(): err = handle_envd_api_transport_exception_with_health(original, client) assert isinstance(err, SandboxUnreachableException) assert "could not be reached" in str(err) + assert "connection refused" in str(err) assert err.__cause__ is original @@ -230,12 +268,12 @@ async def test_async_connect_failure_with_unreachable_health_returns_unreachable assert err.__cause__ is original -async def test_async_connect_failure_with_sandbox_killed_returns_unreachable(): - async with _ahealth_client(lambda request: httpx.Response(502)) as client: +async def test_async_connect_failure_with_sandbox_killed_returns_not_running(): + async with _ahealth_client(_not_found) as client: err = await ahandle_envd_api_transport_exception_with_health( httpx.ConnectError("connection refused"), client ) - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) async def test_async_health_check_raises_unreachable_when_port_closed(): diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index 5a056507e9..f7b512d599 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -5,7 +5,6 @@ from pyqwest import StreamError, StreamErrorCode from e2b.envd.rpc import ( - START_RPC_ERROR_MAP, ahandle_rpc_exception_with_health, handle_rpc_exception, handle_rpc_exception_with_health, @@ -17,7 +16,7 @@ NotFoundException, RateLimitException, SandboxException, - SandboxNotFoundException, + SandboxNotRunningException, SandboxUnreachableException, TimeoutException, ) @@ -51,7 +50,7 @@ def test_maps_resource_exhausted_to_rate_limit(): def test_maps_unavailable_with_port_closed_to_unreachable(): err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, PORT_CLOSED)) assert isinstance(err, SandboxUnreachableException) - assert "The sandbox could not be reached" in str(err) + assert "could not be reached" in str(err) def test_maps_unavailable_to_timeout(): @@ -181,14 +180,22 @@ def test_returns_raw_transport_failure_without_health_result(): assert err is original -def test_unavailable_sandbox_not_found_returns_unreachable(): +def test_unavailable_sandbox_not_found_returns_not_running(): err = handle_rpc_exception( ConnectError(Code.UNAVAILABLE, "The sandbox was not found") ) - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) + assert isinstance(err, TimeoutException) + assert not isinstance(err, SandboxUnreachableException) assert "The sandbox was not found" in str(err) +def test_unavailable_port_closed_is_not_not_running(): + err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, PORT_CLOSED)) + assert isinstance(err, SandboxUnreachableException) + assert not isinstance(err, SandboxNotRunningException) + + def test_unavailable_sandbox_not_found_does_not_probe(): probed = [] @@ -200,7 +207,7 @@ def check(): ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), check ) assert not probed - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) async def test_async_ambiguous_unavailable_probes_health_and_returns_unreachable(): @@ -213,39 +220,23 @@ async def check(): assert isinstance(err, SandboxUnreachableException) -def test_start_error_map_sandbox_not_found_returns_not_found(): - err = handle_rpc_exception( - ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), - START_RPC_ERROR_MAP, - ) - assert isinstance(err, SandboxNotFoundException) - assert "The sandbox was not found" in str(err) - - -def test_start_error_map_port_closed_returns_unreachable(): - err = handle_rpc_exception( - ConnectError(Code.UNAVAILABLE, PORT_CLOSED), START_RPC_ERROR_MAP - ) - assert isinstance(err, SandboxUnreachableException) - assert not isinstance(err, SandboxNotFoundException) - - -async def test_async_start_error_map_sandbox_not_found_returns_not_found(): +async def test_async_unavailable_sandbox_not_found_returns_not_running(): async def check(): - return True + raise AssertionError("health check should not run") err = await ahandle_rpc_exception_with_health( - ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), - check, - START_RPC_ERROR_MAP, + ConnectError(Code.UNAVAILABLE, "The sandbox was not found"), check ) - assert isinstance(err, SandboxNotFoundException) + assert isinstance(err, SandboxNotRunningException) def test_health_check_confirms_sandbox_killed(): - err = handle_rpc_exception_with_health(_stream_reset(), lambda: False) - assert isinstance(err, SandboxUnreachableException) + original = _stream_reset() + err = handle_rpc_exception_with_health(original, lambda: False) + assert isinstance(err, SandboxNotRunningException) + assert not isinstance(err, SandboxUnreachableException) assert "sandbox was killed or reached its end of life" in str(err) + assert err.__cause__ is original def test_health_check_running_returns_raw_error(): @@ -280,6 +271,7 @@ def check(): err = handle_rpc_exception_with_health(original, check) assert isinstance(err, SandboxUnreachableException) assert "could not be reached" in str(err) + assert "health check failed" in str(err) assert err.__cause__ is original @@ -301,9 +293,9 @@ def check(): assert err is original -def test_connect_failure_with_sandbox_killed_returns_unreachable(): +def test_connect_failure_with_sandbox_killed_returns_not_running(): err = handle_rpc_exception_with_health(_connect_refused(), lambda: False) - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) def test_connect_failure_with_unreachable_health_returns_unreachable(): @@ -329,7 +321,7 @@ async def check(): return False err = await ahandle_rpc_exception_with_health(_stream_reset(), check) - assert isinstance(err, SandboxUnreachableException) + assert isinstance(err, SandboxNotRunningException) assert "sandbox was killed or reached its end of life" in str(err) From 19c97cc9d320b4d1a9510e369ead25eee4dc76aa Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:49:42 +0000 Subject: [PATCH 13/16] fix(sdk): keep SandboxNotFoundError on JS command/PTY/watch start, probe ambiguous Unavailable, probe without connection retries - JS: START_RPC_ERROR_MAP maps a 502 'sandbox was not found' back to SandboxNotFoundError on commands.run/connect, pty.create/connect and files.watchDir; every other call keeps SandboxNotRunningError - an Unavailable whose message is neither 'was not found' nor 'port is not open' (envd ending a stream when the sandbox is killed mid-command) is now health-probed: gone -> SandboxNotRunning*, otherwise SandboxUnreachable* with the sandbox state reported as unknown instead of 'running' - Python: the health probe uses an envd client without ConnectionRetryTransport (get_envd_api(..., retry_connect=False) over the same pools), so a sandbox that cannot be reached is not retried again by the probe - drop the unused errorMap/error_map parameters of the filesystem helpers, fix the health-aware RPC handler docstring, stale exceptions comment, class docstrings, explicit ResponseInit in rejectProxyUnavailableResponse, changeset with the final per-path mapping Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 17 ++- packages/js-sdk/src/envd/rpc.ts | 54 +++++++- packages/js-sdk/src/errors.ts | 44 +++++-- packages/js-sdk/src/sandbox/commands/index.ts | 7 +- packages/js-sdk/src/sandbox/commands/pty.ts | 13 +- .../js-sdk/src/sandbox/filesystem/index.ts | 23 +++- .../js-sdk/tests/envd/handleRpcError.test.ts | 121 +++++++++++++++--- .../e2b/api/client_async/__init__.py | 48 +++++-- .../e2b/api/client_sync/__init__.py | 48 +++++-- packages/python-sdk/e2b/envd/api.py | 2 + packages/python-sdk/e2b/envd/rpc.py | 46 +++++-- packages/python-sdk/e2b/exceptions.py | 44 +++++-- .../e2b/sandbox_async/commands/command.py | 10 +- .../e2b/sandbox_async/commands/pty.py | 10 +- .../sandbox_async/filesystem/filesystem.py | 35 ++--- .../e2b/sandbox_sync/commands/command.py | 10 +- .../e2b/sandbox_sync/commands/pty.py | 10 +- .../e2b/sandbox_sync/filesystem/filesystem.py | 41 +++--- .../tests/test_envd_client_transport.py | 37 ++++++ .../tests/test_envd_rpc_exception.py | 65 +++++++++- packages/python-sdk/tests/transport_caches.py | 1 + 21 files changed, 560 insertions(+), 126 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index f18d425ba5..9b49941c6d 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -5,9 +5,18 @@ Classify requests that fail because the sandbox cannot be reached. Two new error classes, both subclasses of the sandbox-timeout `TimeoutError` (JS) / `TimeoutException` (Python) these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working: -- `SandboxNotRunningError` / `SandboxNotRunningException` — the proxy in front of the sandbox answered `502 "The sandbox was not found"` (killed or reached its timeout), from any envd request (RPC or files API) or from the health probe run after a request failed at the connection level (e.g. the sandbox killed while a command was running). -- `SandboxUnreachableError` / `SandboxUnreachableException` — the sandbox is not confirmed stopped but cannot be reached: the proxy reports it running but envd's port not open (e.g. `ip link set eth0 down` inside the sandbox), any other `502`, or a request failed at the connection level and the follow-up health probe got no answer either. The failed request is the error's `cause` / `__cause__`. +- `SandboxNotRunningError` / `SandboxNotRunningException` — the proxy in front of the sandbox answered `502 "The sandbox was not found"` (killed, paused or reached its timeout), or the health probe run after a failed request did. Retrying will not help; create or resume a sandbox. +- `SandboxUnreachableError` / `SandboxUnreachableException` — the sandbox is not confirmed stopped but cannot be reached. The failed request is the error's `cause` / `__cause__`. -Requests whose connection could not be established (`ECONNREFUSED`, DNS, unreachable host, browsers' `Failed to fetch`, `httpx.ConnectError`, …) now run the same health probe as requests whose connection dropped mid-request; when the probe says the sandbox is running (or is inconclusive) the original transport error is raised unchanged. `isRunning()` / `is_running()` still return `false` for any `502`. +What each envd request (RPC or files API) now raises: -JS: starting a command, PTY or directory watch on a sandbox that is gone raises `SandboxNotRunningError` instead of `SandboxNotFoundError: Sandbox is probably not running anymore`, like every other request (Python: `SandboxNotRunningException` instead of `TimeoutException`). The CLI's `sandbox exec` / `sandbox connect` print that error instead of a stack trace. Plain-text `502` bodies keep the proxy's message in JS, matching Python. +| Failure | JS | Python | +| -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| `502` / `Unavailable` "The sandbox was not found" | `SandboxNotRunningError` | `SandboxNotRunningException` | +| … when starting a command, PTY or directory watch (`commands.run` / `connect`, `pty.create` / `connect`, `files.watchDir`) | `SandboxNotFoundError` (unchanged) | `SandboxNotRunningException` (was `TimeoutException`) | +| `502` / `Unavailable` "The sandbox is running but port is not open" (e.g. `ip link set eth0 down` inside the sandbox) | `SandboxUnreachableError` | `SandboxUnreachableException` | +| any other `Unavailable` (e.g. envd ending a stream because the sandbox was killed mid-command) | health probe: gone → `SandboxNotRunningError`, otherwise `SandboxUnreachableError` ("state is unknown") | same, with the Python classes | +| any other `502` | `SandboxUnreachableError` | `SandboxUnreachableException` | +| connection could not be established or dropped mid-request (`ECONNREFUSED`, DNS, unreachable host, browsers' `Failed to fetch`, `httpx.ConnectError`, …) | health probe: gone → `SandboxNotRunningError`; running or inconclusive → the original transport error; no answer → `SandboxUnreachableError` | same, with the Python classes | + +The health probe runs without the SDK's connection retries, so a sandbox that cannot be reached is not tried all over again. `isRunning()` / `is_running()` still return `false` for any `502`. Plain-text `502` bodies keep the proxy's message in JS, matching Python. diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 0c638bb4a4..8f40600d9a 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -8,9 +8,12 @@ import { AuthenticationError, formatSandboxUnavailableError, InvalidArgumentError, + isSandboxNotFoundMessage, + isSandboxPortNotOpenMessage, NotFoundError, RateLimitError, SandboxError, + SandboxNotFoundError, SandboxNotRunningError, SandboxUnreachableError, TimeoutError, @@ -92,6 +95,22 @@ export function isTransportFailure(err: unknown): boolean { ) } +/** + * Checks whether the error is an `Unavailable` whose message the proxy's answers do + * not explain — neither the sandbox not found nor its port not open — e.g. envd + * ending a stream with "the connection to sandbox ... ended before the stream + * completed" when the sandbox is killed mid-command. Whether the sandbox is gone + * can only be told by probing it. + */ +export function isAmbiguousUnavailable(err: unknown): boolean { + return ( + err instanceof ConnectError && + err.code === Code.Unavailable && + !isSandboxNotFoundMessage(err.rawMessage) && + !isSandboxPortNotOpenMessage(err.rawMessage) + ) +} + /** * Builds the error for a request that failed at the connection level when the * follow-up health probe failed too. A probe answered by the proxy with the sandbox @@ -143,6 +162,22 @@ export async function resolveTransportFailure( return undefined } +/** + * Error map for starting a command, PTY or directory watch: an `Unavailable` from + * the proxy with the sandbox not found is a `SandboxNotFoundError` (as it always + * was for these calls); any other `Unavailable` maps as for every other request. + */ +export const START_RPC_ERROR_MAP: Partial< + Record Error> +> = { + [Code.Unavailable]: (message) => + isSandboxNotFoundMessage(message) + ? new SandboxNotFoundError( + `${message}: Sandbox is probably not running anymore` + ) + : formatSandboxUnavailableError(message), +} + const DEFAULT_ERROR_MAP: Partial Error>> = { [Code.InvalidArgument]: (message) => new InvalidArgumentError(message), [Code.Unauthenticated]: (message) => new AuthenticationError(message), @@ -194,12 +229,15 @@ export function handleRpcError( /** * Like {@link handleRpcError}, but when the request failed at the connection level * (the connection to the sandbox could not be established or was dropped - * mid-request) it probes the sandbox health to tell apart the sandbox being killed - * from the sandbox being unreachable or a transient network failure (e.g. a load - * balancer dropping the connection). When the probe confirms the sandbox is gone, a + * mid-request), or envd/the proxy answered an `Unavailable` that does not say + * whether the sandbox is gone ({@link isAmbiguousUnavailable}), it probes the + * sandbox health to tell apart the sandbox being killed from the sandbox being + * unreachable or a transient network failure (e.g. a load balancer dropping the + * connection). When the probe confirms the sandbox is gone, a * `SandboxNotRunningError` is returned — the same error a request answered by the * proxy with the sandbox not found gets; when the probe gets no answer either, a - * `SandboxUnreachableError`. + * `SandboxUnreachableError`; otherwise the error maps as usual (an ambiguous + * `Unavailable` to a `SandboxUnreachableError` with the sandbox state unknown). * * @param err - The caught error, expected to be a `ConnectError` from the gRPC transport. * @param checkHealth - Probe resolving to whether the sandbox is running (`undefined` when unknown) and rejecting when it cannot be reached. @@ -211,7 +249,7 @@ export async function handleRpcErrorWithHealthCheck( checkHealth?: SandboxHealthCheck, errorMap?: Partial Error>> ): Promise { - if (checkHealth && isTransportFailure(err)) { + if (checkHealth && (isTransportFailure(err) || isAmbiguousUnavailable(err))) { const resolved = await resolveTransportFailure( err as ConnectError, checkHealth @@ -286,7 +324,11 @@ export async function rejectProxyUnavailableResponse( ) { // Connect-encoded error (declared string `code`, e.g. "permission_denied") — // let the transport decode it with that code. - return new Response(message, res) + return new Response(message, { + status: res.status, + statusText: res.statusText, + headers: res.headers, + }) } if (typeof body?.message === 'string') { message = body.message diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index 2845891cd4..5218e5c6ac 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -2,16 +2,23 @@ // not exist anymore (killed or reached its timeout) or when it is running but envd's // port is not open (e.g. its network is down). const SANDBOX_NOT_FOUND_MESSAGE = 'was not found' +const SANDBOX_PORT_NOT_OPEN_MESSAGE = 'port is not open' export function isSandboxNotFoundMessage(message: string): boolean { return message.includes(SANDBOX_NOT_FOUND_MESSAGE) } +export function isSandboxPortNotOpenMessage(message: string): boolean { + return message.includes(SANDBOX_PORT_NOT_OPEN_MESSAGE) +} + /** * Maps a 502/Unavailable answered by the proxy in front of envd: the sandbox was not * found (killed or reached its timeout) is a `SandboxNotRunningError`; the sandbox - * running but envd's port not open (e.g. its network is down), or any other message, - * a `SandboxUnreachableError`. + * running but envd's port not open (e.g. its network is down) a + * `SandboxUnreachableError`; any other message (e.g. envd ending a stream because + * the connection to the sandbox was lost) a `SandboxUnreachableError` too, with the + * sandbox state reported as unknown. */ export function formatSandboxUnavailableError(message: string): Error { if (isSandboxNotFoundMessage(message)) { @@ -19,8 +26,13 @@ export function formatSandboxUnavailableError(message: string): Error { `${message}: The sandbox was killed or reached its timeout. You can modify the sandbox timeout by passing 'timeoutMs' when starting the sandbox or calling '.setTimeout' on the sandbox with the desired timeout.` ) } + if (isSandboxPortNotOpenMessage(message)) { + return new SandboxUnreachableError( + `${message}: The sandbox is running but envd inside it could not be reached (e.g. its network is down). Check the sandbox state with 'Sandbox.getInfo()'.` + ) + } return new SandboxUnreachableError( - `${message}: The sandbox is running but envd inside it could not be reached (e.g. its network is down). Check the sandbox state with 'Sandbox.getInfo()'.` + `${message}: The sandbox could not be reached and its state is unknown — it may have been killed or reached its timeout, or the connection to it was lost. Check the sandbox state with 'Sandbox.getInfo()'.` ) } @@ -117,8 +129,16 @@ export class SandboxNotFoundError extends NotFoundError { /** * Thrown when the sandbox is not running anymore: the proxy in front of it * answered a request (or the health probe run after a request failed at the - * connection level) with the sandbox not found — it was killed or reached its - * timeout. Retrying the request will not help; create or resume a sandbox. + * connection level) with the sandbox not found — it was killed, paused or + * reached its timeout. Retrying the request will not help; create or resume a + * sandbox. + * + * The split from `SandboxUnreachableError` is only as good as the proxy's answer + * and the health probe: a sandbox that is gone but could not be asked about is a + * `SandboxUnreachableError`. Starting a command, PTY or directory watch on a + * sandbox that is gone throws `SandboxNotFoundError` instead (as it always did), so + * handlers that want every "sandbox is gone" case have to catch both this class + * and `SandboxNotFoundError` until the next major version. * * Subclass of `TimeoutError`, which this case surfaced as before. */ @@ -134,11 +154,15 @@ export class SandboxNotRunningError extends TimeoutError { /** * Thrown when the sandbox could not be reached while it is not confirmed to be - * stopped: either the proxy in front of the sandbox reports it running but - * envd's port not open (e.g. the sandbox's network is down), or a request - * failed at the connection level (the connection could not be established or - * was dropped mid-request) and the follow-up health probe got no answer from the - * sandbox either. A sandbox confirmed stopped is a `SandboxNotRunningError`. + * stopped: the proxy in front of the sandbox reports it running but envd's port + * not open (e.g. the sandbox's network is down), the proxy or envd answered + * with an unavailability the health probe could not resolve (e.g. the connection + * to the sandbox was lost mid-stream while the probe found it running), or a + * request failed at the connection level (the connection could not be + * established or was dropped mid-request) and the follow-up health probe got + * no answer from the sandbox either. A sandbox confirmed stopped (or paused) is + * a `SandboxNotRunningError` — but only as far as the probe can tell: a sandbox + * that is gone and could not be asked about surfaces here. * * This usually means the sandbox's network is down, envd inside it is not up * (yet), or a network issue between the client and the sandbox. Check the diff --git a/packages/js-sdk/src/sandbox/commands/index.ts b/packages/js-sdk/src/sandbox/commands/index.ts index a9905d084d..a9569d0b17 100644 --- a/packages/js-sdk/src/sandbox/commands/index.ts +++ b/packages/js-sdk/src/sandbox/commands/index.ts @@ -29,6 +29,7 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, + START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE } from '../../envd/versions' @@ -475,7 +476,11 @@ export class Commands { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) + throw await handleRpcErrorWithHealthCheck( + err, + this.checkHealth, + START_RPC_ERROR_MAP + ) } } } diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.ts index 29928a92f3..c65bca09d1 100644 --- a/packages/js-sdk/src/sandbox/commands/pty.ts +++ b/packages/js-sdk/src/sandbox/commands/pty.ts @@ -22,6 +22,7 @@ import { CommandHandle } from './commandHandle' import { authenticationHeader, handleRpcErrorWithHealthCheck, + START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { @@ -161,7 +162,11 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) + throw await handleRpcErrorWithHealthCheck( + err, + this.checkHealth, + START_RPC_ERROR_MAP + ) } } @@ -218,7 +223,11 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) + throw await handleRpcErrorWithHealthCheck( + err, + this.checkHealth, + START_RPC_ERROR_MAP + ) } } diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 712cea23fd..0a48ee7875 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -25,6 +25,7 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, + START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' @@ -65,12 +66,22 @@ const FILESYSTEM_RPC_ERROR_MAP: Partial< [Code.NotFound]: (message: string) => new FileNotFoundError(message), } +const WATCH_DIR_START_RPC_ERROR_MAP: Partial< + Record Error> +> = { + ...FILESYSTEM_RPC_ERROR_MAP, + ...START_RPC_ERROR_MAP, +} + async function handleFilesystemRpcError( err: unknown, - checkHealth?: SandboxHealthCheck, - errorMap = FILESYSTEM_RPC_ERROR_MAP + checkHealth?: SandboxHealthCheck ): Promise { - return handleRpcErrorWithHealthCheck(err, checkHealth, errorMap) + return handleRpcErrorWithHealthCheck( + err, + checkHealth, + FILESYSTEM_RPC_ERROR_MAP + ) } function handleFilesystemEnvdApiError(res: { @@ -1108,7 +1119,11 @@ export class Filesystem { ) } catch (err) { cleanup() - throw await handleFilesystemRpcError(err, this.checkHealth) + throw await handleRpcErrorWithHealthCheck( + err, + this.checkHealth, + WATCH_DIR_START_RPC_ERROR_MAP + ) } } } diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 28f24f653f..4d7dd5a595 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -3,8 +3,10 @@ import { Code, ConnectError } from '@connectrpc/connect' import { handleRpcError, handleRpcErrorWithHealthCheck, + isAmbiguousUnavailable, isTransportFailure, rejectProxyUnavailableResponse, + START_RPC_ERROR_MAP, } from '../../src/envd/rpc' import { handleProcessStartEvent, @@ -16,6 +18,7 @@ import { NotFoundError, RateLimitError, SandboxError, + SandboxNotFoundError, SandboxNotRunningError, SandboxUnreachableError, TimeoutError, @@ -39,6 +42,107 @@ function connectRefused(): ConnectError { // Body of the proxy's 502 when the sandbox is running but nothing listens on envd's port const PORT_CLOSED = 'The sandbox is running but port is not open' +// envd ends a stream with this Unavailable when the sandbox is killed mid-command +const STREAM_ENDED = + 'the connection to sandbox ended before the stream completed' + +function neverProbe(): Promise { + throw new Error('health check should not run') +} + +describe('START_RPC_ERROR_MAP', () => { + test('returns SandboxNotFoundError for Unavailable with the sandbox not found', () => { + const err = handleRpcError( + new ConnectError('The sandbox was not found', Code.Unavailable), + START_RPC_ERROR_MAP + ) + assert.instanceOf(err, SandboxNotFoundError) + assert.include(err.message, 'The sandbox was not found') + }) + + test('maps any other Unavailable like every other request', () => { + const portClosed = handleRpcError( + new ConnectError(PORT_CLOSED, Code.Unavailable), + START_RPC_ERROR_MAP + ) + assert.instanceOf(portClosed, SandboxUnreachableError) + const ambiguous = handleRpcError( + new ConnectError(STREAM_ENDED, Code.Unavailable), + START_RPC_ERROR_MAP + ) + assert.instanceOf(ambiguous, SandboxUnreachableError) + }) + + test('still returns SandboxNotRunningError without the start map', () => { + const err = handleRpcError( + new ConnectError('The sandbox was not found', Code.Unavailable) + ) + assert.instanceOf(err, SandboxNotRunningError) + assert.notInstanceOf(err, SandboxNotFoundError) + }) +}) + +describe('ambiguous Unavailable', () => { + test('is an Unavailable that says neither not found nor port not open', () => { + assert.isTrue( + isAmbiguousUnavailable(new ConnectError(STREAM_ENDED, Code.Unavailable)) + ) + assert.isFalse( + isAmbiguousUnavailable( + new ConnectError('The sandbox was not found', Code.Unavailable) + ) + ) + assert.isFalse( + isAmbiguousUnavailable(new ConnectError(PORT_CLOSED, Code.Unavailable)) + ) + assert.isFalse( + isAmbiguousUnavailable(new ConnectError(STREAM_ENDED, Code.Unknown)) + ) + }) + + test('is probed and returns SandboxNotRunningError when the sandbox is gone', async () => { + const original = new ConnectError(STREAM_ENDED, Code.Unavailable) + const err = await handleRpcErrorWithHealthCheck(original, async () => false) + assert.instanceOf(err, SandboxNotRunningError) + assert.notInstanceOf(err, SandboxUnreachableError) + assert.strictEqual(err.cause, original) + }) + + test('returns SandboxUnreachableError with the state unknown when the sandbox is running', async () => { + const err = await handleRpcErrorWithHealthCheck( + new ConnectError(STREAM_ENDED, Code.Unavailable), + async () => true + ) + assert.instanceOf(err, SandboxUnreachableError) + assert.notInstanceOf(err, SandboxNotRunningError) + assert.include(err.message, 'state is unknown') + assert.notInclude(err.message, 'is running but') + }) + + test('returns SandboxUnreachableError when the probe itself fails', async () => { + const original = new ConnectError(STREAM_ENDED, Code.Unavailable) + const err = await handleRpcErrorWithHealthCheck(original, async () => { + throw new Error('health check failed') + }) + assert.instanceOf(err, SandboxUnreachableError) + assert.include(err.message, 'health check failed') + assert.strictEqual(err.cause, original) + }) + + test('is not probed for the sandbox not found or its port not open', async () => { + const notFound = await handleRpcErrorWithHealthCheck( + new ConnectError('The sandbox was not found', Code.Unavailable), + neverProbe + ) + assert.instanceOf(notFound, SandboxNotRunningError) + const portClosed = await handleRpcErrorWithHealthCheck( + new ConnectError(PORT_CLOSED, Code.Unavailable), + neverProbe + ) + assert.instanceOf(portClosed, SandboxUnreachableError) + assert.include(portClosed.message, 'is running but') + }) +}) describe('handleRpcError', () => { test('returns InvalidArgumentError for InvalidArgument', () => { @@ -122,23 +226,6 @@ describe('handleRpcErrorWithHealthCheck', () => { Browser: 'network error', } - test('returns SandboxUnreachableError for an Unavailable stream termination without probing health', async () => { - let probed = false - const err = await handleRpcErrorWithHealthCheck( - new ConnectError( - 'the connection to sandbox ended before the stream completed', - Code.Unavailable - ), - async () => { - probed = true - return false - } - ) - assert.isFalse(probed) - assert.instanceOf(err, SandboxUnreachableError) - assert.include(err.message, 'ended before the stream completed') - }) - test('does not probe health on an Unavailable for the sandbox not found', async () => { let probed = false const err = await handleRpcErrorWithHealthCheck( diff --git a/packages/python-sdk/e2b/api/client_async/__init__.py b/packages/python-sdk/e2b/api/client_async/__init__.py index af37aa5b81..e8449ec95b 100644 --- a/packages/python-sdk/e2b/api/client_async/__init__.py +++ b/packages/python-sdk/e2b/api/client_async/__init__.py @@ -153,8 +153,11 @@ async def execute(self, request: Request) -> Response: # they replaced, the transports are not bound to an event loop and the caches # are process-global rather than per-loop. _transports: Dict[_TransportKey, ConnectionRetryTransport] = {} +# The balancing transport under each retrying one, for callers that must not +# retry connecting (the health probe run after a request failed). +_pools: Dict[_TransportKey, BalancingTransport] = {} # The httpx adapter over each transport, shared by every httpx client on it. -_httpx_transports: Dict[_TransportKey, AsyncPyqwestTransport] = {} +_httpx_transports: Dict[Tuple[_TransportKey, bool], AsyncPyqwestTransport] = {} def get_pyqwest_transport( @@ -162,7 +165,8 @@ def get_pyqwest_transport( read_timeout: Optional[float] = None, *, http_version: HttpVersion = DEFAULT_HTTP_VERSION, -) -> ConnectionRetryTransport: + retry_connect: bool = True, +) -> Transport: """The shared pyqwest transport with the SDK's tuning — system CA certs (without which TLS through an intercepting proxy fails) and the httpx-equivalent pool limits — behind connect-only retries. It balances @@ -187,7 +191,10 @@ def get_pyqwest_transport( Requests are logged by pyqwest itself on the ``pyqwest.access`` and ``pyqwest`` loggers at ``DEBUG`` (off unless enabled) — the transport-level diagnostics httpcore used to provide. The SDK's own ``logger`` option is - separate and sits above this, on the httpx client.""" + separate and sits above this, on the httpx client. + + ``retry_connect=False`` returns the balancing transport under the retrying + one — the same pools, without the connection retries.""" key = (proxy, read_timeout, http_version) def build() -> HTTPTransport: @@ -209,11 +216,11 @@ def build() -> HTTPTransport: with _transport_lock: transport = _transports.get(key) if transport is None: - transport = ConnectionRetryTransport( - BalancingTransport(build), max_retries=connection_retries - ) + pool = BalancingTransport(build) + transport = ConnectionRetryTransport(pool, max_retries=connection_retries) + _pools[key] = pool _transports[key] = transport - return transport + return transport if retry_connect else _pools[key] def get_httpx_transport( @@ -221,15 +228,19 @@ def get_httpx_transport( read_timeout: Optional[float] = None, *, http_version: HttpVersion = DEFAULT_HTTP_VERSION, + retry_connect: bool = True, ) -> AsyncPyqwestTransport: """The httpx adapter over the shared transport of :func:`get_pyqwest_transport`, for the generated httpx clients (control plane, envd HTTP API, volume content). The adapter holds no state of its own and does not close the pools, so closing an httpx client leaves them - intact for the other clients on it.""" - key = (proxy, read_timeout, http_version) + intact for the other clients on it. ``retry_connect=False`` adapts the + same pools without the connection retries.""" + key = ((proxy, read_timeout, http_version), retry_connect) # Resolve the pool before taking the lock: it takes the same one. - pool = get_pyqwest_transport(proxy, read_timeout, http_version=http_version) + pool = get_pyqwest_transport( + proxy, read_timeout, http_version=http_version, retry_connect=retry_connect + ) with _transport_lock: transport = _httpx_transports.get(key) if transport is None: @@ -243,6 +254,7 @@ def get_transport( *, http_version: Optional[HttpVersion] = None, for_streaming: bool = False, + retry_connect: bool = True, ) -> AsyncPyqwestTransport: """The shared httpx transport factory for the control-plane REST API and envd HTTP API (file transfers, health checks). For TLS connections ALPN @@ -264,16 +276,26 @@ def get_transport( whole-request deadlines rather than idle bounds — so only streamed downloads take it, and they get their own pool (see :func:`get_pyqwest_transport`). + + ``retry_connect=False`` returns the adapter over the same pool without the + connection retries (``E2B_CONNECTION_RETRIES``): for the health probe run + after a request failed, so a sandbox that cannot be reached is not tried + all over again. """ return get_httpx_transport( proxy_to_config(config.proxy), READ_TIMEOUT if for_streaming else None, http_version=(config.http_version if http_version is None else http_version), + retry_connect=retry_connect, ) def get_envd_api( - config: ConnectionConfig, base_url: str, *, for_streaming: bool = False + config: ConnectionConfig, + base_url: str, + *, + for_streaming: bool = False, + retry_connect: bool = True, ) -> httpx.AsyncClient: """An httpx client for a sandbox's envd HTTP API (file transfers, health checks) on the shared transports. The client itself is a cheap stateless @@ -281,7 +303,9 @@ def get_envd_api( is shared and loop-independent.""" return httpx.AsyncClient( base_url=base_url, - transport=get_transport(config, for_streaming=for_streaming), + transport=get_transport( + config, for_streaming=for_streaming, retry_connect=retry_connect + ), headers=config.sandbox_headers, event_hooks=make_async_logging_event_hooks(config.logger), ) diff --git a/packages/python-sdk/e2b/api/client_sync/__init__.py b/packages/python-sdk/e2b/api/client_sync/__init__.py index 720cfdbbda..473625e110 100644 --- a/packages/python-sdk/e2b/api/client_sync/__init__.py +++ b/packages/python-sdk/e2b/api/client_sync/__init__.py @@ -158,8 +158,11 @@ def execute_sync(self, request: SyncRequest) -> SyncResponse: # pyqwest transports are thread-safe, so unlike the httpx transports they # replaced, the caches are process-global rather than per-thread. _transports: Dict[_TransportKey, ConnectionRetryTransport] = {} +# The balancing transport under each retrying one, for callers that must not +# retry connecting (the health probe run after a request failed). +_pools: Dict[_TransportKey, BalancingTransport] = {} # The httpx adapter over each transport, shared by every httpx client on it. -_httpx_transports: Dict[_TransportKey, PyqwestTransport] = {} +_httpx_transports: Dict[Tuple[_TransportKey, bool], PyqwestTransport] = {} def get_pyqwest_transport( @@ -167,7 +170,8 @@ def get_pyqwest_transport( read_timeout: Optional[float] = None, *, http_version: HttpVersion = DEFAULT_HTTP_VERSION, -) -> ConnectionRetryTransport: + retry_connect: bool = True, +) -> SyncTransport: """The shared pyqwest transport with the SDK's tuning — system CA certs (without which TLS through an intercepting proxy fails) and the httpx-equivalent pool limits — behind connect-only retries. It balances @@ -192,7 +196,10 @@ def get_pyqwest_transport( Requests are logged by pyqwest itself on the ``pyqwest.access`` and ``pyqwest`` loggers at ``DEBUG`` (off unless enabled) — the transport-level diagnostics httpcore used to provide. The SDK's own ``logger`` option is - separate and sits above this, on the httpx client.""" + separate and sits above this, on the httpx client. + + ``retry_connect=False`` returns the balancing transport under the retrying + one — the same pools, without the connection retries.""" key = (proxy, read_timeout, http_version) def build() -> SyncHTTPTransport: @@ -214,11 +221,11 @@ def build() -> SyncHTTPTransport: with _transport_lock: transport = _transports.get(key) if transport is None: - transport = ConnectionRetryTransport( - BalancingTransport(build), max_retries=connection_retries - ) + pool = BalancingTransport(build) + transport = ConnectionRetryTransport(pool, max_retries=connection_retries) + _pools[key] = pool _transports[key] = transport - return transport + return transport if retry_connect else _pools[key] def get_httpx_transport( @@ -226,15 +233,19 @@ def get_httpx_transport( read_timeout: Optional[float] = None, *, http_version: HttpVersion = DEFAULT_HTTP_VERSION, + retry_connect: bool = True, ) -> PyqwestTransport: """The httpx adapter over the shared transport of :func:`get_pyqwest_transport`, for the generated httpx clients (control plane, envd HTTP API, volume content). The adapter holds no state of its own and does not close the pools, so closing an httpx client leaves them - intact for the other clients on it.""" - key = (proxy, read_timeout, http_version) + intact for the other clients on it. ``retry_connect=False`` adapts the + same pools without the connection retries.""" + key = ((proxy, read_timeout, http_version), retry_connect) # Resolve the pool before taking the lock: it takes the same one. - pool = get_pyqwest_transport(proxy, read_timeout, http_version=http_version) + pool = get_pyqwest_transport( + proxy, read_timeout, http_version=http_version, retry_connect=retry_connect + ) with _transport_lock: transport = _httpx_transports.get(key) if transport is None: @@ -248,6 +259,7 @@ def get_transport( *, http_version: Optional[HttpVersion] = None, for_streaming: bool = False, + retry_connect: bool = True, ) -> PyqwestTransport: """The shared httpx transport factory for the control-plane REST API and envd HTTP API (file transfers, health checks). For TLS connections ALPN @@ -269,16 +281,26 @@ def get_transport( whole-request deadlines rather than idle bounds — so only streamed downloads take it, and they get their own pool (see :func:`get_pyqwest_transport`). + + ``retry_connect=False`` returns the adapter over the same pool without the + connection retries (``E2B_CONNECTION_RETRIES``): for the health probe run + after a request failed, so a sandbox that cannot be reached is not tried + all over again. """ return get_httpx_transport( proxy_to_config(config.proxy), READ_TIMEOUT if for_streaming else None, http_version=(config.http_version if http_version is None else http_version), + retry_connect=retry_connect, ) def get_envd_api( - config: ConnectionConfig, base_url: str, *, for_streaming: bool = False + config: ConnectionConfig, + base_url: str, + *, + for_streaming: bool = False, + retry_connect: bool = True, ) -> httpx.Client: """An httpx client for a sandbox's envd HTTP API (file transfers, health checks) on the shared transports. The client itself is a cheap stateless @@ -286,7 +308,9 @@ def get_envd_api( is shared and thread-safe.""" return httpx.Client( base_url=base_url, - transport=get_transport(config, for_streaming=for_streaming), + transport=get_transport( + config, for_streaming=for_streaming, retry_connect=retry_connect + ), headers=config.sandbox_headers, event_hooks=make_logging_event_hooks(config.logger), ) diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.py index 4ffc69ebb6..21a0ecb7ec 100644 --- a/packages/python-sdk/e2b/envd/api.py +++ b/packages/python-sdk/e2b/envd/api.py @@ -53,6 +53,8 @@ def check_sandbox_health(envd_api: httpx.Client) -> Optional[bool]: """Probe the sandbox's envd health endpoint. + :param envd_api: The sandbox's envd HTTP client -- the one built with ``retry_connect=False`` when probing after a failed request, so a sandbox that cannot be reached is not tried all over again. + :return: ``True`` if the sandbox is running, ``False`` if it is not, ``None`` if it answered but its state could not be determined. :raises httpx.TransportError: When the sandbox could not be reached (no answer within the probe timeout). :raises SandboxUnreachableException: When the proxy reports the sandbox running but envd's port not open -- the exception a request would get. diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py index 0f8428c015..bb9d7dfa5c 100644 --- a/packages/python-sdk/e2b/envd/rpc.py +++ b/packages/python-sdk/e2b/envd/rpc.py @@ -13,6 +13,8 @@ SandboxUnreachableException, TimeoutException, format_sandbox_unavailable_exception, + is_sandbox_not_found_message, + is_sandbox_port_not_open_message, AuthenticationException, RateLimitException, ) @@ -61,6 +63,20 @@ def is_transport_failure(e: Exception) -> bool: ) +def is_ambiguous_unavailable(e: Exception) -> bool: + """Whether the error is an ``UNAVAILABLE`` whose message the proxy's answers do + not explain -- neither the sandbox not found nor its port not open -- e.g. envd + ending a stream with "the connection to sandbox ... ended before the stream + completed" when the sandbox is killed mid-command. Whether the sandbox is gone + can only be told by probing it.""" + return ( + isinstance(e, ConnectError) + and e.code is Code.UNAVAILABLE + and not is_sandbox_not_found_message(e.message) + and not is_sandbox_port_not_open_message(e.message) + ) + + def format_terminated_exception( e: Exception, sandbox_running: Optional[bool], @@ -105,8 +121,8 @@ def handle_rpc_exception( :param e: The caught exception, expected to be a ``ConnectError``. :param error_map: Optional map of gRPC codes to exception factories that override the defaults. - :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request. - :return: The corresponding exception. A connection dropped mid-request with the sandbox confirmed gone becomes a ``SandboxNotRunningException``; non-``ConnectError`` errors are otherwise returned as-is. + :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request or an ambiguous ``UNAVAILABLE``. + :return: The corresponding exception. A connection dropped mid-request, or an ambiguous ``UNAVAILABLE``, with the sandbox confirmed gone becomes a ``SandboxNotRunningException``; non-``ConnectError`` errors are otherwise returned as-is. """ if isinstance(e, ConnectError): # connectrpc converts asyncio cancellation into a ConnectError with @@ -120,7 +136,9 @@ def handle_rpc_exception( # connection to the sandbox was dropped mid-request — either the # sandbox died or the network failed — so the code mapping below, # which describes envd responses, doesn't apply. - if is_transport_failure(e): + if is_transport_failure(e) or ( + sandbox_running is False and is_ambiguous_unavailable(e) + ): return format_terminated_exception(e, sandbox_running) # Everything else maps by code; classifiable client-side failures @@ -145,14 +163,20 @@ def handle_rpc_exception_with_health( ): """Like :func:`handle_rpc_exception`, but when the request failed at the connection level (the connection to the sandbox could not be established or was dropped - mid-request) it probes the sandbox health to tell apart the sandbox being killed - from the sandbox being unreachable or a transient network failure (e.g. a load - balancer dropping the connection). When the probe confirms the sandbox is gone, a - ``SandboxUnreachableException`` is returned; when the probe gets no answer either (the health - check raises), a ``SandboxUnreachableException``. + mid-request), or envd/the proxy answered an ``UNAVAILABLE`` that does not say + whether the sandbox is gone (:func:`is_ambiguous_unavailable`), it probes the + sandbox health to tell apart the sandbox being killed from the sandbox being + unreachable or a transient network failure (e.g. a load balancer dropping the + connection). When the probe confirms the sandbox is gone, a + ``SandboxNotRunningException`` is returned; when the probe gets no answer either + (the health check raises), a ``SandboxUnreachableException``; otherwise the error + maps as usual (an ambiguous ``UNAVAILABLE`` to a ``SandboxUnreachableException`` + with the sandbox state unknown). """ sandbox_running = None - if check_health is not None and is_transport_failure(e): + if check_health is not None and ( + is_transport_failure(e) or is_ambiguous_unavailable(e) + ): try: sandbox_running = check_health() except Exception as probe_error: @@ -167,7 +191,9 @@ async def ahandle_rpc_exception_with_health( ): """Async version of :func:`handle_rpc_exception_with_health`.""" sandbox_running = None - if check_health is not None and is_transport_failure(e): + if check_health is not None and ( + is_transport_failure(e) or is_ambiguous_unavailable(e) + ): try: sandbox_running = await check_health() except Exception as probe_error: diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index 61ded79eab..d479e21313 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -1,29 +1,38 @@ from typing import Optional -# Message of the 502 the proxy in front of envd answers with when the sandbox -# is running but nothing listens on envd's port — e.g. its network is down. # Messages of the 502 the proxy in front of envd answers with when the sandbox does # not exist anymore (killed or reached its timeout) or when it is running but envd's # port is not open (e.g. its network is down). SANDBOX_NOT_FOUND_MESSAGE = "was not found" +SANDBOX_PORT_NOT_OPEN_MESSAGE = "port is not open" def is_sandbox_not_found_message(message: str) -> bool: return SANDBOX_NOT_FOUND_MESSAGE in message +def is_sandbox_port_not_open_message(message: str) -> bool: + return SANDBOX_PORT_NOT_OPEN_MESSAGE in message + + def format_sandbox_unavailable_exception(message: str) -> Exception: """Map a 502/UNAVAILABLE answered by the proxy in front of envd: the sandbox was not found (killed or reached its timeout) is a ``SandboxNotRunningException``; - the sandbox running but envd's port not open (e.g. its network is down), or any - other message, a ``SandboxUnreachableException``.""" + the sandbox running but envd's port not open (e.g. its network is down) a + ``SandboxUnreachableException``; any other message (e.g. envd ending a stream + because the connection to the sandbox was lost) a ``SandboxUnreachableException`` + too, with the sandbox state reported as unknown.""" if is_sandbox_not_found_message(message): return SandboxNotRunningException( f"{message}: The sandbox was killed or reached its timeout. You can modify the sandbox timeout by passing 'timeout' when starting the sandbox or calling '.set_timeout' on the sandbox with the desired timeout." ) + if is_sandbox_port_not_open_message(message): + return SandboxUnreachableException( + f"{message}: The sandbox is running but envd inside it could not be reached (e.g. its network is down). Check the sandbox state with 'Sandbox.get_info()'." + ) return SandboxUnreachableException( - f"{message}: The sandbox is running but envd inside it could not be reached (e.g. its network is down). Check the sandbox state with 'Sandbox.get_info()'." + f"{message}: The sandbox could not be reached and its state is unknown — it may have been killed or reached its timeout, or the connection to it was lost. Check the sandbox state with 'Sandbox.get_info()'." ) @@ -112,8 +121,15 @@ class SandboxNotRunningException(TimeoutException): """ Raised when the sandbox is not running anymore: the proxy in front of it answered a request (or the health probe run after a request failed at the - connection level) with the sandbox not found -- it was killed or reached its - timeout. Retrying the request will not help; create or resume a sandbox. + connection level) with the sandbox not found -- it was killed, paused or + reached its timeout. Retrying the request will not help; create or resume a + sandbox. + + The split from ``SandboxUnreachableException`` is only as good as the proxy's + answer and the health probe: a sandbox that is gone but could not be asked + about is a ``SandboxUnreachableException``. Handlers that want every "sandbox + is gone" case should catch both this class and ``SandboxNotFoundException`` + (the control-plane API's answer for a sandbox that does not exist). Subclass of ``TimeoutException``, which this case surfaced as before. """ @@ -124,11 +140,15 @@ class SandboxNotRunningException(TimeoutException): class SandboxUnreachableException(TimeoutException): """ Raised when the sandbox could not be reached while it is not confirmed to be - stopped: either the proxy in front of the sandbox reports it running but - envd's port not open (e.g. the sandbox's network is down), or a request - failed at the connection level (the connection could not be established or - was dropped mid-request) and the follow-up health probe got no answer from the - sandbox either. A sandbox confirmed stopped is a ``SandboxNotRunningException``. + stopped: the proxy in front of the sandbox reports it running but envd's port + not open (e.g. the sandbox's network is down), the proxy or envd answered with + an unavailability the health probe could not resolve (e.g. the connection to + the sandbox was lost mid-stream while the probe found it running), or a + request failed at the connection level (the connection could not be + established or was dropped mid-request) and the follow-up health probe got no + answer from the sandbox either. A sandbox confirmed stopped (or paused) is a + ``SandboxNotRunningException`` -- but only as far as the probe can tell: a + sandbox that is gone and could not be asked about surfaces here. This usually means the sandbox's network is down, envd inside it is not up (yet), or a network issue between the client and the sandbox. Check the diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command.py b/packages/python-sdk/e2b/sandbox_async/commands/command.py index f2ad186066..514add11af 100644 --- a/packages/python-sdk/e2b/sandbox_async/commands/command.py +++ b/packages/python-sdk/e2b/sandbox_async/commands/command.py @@ -1,6 +1,8 @@ from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Union, overload import httpx + +from e2b.api.client_async import get_envd_api from connectrpc.code import Code from connectrpc.errors import ConnectError from packaging.version import Version @@ -49,9 +51,15 @@ def __init__( connection_config, ) self._envd_api = envd_api + # The health probe run after a request failed goes without the + # connection retries, so a sandbox that cannot be reached is not tried + # all over again (see `get_transport`). + self._envd_health_api = get_envd_api( + connection_config, envd_api_url, retry_connect=False + ) async def _check_health(self) -> Optional[bool]: - return await acheck_sandbox_health(self._envd_api) + return await acheck_sandbox_health(self._envd_health_api) async def list( self, diff --git a/packages/python-sdk/e2b/sandbox_async/commands/pty.py b/packages/python-sdk/e2b/sandbox_async/commands/pty.py index dda9d2e6f3..1047a81140 100644 --- a/packages/python-sdk/e2b/sandbox_async/commands/pty.py +++ b/packages/python-sdk/e2b/sandbox_async/commands/pty.py @@ -2,6 +2,8 @@ import httpx +from e2b.api.client_async import get_envd_api + from connectrpc.code import Code from connectrpc.errors import ConnectError from packaging.version import Version @@ -50,9 +52,15 @@ def __init__( connection_config, ) self._envd_api = envd_api + # The health probe run after a request failed goes without the + # connection retries, so a sandbox that cannot be reached is not tried + # all over again (see `get_transport`). + self._envd_health_api = get_envd_api( + connection_config, envd_api_url, retry_connect=False + ) async def _check_health(self) -> Optional[bool]: - return await acheck_sandbox_health(self._envd_api) + return await acheck_sandbox_health(self._envd_health_api) async def kill( self, diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index f4a8a4cfd7..e59612f926 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -1,5 +1,5 @@ import asyncio -from typing import IO, Callable, Dict, List, Literal, Optional, Union, overload +from typing import IO, Dict, List, Literal, Optional, Union, overload import httpx @@ -70,10 +70,9 @@ async def _ahandle_filesystem_rpc_exception( e: Exception, envd_api: httpx.AsyncClient, - error_map: dict[Code, Callable[[str], Exception]] = _FILESYSTEM_RPC_ERROR_MAP, ) -> Exception: return await ahandle_rpc_exception_with_health( - e, lambda: acheck_sandbox_health(envd_api), error_map + e, lambda: acheck_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP ) @@ -106,6 +105,12 @@ def __init__( self._envd_api_streaming = get_envd_api( connection_config, envd_api_url, for_streaming=True ) + # The health probe run after a request failed goes without the + # connection retries, so a sandbox that cannot be reached is not tried + # all over again. + self._envd_health_api = get_envd_api( + connection_config, envd_api_url, retry_connect=False + ) @overload async def read( @@ -248,7 +253,7 @@ async def read( r = await client.send(request, stream=True) except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( - e, self._envd_api + e, self._envd_health_api ) except asyncio.TimeoutError as e: # wait_for's expiry; keep the httpx exception the @@ -271,7 +276,7 @@ async def read( ) except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( - e, self._envd_api + e, self._envd_health_api ) err = await _ahandle_filesystem_envd_api_exception(r) @@ -416,7 +421,7 @@ async def _upload_file(file): ) except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( - e, self._envd_api + e, self._envd_health_api ) err = await _ahandle_filesystem_envd_api_exception(r) @@ -465,7 +470,7 @@ async def _upload_file(file): ) except httpx.TransportError as e: raise await ahandle_envd_api_transport_exception_with_health( - e, self._envd_api + e, self._envd_health_api ) err = await _ahandle_filesystem_envd_api_exception(r) @@ -520,7 +525,7 @@ async def list( return entries except Exception as e: - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_health_api) async def exists( self, @@ -552,7 +557,7 @@ async def exists( if isinstance(e, ConnectError): if e.code == Code.NOT_FOUND: return False - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_health_api) async def get_info( self, @@ -580,7 +585,7 @@ async def get_info( return map_entry_info(r.entry or filesystem_pb.EntryInfo()) except Exception as e: - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_health_api) async def remove( self, @@ -604,7 +609,7 @@ async def remove( headers=authentication_header(self._envd_version, user), ) except Exception as e: - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_health_api) async def rename( self, @@ -637,7 +642,7 @@ async def rename( return map_entry_info(r.entry or filesystem_pb.EntryInfo()) except Exception as e: - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_health_api) async def make_dir( self, @@ -668,7 +673,7 @@ async def make_dir( if isinstance(e, ConnectError): if e.code == Code.ALREADY_EXISTS: return False - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_health_api) async def watch_dir( self, @@ -751,11 +756,11 @@ async def watch_dir( events=events, on_event=on_event, on_exit=on_exit, - check_health=lambda: acheck_sandbox_health(self._envd_api), + check_health=lambda: acheck_sandbox_health(self._envd_health_api), ) except Exception as e: try: await events.aclose() except Exception: pass - raise await _ahandle_filesystem_rpc_exception(e, self._envd_api) + raise await _ahandle_filesystem_rpc_exception(e, self._envd_health_api) diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command.py b/packages/python-sdk/e2b/sandbox_sync/commands/command.py index d929301e4e..a46a84dc98 100644 --- a/packages/python-sdk/e2b/sandbox_sync/commands/command.py +++ b/packages/python-sdk/e2b/sandbox_sync/commands/command.py @@ -11,6 +11,8 @@ ) import httpx + +from e2b.api.client_sync import get_envd_api from connectrpc.code import Code from connectrpc.errors import ConnectError from packaging.version import Version @@ -58,9 +60,15 @@ def __init__( connection_config, ) self._envd_api = envd_api + # The health probe run after a request failed goes without the + # connection retries, so a sandbox that cannot be reached is not tried + # all over again (see `get_transport`). + self._envd_health_api = get_envd_api( + connection_config, envd_api_url, retry_connect=False + ) def _check_health(self) -> Optional[bool]: - return check_sandbox_health(self._envd_api) + return check_sandbox_health(self._envd_health_api) def list( self, diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py index 9465007759..5ea770d85c 100644 --- a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py +++ b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py @@ -2,6 +2,8 @@ import httpx +from e2b.api.client_sync import get_envd_api + from connectrpc.code import Code from connectrpc.errors import ConnectError from packaging.version import Version @@ -46,9 +48,15 @@ def __init__( connection_config, ) self._envd_api = envd_api + # The health probe run after a request failed goes without the + # connection retries, so a sandbox that cannot be reached is not tried + # all over again (see `get_transport`). + self._envd_health_api = get_envd_api( + connection_config, envd_api_url, retry_connect=False + ) def _check_health(self) -> Optional[bool]: - return check_sandbox_health(self._envd_api) + return check_sandbox_health(self._envd_health_api) def kill( self, diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 4ba007e230..a61936d58b 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,4 +1,4 @@ -from typing import IO, Callable, Dict, List, Literal, Optional, Union, overload +from typing import IO, Dict, List, Literal, Optional, Union, overload import httpx from connectrpc.code import Code @@ -66,10 +66,9 @@ def _handle_filesystem_rpc_exception( e: Exception, envd_api: httpx.Client, - error_map: dict[Code, Callable[[str], Exception]] = _FILESYSTEM_RPC_ERROR_MAP, ) -> Exception: return handle_rpc_exception_with_health( - e, lambda: check_sandbox_health(envd_api), error_map + e, lambda: check_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP ) @@ -104,6 +103,12 @@ def __init__( self._envd_api_streaming = get_envd_api( connection_config, envd_api_url, for_streaming=True ) + # The health probe run after a request failed goes without the + # connection retries, so a sandbox that cannot be reached is not tried + # all over again. + self._envd_health_api = get_envd_api( + connection_config, envd_api_url, retry_connect=False + ) @overload def read( @@ -228,7 +233,9 @@ def read( try: r = self._envd_api_streaming.send(request, stream=True) except httpx.TransportError as e: - raise handle_envd_api_transport_exception_with_health(e, self._envd_api) + raise handle_envd_api_transport_exception_with_health( + e, self._envd_health_api + ) err = _handle_filesystem_envd_api_exception(r) if err: @@ -245,7 +252,9 @@ def read( timeout=timeout, ) except httpx.TransportError as e: - raise handle_envd_api_transport_exception_with_health(e, self._envd_api) + raise handle_envd_api_transport_exception_with_health( + e, self._envd_health_api + ) err = _handle_filesystem_envd_api_exception(r) if err: @@ -388,7 +397,7 @@ def write_files( ) except httpx.TransportError as e: raise handle_envd_api_transport_exception_with_health( - e, self._envd_api + e, self._envd_health_api ) err = _handle_filesystem_envd_api_exception(r) @@ -430,7 +439,9 @@ def write_files( ), ) except httpx.TransportError as e: - raise handle_envd_api_transport_exception_with_health(e, self._envd_api) + raise handle_envd_api_transport_exception_with_health( + e, self._envd_health_api + ) err = _handle_filesystem_envd_api_exception(r) if err: @@ -484,7 +495,7 @@ def list( return entries except Exception as e: - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception(e, self._envd_health_api) def exists( self, @@ -515,7 +526,7 @@ def exists( if isinstance(e, ConnectError): if e.code == Code.NOT_FOUND: return False - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception(e, self._envd_health_api) def get_info( self, @@ -543,7 +554,7 @@ def get_info( return map_entry_info(r.entry or filesystem_pb.EntryInfo()) except Exception as e: - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception(e, self._envd_health_api) def remove( self, @@ -567,7 +578,7 @@ def remove( headers=authentication_header(self._envd_version, user), ) except Exception as e: - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception(e, self._envd_health_api) def rename( self, @@ -600,7 +611,7 @@ def rename( return map_entry_info(r.entry or filesystem_pb.EntryInfo()) except Exception as e: - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception(e, self._envd_health_api) def make_dir( self, @@ -631,7 +642,7 @@ def make_dir( if isinstance(e, ConnectError): if e.code == Code.ALREADY_EXISTS: return False - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception(e, self._envd_health_api) def watch_dir( self, @@ -689,7 +700,7 @@ def watch_dir( }, ) except Exception as e: - raise _handle_filesystem_rpc_exception(e, self._envd_api) + raise _handle_filesystem_rpc_exception(e, self._envd_health_api) return WatchHandle( self._rpc, @@ -697,5 +708,5 @@ def watch_dir( self._connection_config, self._envd_version, user, - lambda: check_sandbox_health(self._envd_api), + lambda: check_sandbox_health(self._envd_health_api), ) diff --git a/packages/python-sdk/tests/test_envd_client_transport.py b/packages/python-sdk/tests/test_envd_client_transport.py index 5c74c5aa09..4aaee65783 100644 --- a/packages/python-sdk/tests/test_envd_client_transport.py +++ b/packages/python-sdk/tests/test_envd_client_transport.py @@ -110,6 +110,43 @@ def test_sync_pool_is_cached_per_proxy(): assert pool_a is not pool_c +def test_sync_retry_connect_false_returns_the_pool_under_the_retrying_transport(): + key = (None, None, api_client_sync.DEFAULT_HTTP_VERSION) + retrying = api_client_sync.get_pyqwest_transport(None) + plain = api_client_sync.get_pyqwest_transport(None, retry_connect=False) + + assert isinstance(retrying, api_client_sync.ConnectionRetryTransport) + assert isinstance(plain, api_client_sync.BalancingTransport) + assert plain is api_client_sync._pools[key] + assert plain is api_client_sync.get_pyqwest_transport(None, retry_connect=False) + + # The httpx adapters are cached per flag, each over its own transport. + adapter = api_client_sync.get_httpx_transport(None) + plain_adapter = api_client_sync.get_httpx_transport(None, retry_connect=False) + assert adapter is api_client_sync.get_httpx_transport(None) + assert plain_adapter is api_client_sync.get_httpx_transport( + None, retry_connect=False + ) + assert adapter is not plain_adapter + + +def test_async_retry_connect_false_returns_the_pool_under_the_retrying_transport(): + key = (None, None, api_client_async.DEFAULT_HTTP_VERSION) + retrying = api_client_async.get_pyqwest_transport(None) + plain = api_client_async.get_pyqwest_transport(None, retry_connect=False) + + assert isinstance(retrying, api_client_async.ConnectionRetryTransport) + assert isinstance(plain, api_client_async.BalancingTransport) + assert plain is api_client_async._pools[key] + + adapter = api_client_async.get_httpx_transport(None) + plain_adapter = api_client_async.get_httpx_transport(None, retry_connect=False) + assert plain_adapter is api_client_async.get_httpx_transport( + None, retry_connect=False + ) + assert adapter is not plain_adapter + + def test_sync_pool_is_not_shared_across_proxy_credentials(): # Same proxy URL, different credentials or headers: separate pools, since # the proxy configuration is fixed per transport. diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py index f7b512d599..9208e9ba64 100644 --- a/packages/python-sdk/tests/test_envd_rpc_exception.py +++ b/packages/python-sdk/tests/test_envd_rpc_exception.py @@ -210,14 +210,75 @@ def check(): assert isinstance(err, SandboxNotRunningException) -async def test_async_ambiguous_unavailable_probes_health_and_returns_unreachable(): +# envd ends a stream with this UNAVAILABLE when the sandbox is killed mid-command +STREAM_ENDED = "the connection to sandbox ended before the stream completed" + + +def test_ambiguous_unavailable_probes_health_and_returns_not_running(): + original = ConnectError(Code.UNAVAILABLE, STREAM_ENDED) + err = handle_rpc_exception_with_health(original, lambda: False) + assert isinstance(err, SandboxNotRunningException) + assert not isinstance(err, SandboxUnreachableException) + assert err.__cause__ is original + + +def test_ambiguous_unavailable_with_sandbox_running_is_unreachable_with_unknown_state(): + err = handle_rpc_exception_with_health( + ConnectError(Code.UNAVAILABLE, STREAM_ENDED), lambda: True + ) + assert isinstance(err, SandboxUnreachableException) + assert not isinstance(err, SandboxNotRunningException) + assert "state is unknown" in str(err) + assert "is running but" not in str(err) + + +def test_ambiguous_unavailable_with_failing_probe_is_unreachable(): + def check(): + raise RuntimeError("health check failed") + + original = ConnectError(Code.UNAVAILABLE, STREAM_ENDED) + err = handle_rpc_exception_with_health(original, check) + assert isinstance(err, SandboxUnreachableException) + assert "health check failed" in str(err) + assert err.__cause__ is original + + +def test_ambiguous_unavailable_without_probe_is_unreachable_with_unknown_state(): + err = handle_rpc_exception(ConnectError(Code.UNAVAILABLE, STREAM_ENDED)) + assert isinstance(err, SandboxUnreachableException) + assert "state is unknown" in str(err) + + +def test_port_closed_unavailable_does_not_probe(): + def check(): + raise AssertionError("health check should not run") + + err = handle_rpc_exception_with_health( + ConnectError(Code.UNAVAILABLE, PORT_CLOSED), check + ) + assert isinstance(err, SandboxUnreachableException) + assert "is running but" in str(err) + + +async def test_async_ambiguous_unavailable_probes_health_and_returns_not_running(): async def check(): return False + original = ConnectError(Code.UNAVAILABLE, STREAM_ENDED) + err = await ahandle_rpc_exception_with_health(original, check) + assert isinstance(err, SandboxNotRunningException) + assert err.__cause__ is original + + +async def test_async_ambiguous_unavailable_with_sandbox_running_is_unreachable(): + async def check(): + return True + err = await ahandle_rpc_exception_with_health( - ConnectError(Code.UNAVAILABLE, "stream ended"), check + ConnectError(Code.UNAVAILABLE, STREAM_ENDED), check ) assert isinstance(err, SandboxUnreachableException) + assert "state is unknown" in str(err) async def test_async_unavailable_sandbox_not_found_returns_not_running(): diff --git a/packages/python-sdk/tests/transport_caches.py b/packages/python-sdk/tests/transport_caches.py index 0ff5ce5f55..ae4a861df8 100644 --- a/packages/python-sdk/tests/transport_caches.py +++ b/packages/python-sdk/tests/transport_caches.py @@ -18,4 +18,5 @@ def reset_transport_caches() -> None: for module in (api_client_sync, api_client_async): module._transports.clear() + module._pools.clear() module._httpx_transports.clear() From 5a6f97a0418d1f6e2bddc1f6536436be5409e84d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:57:45 +0000 Subject: [PATCH 14/16] feat(js-sdk)!: command/PTY/watch start on a gone sandbox throws SandboxNotRunningError like every other call Drop START_RPC_ERROR_MAP: a 502 'The sandbox was not found' maps to SandboxNotRunningError from every method, JS and Python alike. SandboxNotFoundError is only raised by the control plane API now. Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 17 ++++++------ packages/js-sdk/src/envd/rpc.ts | 17 ------------ packages/js-sdk/src/errors.ts | 6 ++--- packages/js-sdk/src/sandbox/commands/index.ts | 7 +---- packages/js-sdk/src/sandbox/commands/pty.ts | 13 ++------- .../js-sdk/src/sandbox/filesystem/index.ts | 14 +--------- .../js-sdk/tests/envd/handleRpcError.test.ts | 27 ++----------------- 7 files changed, 16 insertions(+), 85 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index 9b49941c6d..189e0d5576 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -10,13 +10,12 @@ Classify requests that fail because the sandbox cannot be reached. Two new error What each envd request (RPC or files API) now raises: -| Failure | JS | Python | -| -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | -| `502` / `Unavailable` "The sandbox was not found" | `SandboxNotRunningError` | `SandboxNotRunningException` | -| … when starting a command, PTY or directory watch (`commands.run` / `connect`, `pty.create` / `connect`, `files.watchDir`) | `SandboxNotFoundError` (unchanged) | `SandboxNotRunningException` (was `TimeoutException`) | -| `502` / `Unavailable` "The sandbox is running but port is not open" (e.g. `ip link set eth0 down` inside the sandbox) | `SandboxUnreachableError` | `SandboxUnreachableException` | -| any other `Unavailable` (e.g. envd ending a stream because the sandbox was killed mid-command) | health probe: gone → `SandboxNotRunningError`, otherwise `SandboxUnreachableError` ("state is unknown") | same, with the Python classes | -| any other `502` | `SandboxUnreachableError` | `SandboxUnreachableException` | -| connection could not be established or dropped mid-request (`ECONNREFUSED`, DNS, unreachable host, browsers' `Failed to fetch`, `httpx.ConnectError`, …) | health probe: gone → `SandboxNotRunningError`; running or inconclusive → the original transport error; no answer → `SandboxUnreachableError` | same, with the Python classes | +| Failure | JS | Python | +| -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | +| `502` / `Unavailable` "The sandbox was not found" | `SandboxNotRunningError` | `SandboxNotRunningException` | +| `502` / `Unavailable` "The sandbox is running but port is not open" (e.g. `ip link set eth0 down` inside the sandbox) | `SandboxUnreachableError` | `SandboxUnreachableException` | +| any other `Unavailable` (e.g. envd ending a stream because the sandbox was killed mid-command) | health probe: gone → `SandboxNotRunningError`, otherwise `SandboxUnreachableError` ("state is unknown") | same, with the Python classes | +| any other `502` | `SandboxUnreachableError` | `SandboxUnreachableException` | +| connection could not be established or dropped mid-request (`ECONNREFUSED`, DNS, unreachable host, browsers' `Failed to fetch`, `httpx.ConnectError`, …) | health probe: gone → `SandboxNotRunningError`; running or inconclusive → the original transport error; no answer → `SandboxUnreachableError` | same, with the Python classes | -The health probe runs without the SDK's connection retries, so a sandbox that cannot be reached is not tried all over again. `isRunning()` / `is_running()` still return `false` for any `502`. Plain-text `502` bodies keep the proxy's message in JS, matching Python. +The health probe runs without the SDK's connection retries, so a sandbox that cannot be reached is not tried all over again. **Breaking for JS:** `commands.run` / `connect`, `pty.create` / `connect` and `files.watchDir` on a sandbox that is gone threw `SandboxNotFoundError` ("Sandbox is probably not running anymore"); like every other call they now throw `SandboxNotRunningError` (a `TimeoutError`, not a `NotFoundError`). `SandboxNotFoundError` is only raised by the control plane API. `isRunning()` / `is_running()` still return `false` for any `502`. Plain-text `502` bodies keep the proxy's message in JS, matching Python. diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts index 8f40600d9a..893af57caa 100644 --- a/packages/js-sdk/src/envd/rpc.ts +++ b/packages/js-sdk/src/envd/rpc.ts @@ -13,7 +13,6 @@ import { NotFoundError, RateLimitError, SandboxError, - SandboxNotFoundError, SandboxNotRunningError, SandboxUnreachableError, TimeoutError, @@ -162,22 +161,6 @@ export async function resolveTransportFailure( return undefined } -/** - * Error map for starting a command, PTY or directory watch: an `Unavailable` from - * the proxy with the sandbox not found is a `SandboxNotFoundError` (as it always - * was for these calls); any other `Unavailable` maps as for every other request. - */ -export const START_RPC_ERROR_MAP: Partial< - Record Error> -> = { - [Code.Unavailable]: (message) => - isSandboxNotFoundMessage(message) - ? new SandboxNotFoundError( - `${message}: Sandbox is probably not running anymore` - ) - : formatSandboxUnavailableError(message), -} - const DEFAULT_ERROR_MAP: Partial Error>> = { [Code.InvalidArgument]: (message) => new InvalidArgumentError(message), [Code.Unauthenticated]: (message) => new AuthenticationError(message), diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index 5218e5c6ac..176fed680f 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -135,10 +135,8 @@ export class SandboxNotFoundError extends NotFoundError { * * The split from `SandboxUnreachableError` is only as good as the proxy's answer * and the health probe: a sandbox that is gone but could not be asked about is a - * `SandboxUnreachableError`. Starting a command, PTY or directory watch on a - * sandbox that is gone throws `SandboxNotFoundError` instead (as it always did), so - * handlers that want every "sandbox is gone" case have to catch both this class - * and `SandboxNotFoundError` until the next major version. + * `SandboxUnreachableError`. `SandboxNotFoundError` is only raised by the control + * plane API (e.g. `Sandbox.connect()` to a sandbox that does not exist). * * Subclass of `TimeoutError`, which this case surfaced as before. */ diff --git a/packages/js-sdk/src/sandbox/commands/index.ts b/packages/js-sdk/src/sandbox/commands/index.ts index a9569d0b17..a9905d084d 100644 --- a/packages/js-sdk/src/sandbox/commands/index.ts +++ b/packages/js-sdk/src/sandbox/commands/index.ts @@ -29,7 +29,6 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, - START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE } from '../../envd/versions' @@ -476,11 +475,7 @@ export class Commands { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck( - err, - this.checkHealth, - START_RPC_ERROR_MAP - ) + throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) } } } diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.ts index c65bca09d1..29928a92f3 100644 --- a/packages/js-sdk/src/sandbox/commands/pty.ts +++ b/packages/js-sdk/src/sandbox/commands/pty.ts @@ -22,7 +22,6 @@ import { CommandHandle } from './commandHandle' import { authenticationHeader, handleRpcErrorWithHealthCheck, - START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' import { @@ -162,11 +161,7 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck( - err, - this.checkHealth, - START_RPC_ERROR_MAP - ) + throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) } } @@ -223,11 +218,7 @@ export class Pty { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck( - err, - this.checkHealth, - START_RPC_ERROR_MAP - ) + throw await handleRpcErrorWithHealthCheck(err, this.checkHealth) } } diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 0a48ee7875..f3bae63862 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -25,7 +25,6 @@ import { import { authenticationHeader, handleRpcErrorWithHealthCheck, - START_RPC_ERROR_MAP, SandboxHealthCheck, } from '../../envd/rpc' @@ -66,13 +65,6 @@ const FILESYSTEM_RPC_ERROR_MAP: Partial< [Code.NotFound]: (message: string) => new FileNotFoundError(message), } -const WATCH_DIR_START_RPC_ERROR_MAP: Partial< - Record Error> -> = { - ...FILESYSTEM_RPC_ERROR_MAP, - ...START_RPC_ERROR_MAP, -} - async function handleFilesystemRpcError( err: unknown, checkHealth?: SandboxHealthCheck @@ -1119,11 +1111,7 @@ export class Filesystem { ) } catch (err) { cleanup() - throw await handleRpcErrorWithHealthCheck( - err, - this.checkHealth, - WATCH_DIR_START_RPC_ERROR_MAP - ) + throw await handleFilesystemRpcError(err, this.checkHealth) } } } diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts index 4d7dd5a595..a3ffcc8bf3 100644 --- a/packages/js-sdk/tests/envd/handleRpcError.test.ts +++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts @@ -6,7 +6,6 @@ import { isAmbiguousUnavailable, isTransportFailure, rejectProxyUnavailableResponse, - START_RPC_ERROR_MAP, } from '../../src/envd/rpc' import { handleProcessStartEvent, @@ -50,30 +49,8 @@ function neverProbe(): Promise { throw new Error('health check should not run') } -describe('START_RPC_ERROR_MAP', () => { - test('returns SandboxNotFoundError for Unavailable with the sandbox not found', () => { - const err = handleRpcError( - new ConnectError('The sandbox was not found', Code.Unavailable), - START_RPC_ERROR_MAP - ) - assert.instanceOf(err, SandboxNotFoundError) - assert.include(err.message, 'The sandbox was not found') - }) - - test('maps any other Unavailable like every other request', () => { - const portClosed = handleRpcError( - new ConnectError(PORT_CLOSED, Code.Unavailable), - START_RPC_ERROR_MAP - ) - assert.instanceOf(portClosed, SandboxUnreachableError) - const ambiguous = handleRpcError( - new ConnectError(STREAM_ENDED, Code.Unavailable), - START_RPC_ERROR_MAP - ) - assert.instanceOf(ambiguous, SandboxUnreachableError) - }) - - test('still returns SandboxNotRunningError without the start map', () => { +describe('Unavailable with the sandbox not found', () => { + test('returns SandboxNotRunningError, not SandboxNotFoundError, from every call', () => { const err = handleRpcError( new ConnectError('The sandbox was not found', Code.Unavailable) ) From e730668aa1269957f275f0e2172dbd4dc3dcf180 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:12:36 +0000 Subject: [PATCH 15/16] chore: bump the sandbox unreachable changeset to minor Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index 189e0d5576..ea6e531563 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -1,6 +1,6 @@ --- -'@e2b/python-sdk': patch -'e2b': patch +'@e2b/python-sdk': minor +'e2b': minor --- Classify requests that fail because the sandbox cannot be reached. Two new error classes, both subclasses of the sandbox-timeout `TimeoutError` (JS) / `TimeoutException` (Python) these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working: From 9b42f7851b9c74537cb2e2a69a85b8a7dea6bbfd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:19:12 +0000 Subject: [PATCH 16/16] docs: shorten the sandbox unreachable changeset Co-Authored-By: mish@e2b.dev --- .changeset/sandbox-unreachable-error.md | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/.changeset/sandbox-unreachable-error.md b/.changeset/sandbox-unreachable-error.md index ea6e531563..3ceed3b62f 100644 --- a/.changeset/sandbox-unreachable-error.md +++ b/.changeset/sandbox-unreachable-error.md @@ -3,19 +3,10 @@ 'e2b': minor --- -Classify requests that fail because the sandbox cannot be reached. Two new error classes, both subclasses of the sandbox-timeout `TimeoutError` (JS) / `TimeoutException` (Python) these cases surfaced as before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working: +Add `SandboxNotRunningError` / `SandboxNotRunningException` and `SandboxUnreachableError` / `SandboxUnreachableException` for envd requests that fail because the sandbox cannot be reached. Both extend `TimeoutError` / `TimeoutException`, which these cases raised before, so existing `instanceof TimeoutError` / `except TimeoutException` handling keeps working. -- `SandboxNotRunningError` / `SandboxNotRunningException` — the proxy in front of the sandbox answered `502 "The sandbox was not found"` (killed, paused or reached its timeout), or the health probe run after a failed request did. Retrying will not help; create or resume a sandbox. -- `SandboxUnreachableError` / `SandboxUnreachableException` — the sandbox is not confirmed stopped but cannot be reached. The failed request is the error's `cause` / `__cause__`. +- `SandboxNotRunning*` — the proxy answered `502 "The sandbox was not found"` (killed, paused or timed out), either directly or from the health probe the SDK runs after a dropped connection or an unexplained `Unavailable` (e.g. the sandbox was killed mid-command). Retrying will not help. +- `SandboxUnreachable*` — the sandbox is not confirmed stopped but cannot be reached: `502 "port is not open"`, other `502`s, or a connection failure whose health probe got no answer. The original error is the `cause` / `__cause__`. +- A connection failure whose health probe finds the sandbox running still raises the original transport error. The probe itself runs without connection retries. -What each envd request (RPC or files API) now raises: - -| Failure | JS | Python | -| -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | -| `502` / `Unavailable` "The sandbox was not found" | `SandboxNotRunningError` | `SandboxNotRunningException` | -| `502` / `Unavailable` "The sandbox is running but port is not open" (e.g. `ip link set eth0 down` inside the sandbox) | `SandboxUnreachableError` | `SandboxUnreachableException` | -| any other `Unavailable` (e.g. envd ending a stream because the sandbox was killed mid-command) | health probe: gone → `SandboxNotRunningError`, otherwise `SandboxUnreachableError` ("state is unknown") | same, with the Python classes | -| any other `502` | `SandboxUnreachableError` | `SandboxUnreachableException` | -| connection could not be established or dropped mid-request (`ECONNREFUSED`, DNS, unreachable host, browsers' `Failed to fetch`, `httpx.ConnectError`, …) | health probe: gone → `SandboxNotRunningError`; running or inconclusive → the original transport error; no answer → `SandboxUnreachableError` | same, with the Python classes | - -The health probe runs without the SDK's connection retries, so a sandbox that cannot be reached is not tried all over again. **Breaking for JS:** `commands.run` / `connect`, `pty.create` / `connect` and `files.watchDir` on a sandbox that is gone threw `SandboxNotFoundError` ("Sandbox is probably not running anymore"); like every other call they now throw `SandboxNotRunningError` (a `TimeoutError`, not a `NotFoundError`). `SandboxNotFoundError` is only raised by the control plane API. `isRunning()` / `is_running()` still return `false` for any `502`. Plain-text `502` bodies keep the proxy's message in JS, matching Python. +**Breaking (JS):** `commands.run` / `connect`, `pty.create` / `connect` and `files.watchDir` on a gone sandbox now throw `SandboxNotRunningError` instead of `SandboxNotFoundError`, like every other call. `SandboxNotFoundError` is only raised by the control plane API.