fix(cloud): re-send a flow submit whose response was lost

src/commands/cloud.ts

@@ -13,6 +13,10 @@ import {
 } from '../methods.js';
 import { DeviceValidationService } from '../services/device-validation.service.js';
 import { plan } from '../services/execution-plan.service.js';
+import {
+  isSubmitEndpointMissing,
+  submitWithRetry,
+} from '../services/flow-submit.service.js';
 import {
   buildTestMetadataMap,
   computeCommonRoot,
@@ -28,7 +32,7 @@ import { telemetry } from '../services/telemetry.service.js';
 import { TestSubmissionService } from '../services/test-submission.service.js';
 import { VersionService } from '../services/version.service.js';
 import { isIosMatrixConfig } from '../types/domain/device.types.js';
-import { resolveAuth } from '../utils/auth.js';
+import { refreshAuth, resolveAuth } from '../utils/auth.js';
 import {
   assertMatrixSupported,
   matrixIsIos,
@@ -929,9 +933,27 @@ export const cloudCommand = defineCommand({
       // 404s (route undefined), some proxies 405 (path/method not allowed); in
       // either case fall back to the legacy multipart POST /uploads/flow, which
       // is byte-identical bar the storage reference.
-      let response: Awaited<ReturnType<typeof ApiGateway.submitFlowTest>>;
-      try {
-        const storageRef = await uploadFlowZip({ apiUrl, auth, buffer, debug });
+      const submitMultipart = (status: number) => {
+        if (debug) {
+          out(
+            `[DEBUG] Client-direct flow upload unavailable (HTTP ${status}); falling back to multipart ${apiUrl}/uploads/flow`,
+          );
+        }
+
+        const testFormData = testSubmissionService.buildFormData(fields, buffer);
+        return ApiGateway.uploadFlow(apiUrl, auth, testFormData);
+      };
+
+      const submitFlows = async (): Promise<
+        Awaited<ReturnType<typeof ApiGateway.submitFlowTest>>
+      > => {
+        let storageRef: Awaited<ReturnType<typeof uploadFlowZip>>;
+        try {
+          storageRef = await uploadFlowZip({ apiUrl, auth, buffer, debug });
+        } catch (error) {
+          if (!isSubmitEndpointMissing(error)) throw error;
+          return submitMultipart((error as ApiError).status);
+        }
 
         if (debug) {
           out(
@@ -940,27 +962,29 @@ export const cloudCommand = defineCommand({
           out(`[DEBUG] Submitting flow test to ${apiUrl}/uploads/submitFlowTest`);
         }
 
-        response = await ApiGateway.submitFlowTest(apiUrl, auth, {
-          ...fields,
-          ...storageRef,
+        // The same body on every attempt: its upload id is what lets the API
+        // answer a repeat with the run it already created (dcd#1617 follow-up).
+        const submitBody = { ...fields, ...storageRef };
+        // The retries can outlast a `dcd login` access token, so each attempt
+        // re-checks it, as the poll loop does.
+        let submitAuth = auth;
+        const submitted = await submitWithRetry({
+          fallback: (error) => submitMultipart(error.status),
+          // Recorded before each attempt, so `command failed` carries it too.
+          onAttempt: (attempt) => telemetry.recordSubmitAttempts(attempt),
+          onRetry: ({ attempt, attempts, delayMs, reason }) =>
+            warnOut(
+              `Could not confirm the test submission (${reason}); sending it again in ${delayMs / 1000}s (attempt ${attempt}/${attempts}). It will not run twice.`,
+            ),
+          async submit() {
+            submitAuth = await refreshAuth(submitAuth);
+            return ApiGateway.submitFlowTest(apiUrl, submitAuth, submitBody);
+          },
         });
-      } catch (error) {
-        if (
-          error instanceof ApiError &&
-          (error.status === 404 || error.status === 405)
-        ) {
-          if (debug) {
-            out(
-              `[DEBUG] Client-direct flow upload unavailable (HTTP ${error.status}); falling back to multipart ${apiUrl}/uploads/flow`,
-            );
-          }
+        return submitted.response;
+      };
 
-          const testFormData = testSubmissionService.buildFormData(fields, buffer);
-          response = await ApiGateway.uploadFlow(apiUrl, auth, testFormData);
-        } else {
-          throw error;
-        }
-      }
+      const response = await submitFlows();
 
       const { message, results } = response;
 

src/gateways/api-gateway.ts

@@ -82,9 +82,14 @@ export const ApiGateway = {
     message += `  • Verify API URL is correct: ${origin}\n`;
     message += `  • Check for proxy/VPN interference\n`;
     message += `  • Try again in a few moments if server is temporarily down\n\n`;
-    message += `Original error: ${error.message}`;
-
-    const enhancedError = new Error(message);
+    // undici puts the real reason on `cause` (UND_ERR_HEADERS_TIMEOUT,
+    // ECONNRESET, ENOTFOUND, …); without it a timeout, a reset and a DNS
+    // failure all read as the same "fetch failed".
+    const cause = (error as { cause?: unknown }).cause;
+    const code = (cause as { code?: unknown } | undefined)?.code;
+    message += `Original error: ${error.message}${typeof code === 'string' ? ` (${code})` : ''}`;
+
+    const enhancedError = new Error(message, { cause: cause ?? error });
     enhancedError.name = 'NetworkError';
     enhancedError.stack = error.stack;
     return enhancedError;
@@ -761,6 +766,7 @@ export const ApiGateway = {
           ...auth.headers,
         },
         method: 'POST',
+        signal: AbortSignal.timeout(this.submitAttemptTimeoutMs),
       });
       if (!res.ok) {
         await this.handleApiError(res, 'Failed to submit test flows');
@@ -776,10 +782,29 @@ export const ApiGateway = {
         throw this.enhanceFetchError(error, `${baseUrl}/uploads/submitFlowTest`);
       }
 
+      if (error instanceof Error && error.name === 'TimeoutError') {
+        const timeout = new Error(
+          `Failed to submit test flows: no response from the API within ${this.submitAttemptTimeoutMs / 1000}s`,
+          { cause: error },
+        );
+        timeout.name = 'SubmitTimeoutError';
+        throw timeout;
+      }
+
       throw error;
     }
   },
 
+  /**
+   * How long one submit attempt may take (dcd#1617 follow-up). A submit
+   * normally answers in a second or two; one held by a Supabase stall is
+   * abandoned and re-sent by `submitWithRetry`, which is safe because the API
+   * answers a repeat with the run it already created. Well under the 300s at
+   * which Railway's edge would cut it with a 502. Settable so tests can
+   * shorten it.
+   */
+  submitAttemptTimeoutMs: 90_000,
+
   /**
    * Dry-run cost + cell-count estimate for a (possibly device-matrix)
    * submission. Runs the same resolve → validate → fan-out → price core as the

src/mcp/tools/run-cloud-test.ts

@@ -3,8 +3,12 @@ import * as path from 'node:path';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { z } from 'zod';
 
-import { ApiError, ApiGateway } from '../../gateways/api-gateway.js';
+import { ApiGateway } from '../../gateways/api-gateway.js';
 import { plan } from '../../services/execution-plan.service.js';
+import {
+  isSubmitEndpointMissing,
+  submitWithRetry,
+} from '../../services/flow-submit.service.js';
 import { computeCommonRoot, buildTestMetadataMap } from '../../services/flow-paths.js';
 import { DeviceValidationService } from '../../services/device-validation.service.js';
 import { platformFromAppFile } from '../../services/notices.service.js';
@@ -282,27 +286,48 @@ export function registerRunCloudTest(server: McpServer): void {
         // referencing it. Older API deployments lack these endpoints (404/405);
         // fall back to the legacy multipart POST /uploads/flow. Mirrors
         // `dcd cloud`.
-        let response: Awaited<ReturnType<typeof ApiGateway.submitFlowTest>>;
-        try {
-          const storageRef = await uploadFlowZip({ apiUrl, auth, buffer });
-          response = await ApiGateway.submitFlowTest(apiUrl, auth, {
-            ...fields,
-            ...storageRef,
-          });
-        } catch (error) {
-          if (
-            error instanceof ApiError &&
-            (error.status === 404 || error.status === 405)
-          ) {
-            const testFormData = testSubmissionService.buildFormData(
-              fields,
-              buffer,
-            );
-            response = await ApiGateway.uploadFlow(apiUrl, auth, testFormData);
-          } else {
-            throw error;
+        const submitMultipart = () =>
+          ApiGateway.uploadFlow(
+            apiUrl,
+            auth,
+            testSubmissionService.buildFormData(fields, buffer),
+          );
+        const submitFlows = async (): Promise<
+          Awaited<ReturnType<typeof ApiGateway.submitFlowTest>>
+        > => {
+          let storageRef: Awaited<ReturnType<typeof uploadFlowZip>>;
+          try {
+            storageRef = await uploadFlowZip({ apiUrl, auth, buffer });
+          } catch (error) {
+            if (!isSubmitEndpointMissing(error)) throw error;
+            return submitMultipart();
           }
-        }
+
+          // Re-sent while its outcome is unknown, always with the same body;
+          // see submitWithRetry (dcd#1617 follow-up).
+          const submitBody = { ...fields, ...storageRef };
+          // The retries can outlast a `dcd login` access token, so each
+          // attempt re-checks it, as the wait loop below does.
+          let submitAuth = auth;
+          const submitted = await submitWithRetry({
+            fallback: submitMultipart,
+            // Recorded before each attempt, so a failed tool call carries it.
+            onAttempt: (attempt) => {
+              telemetryExtra.submitAttempts = attempt;
+            },
+            onRetry: ({ attempt, attempts, delayMs, reason }) =>
+              logStderr(
+                `Could not confirm the test submission (${reason}); sending it again in ${delayMs / 1000}s (attempt ${attempt}/${attempts}). It will not run twice.`,
+              ),
+            async submit() {
+              submitAuth = await refreshAuth(submitAuth);
+              return ApiGateway.submitFlowTest(apiUrl, submitAuth, submitBody);
+            },
+          });
+          return submitted.response;
+        };
+
+        const response = await submitFlows();
 
         const { message, results } = response;
         if (!results?.length) {

src/services/flow-submit.service.ts

@@ -0,0 +1,150 @@
+import { ApiError } from '../gateways/api-gateway.js';
+import { isDefinitiveAuthFailure } from '../utils/auth.js';
+
+/**
+ * Submitting a flow test when the response may be lost (dcd#1617 follow-up).
+ *
+ * Supabase's ingress sometimes holds a request for minutes before answering,
+ * and Railway's edge cuts a request that is silent for 300s with a 502. Either
+ * way the CLI cannot tell whether its run was created. The API makes that safe
+ * to find out: it answers a repeat of the same submission (same body, same
+ * upload id from getFlowUploadUrl) with the run it already created, or a 409
+ * while that run is still being created. It never creates a second run or a
+ * second charge. So a submit whose outcome is unknown is simply sent again.
+ */
+
+/**
+ * Waits before each re-send: 2s, then up to 30s apart, about three minutes in
+ * all. That is long enough for a write held in a stall to land, after which the
+ * repeat is answered with the run.
+ */
+export const SUBMIT_RETRY_DELAYS_MS: readonly number[] = [
+  2000, 5000, 10_000, 20_000, 30_000, 30_000, 30_000, 30_000,
+];
+
+/**
+ * No attempt starts more than this long after the first. Each attempt can take
+ * up to its 90s timeout, so without a cap every attempt timing out would hold
+ * the command for about 16 minutes; with it, the worst case is about 6.5.
+ */
+export const SUBMIT_RETRY_BUDGET_MS = 5 * 60_000;
+
+/**
+ * Statuses that leave the outcome unknown (502/503/504 from the edge or the
+ * API), say the API could not finish (500), or say the first attempt is still
+ * creating the run (409 SUBMISSION_IN_PROGRESS). Re-sending is only safe
+ * because the API dedupes on the upload id.
+ */
+const RETRYABLE_STATUSES = new Set([409, 500, 502, 503, 504]);
+
+/**
+ * Whether a failed attempt is worth sending again. Anything that is not an API
+ * answer is a transport failure: a network error, the attempt timeout, or a
+ * body that died mid-read after a 2xx. In every case the run may already exist,
+ * which a repeat finds out. The npm build throws undici's errors and the bun
+ * binaries their own; neither is an ApiError, so both retry.
+ */
+export function isRetryableSubmitError(error: unknown): boolean {
+  if (error instanceof ApiError) return RETRYABLE_STATUSES.has(error.status);
+  // The session can't be refreshed (refused, or logged out): only `dcd login`
+  // fixes that, so another attempt would fail the same way.
+  if (isDefinitiveAuthFailure(error)) return false;
+  return true;
+}
+
+/** A one-line reason for the retry notice. Network errors carry a long help text. */
+export function describeSubmitError(error: unknown): string {
+  if (error instanceof ApiError) return `HTTP ${error.status}`;
+  if (error instanceof Error) {
+    if (error.name === 'SubmitTimeoutError') return 'no response in time';
+    const code = (error.cause as { code?: unknown } | undefined)?.code;
+    if (typeof code === 'string') return code;
+    return error.name === 'NetworkError' ? 'network error' : error.name;
+  }
+
+  return String(error);
+}
+
+/** The API predates the client-direct submit: a real API 404s, some proxies 405. */
+export function isSubmitEndpointMissing(error: unknown): boolean {
+  return (
+    error instanceof ApiError && (error.status === 404 || error.status === 405)
+  );
+}
+
+export interface SubmitRetryNotice {
+  /** The attempt about to be sent, counting from 1. */
+  attempt: number;
+  attempts: number;
+  delayMs: number;
+  reason: string;
+}
+
+/**
+ * Send a submission, sending it again while its outcome is unknown.
+ *
+ * `fallback` is the legacy multipart submit for an API that predates this
+ * endpoint. It is used only when the *first* attempt says the endpoint is
+ * missing. A 404 on a later attempt comes from an API that took the earlier
+ * attempt, so a multipart submit then would create a second run.
+ *
+ * Resolves with the response and how many attempts it took. Throws the last
+ * error when it is not retryable or the attempts run out.
+ */
+export async function submitWithRetry<T>(opts: {
+  budgetMs?: number;
+  delaysMs?: readonly number[];
+  fallback: (error: ApiError) => Promise<T>;
+  now?: () => number;
+  /**
+   * Called before each attempt with its number, counting from 1, so a caller
+   * can record how many were made whether the submit succeeds or not.
+   */
+  onAttempt?: (attempt: number) => void;
+  onRetry?: (notice: SubmitRetryNotice) => void;
+  sleep?: (ms: number) => Promise<void>;
+  submit: () => Promise<T>;
+}): Promise<{ attempts: number; response: T }> {
+  const delaysMs = opts.delaysMs ?? SUBMIT_RETRY_DELAYS_MS;
+  const budgetMs = opts.budgetMs ?? SUBMIT_RETRY_BUDGET_MS;
+  const now = opts.now ?? Date.now;
+  const startedAt = now();
+  const sleep =
+    opts.sleep ??
+    ((ms: number) =>
+      new Promise<void>((resolve) => {
+        setTimeout(resolve, ms);
+      }));
+  const attempts = delaysMs.length + 1;
+
+  for (let attempt = 1; ; attempt++) {
+    opts.onAttempt?.(attempt);
+    try {
+      return { attempts: attempt, response: await opts.submit() };
+    } catch (error) {
+      if (attempt === 1 && isSubmitEndpointMissing(error)) {
+        return {
+          attempts: attempt,
+          response: await opts.fallback(error as ApiError),
+        };
+      }
+
+      const delayMs = delaysMs[attempt - 1];
+      if (
+        delayMs === undefined ||
+        !isRetryableSubmitError(error) ||
+        now() - startedAt + delayMs > budgetMs
+      ) {
+        throw error;
+      }
+
+      opts.onRetry?.({
+        attempt: attempt + 1,
+        attempts,
+        delayMs,
+        reason: describeSubmitError(error),
+      });
+      await sleep(delayMs);
+    }
+  }
+}