feat(webapp): deployment lifecycle telemetry events per build path

Replaces the deployment.outcome span with a wide deployment.lifecycle
event emitted once per terminal transition (DEPLOYED/FAILED/TIMED_OUT/
CANCELED), backdated createdAt-to-terminal, carrying build path (depot/
native/local_bundle), per-phase durations derived from the persisted
timestamp chain, error class, org/project/env, runtime, CLI version and
trigger source as attributes. A zero-duration deployment.initialized
event at creation provides the funnel denominator for stuck-deployment
detection.

Events are emitted on ROOT_CONTEXT with the forceRecording attribute:
the previous span was started under the ambient request context, where
the parent-based sampler drops ~95% of traffic before the force-record
check runs. SEMINTATTRS_FORCE_RECORDING is now exported for this.

The fail, timeout and finalize transitions now use guarded updateMany
writes so exactly one caller commits a terminal status and emits the
event; this also stops a late timeout from overwriting DEPLOYED. The
cancel path now emits too (it previously recorded nothing).

Also: cliVersion is stamped onto WorkerDeployment at initialization from
the x-trigger-cli-version header (previously only available post-index
via BackgroundWorker, i.e. null for pre-index failures); an optional
second OTLP exporter (INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL)
mirrors deployment.* spans into a dedicated dataset; the tracer provider
is flushed on SIGTERM/SIGINT so shutdowns stop dropping the last batch.

Author: myftija

apps/webapp/app/env.server.ts

@@ -935,6 +935,12 @@ const EnvironmentSchema = z
     DISABLE_HTTP_INSTRUMENTATION: BoolEnv.default(false),
 
     INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(),
+
+    // Optional second OTLP trace exporter that receives only `deployment.*`
+    // spans (deployment lifecycle analytics), e.g. a dedicated long-retention
+    // Axiom dataset. The spans also still flow to the main trace exporter.
+    INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL: z.string().optional(),
+    INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS: z.string().optional(),
     INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(),
     INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(),
     INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"),

apps/webapp/app/routes/api.v1.deployments.ts

@@ -42,7 +42,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const service = new InitializeDeploymentService();
 
   try {
-    const result = await service.call(authenticatedEnv, body.data);
+    const result = await service.call(authenticatedEnv, body.data, {
+      cliVersion: request.headers.get("x-trigger-cli-version") ?? undefined,
+    });
     const { deployment, imageRef } = result;
 
     const responseBody: InitializeDeploymentResponseBody = {

apps/webapp/app/v3/DEPLOYMENT_TELEMETRY_ATTRIBUTES.md

@@ -0,0 +1,44 @@
+# Deployment telemetry attributes
+
+`deploymentTelemetry.ts` is the single owner of these names. `deployment.lifecycle`
+(one wide event per terminal transition, span backdated createdAt → terminal) and
+`deployment.initialized` (zero-duration funnel event at creation) are emitted by
+`services/recordDeploymentLifecycle.server.ts`. Axiom queries, dashboards, and
+monitors reference these names — treat renames as breaking changes.
+
+| Attribute                        | Events           | Values / notes                                                             |
+| -------------------------------- | ---------------- | -------------------------------------------------------------------------- |
+| `$trigger.org.id`                | both             | Organization id                                                             |
+| `$trigger.project.id`            | both             | Project id                                                                  |
+| `$trigger.project.ref`           | both             | Project external ref (`proj_…`)                                            |
+| `$trigger.env.id`                | both             | Environment id                                                              |
+| `$trigger.env.type`              | both             | `PRODUCTION` / `STAGING` / `PREVIEW` / `DEVELOPMENT`                        |
+| `deployment.id`                  | both             | Deployment friendly id — dedup key (`arg_max(_time, *) by deployment.id`)   |
+| `deployment.version`             | both             | Deployment version, e.g. `20260825.3`                                       |
+| `deployment.status`              | both             | lifecycle: terminal status; initialized: initial status (`PENDING`/`BUILDING`) |
+| `deployment.success`             | lifecycle        | `status === "DEPLOYED"`. CANCELED is excluded from failure rates            |
+| `deployment.build_path`          | both             | `depot` / `native` / `local_bundle` (rare `--local-build` lands in `depot`) |
+| `deployment.worker_type`         | both             | `V1` / `MANAGED` (run engine)                                               |
+| `deployment.runtime`             | both             | `node` / `node-22` / `bun` / …                                              |
+| `deployment.runtime_version`     | lifecycle        | Set at indexing; null for pre-index failures                                |
+| `deployment.cli_version`         | both             | From `x-trigger-cli-version` at init; null for pre-column history           |
+| `deployment.triggered_via`       | both             | e.g. `cli`, GitHub/Vercel integrations                                      |
+| `deployment.commit_sha`          | lifecycle        | From git meta when present                                                  |
+| `deployment.error.name`          | lifecycle        | Error class from `errorData` (`TimeoutError`, build errors, …)              |
+| `deployment.error.message`       | lifecycle        | Human-readable failure reason                                               |
+| `deployment.canceled_reason`     | lifecycle        | Only on CANCELED                                                            |
+| `deployment.duration.total_ms`   | lifecycle        | createdAt → terminal (also the span's own duration)                         |
+| `deployment.duration.queue_ms`   | lifecycle        | createdAt → startedAt; ≈0 when created directly in BUILDING (depot)         |
+| `deployment.duration.install_ms` | lifecycle        | startedAt → installedAt; build-server paths only (depot never sets it)      |
+| `deployment.duration.building_ms`| lifecycle        | (installedAt ?? startedAt) → builtAt                                        |
+| `deployment.duration.deploying_ms`| lifecycle       | builtAt → terminal; for depot dominated by the server-side registry push    |
+
+The span's `_time` is the deployment's **createdAt**, so a TIMED_OUT event lands
+backdated by up to the full deploy timeout (~23 min at current defaults) — monitors
+must use windows longer than the max timeout or they will systematically miss the
+stuck deployments they exist to catch.
+
+Phase durations are omitted (not zero) when a boundary timestamp is missing —
+timestamp chains are path-shaped. Compare only shared phases across build paths;
+`total_ms` excludes local-bundle's pre-init client work (esbuild + upload) until
+the CLI passes client timings.

apps/webapp/app/v3/deploymentTelemetry.ts

@@ -0,0 +1,95 @@
+import { BuildServerMetadata } from "@trigger.dev/core/v3";
+
+// Attribute names for the deployment telemetry events (see
+// DEPLOYMENT_TELEMETRY_ATTRIBUTES.md next to this file). This module is the single owner of these names — Axiom
+// queries, dashboards, and monitors reference them, so treat renames as
+// breaking changes.
+export const DeploymentTelemetryAttributes = {
+  ORG_ID: "$trigger.org.id",
+  PROJECT_ID: "$trigger.project.id",
+  PROJECT_REF: "$trigger.project.ref",
+  ENV_ID: "$trigger.env.id",
+  ENV_TYPE: "$trigger.env.type",
+  DEPLOYMENT_ID: "deployment.id",
+  VERSION: "deployment.version",
+  STATUS: "deployment.status",
+  SUCCESS: "deployment.success",
+  BUILD_PATH: "deployment.build_path",
+  WORKER_TYPE: "deployment.worker_type",
+  RUNTIME: "deployment.runtime",
+  RUNTIME_VERSION: "deployment.runtime_version",
+  CLI_VERSION: "deployment.cli_version",
+  TRIGGERED_VIA: "deployment.triggered_via",
+  COMMIT_SHA: "deployment.commit_sha",
+  ERROR_NAME: "deployment.error.name",
+  ERROR_MESSAGE: "deployment.error.message",
+  CANCELED_REASON: "deployment.canceled_reason",
+  DURATION_TOTAL_MS: "deployment.duration.total_ms",
+  DURATION_QUEUE_MS: "deployment.duration.queue_ms",
+  DURATION_INSTALL_MS: "deployment.duration.install_ms",
+  DURATION_BUILDING_MS: "deployment.duration.building_ms",
+  DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms",
+} as const;
+
+export type DeploymentBuildPath = "local_bundle" | "native" | "depot";
+
+/**
+ * Classifies which build path produced a deployment, from its persisted
+ * metadata. Everything that is not a native-build-server deployment falls into
+ * the depot bucket — including rare `--local-build` deploys, whose flag is not
+ * persisted. `externalBuildData` is NOT usable as a depot signal: init writes a
+ * placeholder (`"-"` fields) for every path.
+ */
+export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPath {
+  const metadata = BuildServerMetadata.safeParse(buildServerMetadata);
+
+  if (metadata.success && metadata.data.isNativeBuild) {
+    return metadata.data.fromBundle ? "local_bundle" : "native";
+  }
+
+  return "depot";
+}
+
+export type DeploymentTimestamps = {
+  createdAt: Date;
+  startedAt?: Date | null;
+  installedAt?: Date | null;
+  builtAt?: Date | null;
+};
+
+export type DeploymentDurations = {
+  totalMs: number;
+  queueMs?: number;
+  installMs?: number;
+  buildingMs?: number;
+  deployingMs?: number;
+};
+
+/**
+ * Derives per-phase durations from the persisted timestamp chain
+ * (createdAt → startedAt → installedAt → builtAt → terminal). Chains are
+ * path-shaped: depot never sets installedAt (the /progress route is
+ * build-server-only) and PENDING-skipping deploys have queue ≈ 0 — each phase
+ * is emitted only when both of its boundary timestamps exist and are ordered.
+ */
+export function deriveDeploymentDurations(
+  timestamps: DeploymentTimestamps,
+  terminalAt: Date
+): DeploymentDurations {
+  const { createdAt, startedAt, installedAt, builtAt } = timestamps;
+  const buildingFrom = installedAt ?? startedAt;
+
+  return {
+    totalMs: Math.max(terminalAt.getTime() - createdAt.getTime(), 0),
+    queueMs: msBetween(createdAt, startedAt),
+    installMs: msBetween(startedAt, installedAt),
+    buildingMs: msBetween(buildingFrom, builtAt),
+    deployingMs: msBetween(builtAt, terminalAt),
+  };
+}
+
+function msBetween(from?: Date | null, to?: Date | null): number | undefined {
+  if (!from || !to) return undefined;
+  const ms = to.getTime() - from.getTime();
+  return ms >= 0 ? ms : undefined;
+}

apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts

@@ -18,7 +18,7 @@ import {
 } from "./createBackgroundWorker.server";
 import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server";
 import { TimeoutDeploymentService } from "./timeoutDeployment.server";
-import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
+import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server";
 import { env } from "~/env.server";
 import { webhookPrisma } from "~/db.server";
 
@@ -298,6 +298,12 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
     error: Error,
     environment: AuthenticatedEnvironment
   ) {
+    const failedAt = new Date();
+    const errorData = {
+      name: error.name,
+      message: error.message,
+    };
+
     // Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING
     // transition in `call()`. With idempotent retries, two attempts can run side-by-side;
     // without the predicate, one attempt's failure could downgrade the deployment after
@@ -309,11 +315,8 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
       },
       data: {
         status: "FAILED",
-        failedAt: new Date(),
-        errorData: {
-          name: error.name,
-          message: error.message,
-        },
+        failedAt,
+        errorData,
         buildEnvVars: Prisma.DbNull,
       },
     });
@@ -332,13 +335,16 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
       // BUILDING → DEPLOYING transition.
       await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
 
-      recordDeploymentOutcome({
+      recordDeploymentLifecycle({
         status: "FAILED",
-        deploymentFriendlyId: deployment.friendlyId,
-        organizationId: environment.organizationId,
-        projectId: environment.projectId,
-        environmentId: environment.id,
-        environmentType: environment.type,
+        deployment: { ...deployment, status: "FAILED", failedAt, errorData },
+        environment: {
+          organizationId: environment.organizationId,
+          projectId: environment.projectId,
+          projectRef: environment.project.externalRef,
+          environmentId: environment.id,
+          environmentType: environment.type,
+        },
         reason: error.message,
       });
     }

apps/webapp/app/v3/services/deployment.server.ts

@@ -9,6 +9,7 @@ import {
   type DeploymentEvent,
 } from "@trigger.dev/core/v3";
 import { TimeoutDeploymentService } from "./timeoutDeployment.server";
+import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server";
 import { env } from "~/env.server";
 import { createRemoteImageBuild } from "../remoteImageBuilder.server";
 import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server";
@@ -238,6 +239,8 @@ export class DeploymentService extends BaseService {
         if (result.count === 0) {
           return errAsync({ type: "deployment_cannot_be_cancelled" as const });
         }
+        // Fire-and-forget: telemetry must never affect the cancel result.
+        void this.#recordCanceledLifecycle(deployment.id);
         return okAsync({ deployment });
       });
 
@@ -474,6 +477,41 @@ export class DeploymentService extends BaseService {
     );
   }
 
+  // The cancel path only carries a narrow row selection, so re-fetch the full
+  // row (post-update, status already CANCELED) for the lifecycle event.
+  async #recordCanceledLifecycle(deploymentId: string) {
+    try {
+      const canceled = await this._prisma.workerDeployment.findFirst({
+        where: { id: deploymentId },
+        include: {
+          environment: {
+            include: {
+              project: {
+                select: { id: true, organizationId: true, externalRef: true },
+              },
+            },
+          },
+        },
+      });
+
+      if (!canceled || canceled.status !== "CANCELED") return;
+
+      recordDeploymentLifecycle({
+        status: "CANCELED",
+        deployment: canceled,
+        environment: {
+          organizationId: canceled.environment.project.organizationId,
+          projectId: canceled.environment.project.id,
+          projectRef: canceled.environment.project.externalRef,
+          environmentId: canceled.environmentId,
+          environmentType: canceled.environment.type,
+        },
+      });
+    } catch (error) {
+      logger.error("Failed to record canceled deployment lifecycle", { deploymentId, error });
+    }
+  }
+
   private getDeployment(environmentId: string, friendlyId: string) {
     return fromPromise(
       this._prisma.workerDeployment.findFirst({

apps/webapp/app/v3/services/failDeployment.server.ts

@@ -5,7 +5,7 @@ import { Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database";
 import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
 import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { DeploymentService } from "./deployment.server";
-import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
+import { recordDeploymentLifecycle } from "./recordDeploymentLifecycle.server";
 
 export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
   "CANCELED",
@@ -41,25 +41,50 @@ export class FailDeploymentService extends BaseService {
       return;
     }
 
-    const failedDeployment = await this._prisma.workerDeployment.update({
+    const failedAt = new Date();
+
+    // Guarded transition: a concurrent finalize/timeout/cancel can win between
+    // the check above and this write; the predicate makes exactly one caller
+    // commit the terminal status (and emit the lifecycle event).
+    const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({
       where: {
         id: deployment.id,
+        status: { notIn: FINAL_DEPLOYMENT_STATUSES },
       },
       data: {
         status: "FAILED",
-        failedAt: new Date(),
+        failedAt,
         errorData: params.error,
         buildEnvVars: Prisma.DbNull,
       },
     });
 
-    recordDeploymentOutcome({
+    if (updatedCount === 0) {
+      logger.warn("Worker deployment reached a final state concurrently, skipping fail", {
+        id: deployment.id,
+        friendlyId,
+      });
+      return;
+    }
+
+    const failedDeployment = {
+      ...deployment,
+      status: "FAILED" as const,
+      failedAt,
+      errorData: params.error,
+      buildEnvVars: null,
+    };
+
+    recordDeploymentLifecycle({
       status: "FAILED",
-      deploymentFriendlyId: friendlyId,
-      organizationId: authenticatedEnv.organizationId,
-      projectId: authenticatedEnv.projectId,
-      environmentId: authenticatedEnv.id,
-      environmentType: authenticatedEnv.type,
+      deployment: failedDeployment,
+      environment: {
+        organizationId: authenticatedEnv.organizationId,
+        projectId: authenticatedEnv.projectId,
+        projectRef: authenticatedEnv.project.externalRef,
+        environmentId: authenticatedEnv.id,
+        environmentType: authenticatedEnv.type,
+      },
       reason: params.error.message,
     });