feat(webapp): resolve which shard an environment mints run roots into

apps/webapp/app/env.server.ts

@@ -4,6 +4,7 @@ import { BoolEnv } from "./utils/boolEnv";
 import { isValidDatabaseUrl } from "./utils/db";
 import { isValidRegex } from "./utils/regex";
 import { isValidDuration } from "./services/realtime/duration.server";
+import { parseShardCsv } from "./v3/runOpsMigration/mintShardGrace";
 
 // `z.string()` constrained to a `parseDuration`-parseable string (e.g.
 // `7d`, `1h`). Validated at boot so a typo'd duration fails fast.
@@ -41,6 +42,23 @@ const parseMachinePresetCsv = (raw: string, ctx: z.RefinementCtx): MachinePreset
   return out;
 };
 
+// A CSV of gen-2 mint shard keys, validated at boot by parseShardCsv. Kept as the raw string:
+// the resolution is built once in runOpsMintShard.server.ts, and this only has to fail fast.
+const shardCsvString = () =>
+  z
+    .string()
+    .default("")
+    .superRefine((raw, ctx) => {
+      try {
+        parseShardCsv(raw);
+      } catch (error) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: error instanceof Error ? error.message : "invalid shard key CSV",
+        });
+      }
+    });
+
 const GithubAppEnvSchema = z.preprocess(
   (val) => {
     const obj = val as any;
@@ -1998,6 +2016,15 @@ const EnvironmentSchema = z
     // (stale or fresh) resolves to the same kind for the whole window. See mintFlipGrace.ts.
     RUN_OPS_MINT_FLIP_GRACE_MS: z.coerce.number().int().default(90_000),
 
+    // Gen-2 mint shards — CSV of single-char [a-z0-9] keys eligible for ROOT minting. Unset or
+    // empty means no gen-2 minting, which is today's behaviour. Validated at boot: an invalid
+    // key would mint an id that cannot be routed. _PREV + _FLIPPED_AT stamp a set change so
+    // every process crosses the cutover together; set both, or the grace never applies.
+    // Removing a key stops new roots on it and never stops routing it. See mintShardGrace.ts.
+    RUN_OPS_MINT_SHARDS: shardCsvString(),
+    RUN_OPS_MINT_SHARDS_PREV: shardCsvString(),
+    RUN_OPS_MINT_SHARDS_FLIPPED_AT: z.string().datetime().optional(),
+
     // Session replication (Postgres → ClickHouse sessions_v1). Shares Redis
     // with the runs replicator for leader locking but has its own slot and
     // publication so the two consume independently.

apps/webapp/app/v3/featureFlags.ts

@@ -26,6 +26,9 @@ export const FEATURE_FLAG = {
   // Grace-linger stamp carried alongside runOpsMintKind on flip. See mintFlipGrace.ts.
   runOpsMintKindPrev: "runOpsMintKindPrev",
   runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt",
+  // Gen-2 mint shard pins, read from the org override blob only. See runOpsMintShard.server.ts.
+  runOpsMintShard: "runOpsMintShard",
+  runOpsMintShardEnvPins: "runOpsMintShardEnvPins",
   queueMetricsUiEnabled: "queueMetricsUiEnabled",
   // Per-organization rollout for creating additional environment API keys.
   additionalApiKeysEnabled: "additionalApiKeysEnabled",
@@ -89,6 +92,32 @@ export const FeatureFlagCatalog = {
   // by stampMintKindFlip on a genuine flip. Display-only (see ORG_LOCKED_FLAGS).
   [FEATURE_FLAG.runOpsMintKindPrev]: z.enum(["cuid", "runOpsId"]),
   [FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(),
+  // Pins one org to a gen-2 mint shard. "new" holds the org on gen-1 run-ops ids, which is how
+  // a canary keeps the fleet's default while one org moves. Only honored while the key is in
+  // the active set (RUN_OPS_MINT_SHARDS); a drained key falls through to the hash.
+  [FEATURE_FLAG.runOpsMintShard]: z
+    .string()
+    .refine((v) => v === "new" || /^[a-z0-9]$/.test(v), 'must be a single [a-z0-9] char, or "new"'),
+  // Per-environment pins as JSON: {"<environmentId>": "<shard key>"}. A JSON string because
+  // this catalog is scalar-only. Rejected at write, so a typo cannot silently un-pin an env.
+  [FEATURE_FLAG.runOpsMintShardEnvPins]: z.string().superRefine((raw, ctx) => {
+    const fail = (message: string) => ctx.addIssue({ code: z.ZodIssueCode.custom, message });
+
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      return fail("must be valid JSON");
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return fail("must be a JSON object mapping environment id to shard key");
+    }
+    for (const [environmentId, value] of Object.entries(parsed)) {
+      if (typeof value !== "string" || !(value === "new" || /^[a-z0-9]$/.test(value))) {
+        fail(`"${environmentId}" must map to a single [a-z0-9] char, or "new"`);
+      }
+    }
+  }),
   // Per-org access to the Queue Metrics dashboard UI (view only; emission is global and
   // separate). Off unless enabled for the org.
   [FEATURE_FLAG.queueMetricsUiEnabled]: z.coerce.boolean(),
@@ -101,11 +130,15 @@ export const FeatureFlagCatalog = {
 
 export type FeatureFlagKey = keyof typeof FeatureFlagCatalog;
 
-// Infrastructure flags that are read-only on the global flags page.
-// Shown with current/resolved value but no controls.
+// Infrastructure flags, plus org-scoped-only flags, that are read-only on the global flags
+// page. Shown with current/resolved value but no controls. An org-scoped-only flag belongs
+// here because its resolver never reads a global row, so an editable global control would
+// offer a setting that does nothing.
 export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [
   FEATURE_FLAG.defaultWorkerInstanceGroupId,
   FEATURE_FLAG.taskEventRepository,
+  FEATURE_FLAG.runOpsMintShard,
+  FEATURE_FLAG.runOpsMintShardEnvPins,
 ];
 
 // Flags that are read-only on the org-level dialog.

apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts

@@ -0,0 +1,148 @@
+import { describe, expect, it } from "vitest";
+import { generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic";
+import {
+  buildMintShardResolution,
+  effectiveMintShardSet,
+  isValidPinValue,
+  parseShardCsv,
+  SHARD_KEY_PATTERN,
+  type MintShardSetResolution,
+} from "./mintShardGrace";
+
+const GRACE_MS = 90_000;
+const T = 1_000_000;
+
+describe("parseShardCsv", () => {
+  it("returns an empty list for unset, empty and whitespace input", () => {
+    expect(parseShardCsv(undefined)).toEqual([]);
+    expect(parseShardCsv("")).toEqual([]);
+    expect(parseShardCsv("  ")).toEqual([]);
+    expect(parseShardCsv(",,")).toEqual([]);
+  });
+
+  it("trims, dedupes and SORTS, so operator typing order cannot change HRW", () => {
+    expect(parseShardCsv("b, a ,b")).toEqual(["a", "b"]);
+    expect(parseShardCsv("a,b,c")).toEqual(parseShardCsv("c,b,a"));
+    expect(parseShardCsv("b,c,a")).toEqual(parseShardCsv("a,c,b"));
+  });
+
+  it("accepts every one of the 36 legal shard keys", () => {
+    const all = "abcdefghijklmnopqrstuvwxyz0123456789".split("");
+    expect(parseShardCsv(all.join(","))).toEqual([...all].sort());
+  });
+
+  it("throws on a key outside [a-z0-9]", () => {
+    // generateRunOpsIdV2 throws on these; an unvalidated key MUST fail at boot, not at mint.
+    expect(() => parseShardCsv("A")).toThrow(/shard key/i);
+    expect(() => parseShardCsv("ab")).toThrow(/shard key/i);
+    expect(() => parseShardCsv("a,-")).toThrow(/shard key/i);
+    expect(() => parseShardCsv("a,_")).toThrow(/shard key/i);
+  });
+
+  it("rejects the reserved keys by name", () => {
+    expect(() => parseShardCsv("new")).toThrow(/reserved/i);
+    expect(() => parseShardCsv("a,legacy")).toThrow(/reserved/i);
+  });
+});
+
+// Core does not export its shard-char pattern, so pin the local one to the real minter.
+describe("shard alphabet agrees with the core minter", () => {
+  it("accepts exactly the characters generateRunOpsIdV2 accepts", () => {
+    const candidates = [
+      ..."abcdefghijklmnopqrstuvwxyz0123456789".split(""),
+      ..."ABZ-_. +/é!".split(""),
+      "",
+      "ab",
+    ];
+
+    for (const candidate of candidates) {
+      let minterAccepts = true;
+      try {
+        generateRunOpsIdV2(candidate);
+      } catch {
+        minterAccepts = false;
+      }
+
+      expect(SHARD_KEY_PATTERN.test(candidate)).toBe(minterAccepts);
+    }
+  });
+});
+
+describe("isValidPinValue", () => {
+  it('accepts a shard key, and accepts "new" as the gen-1 hold value', () => {
+    expect(isValidPinValue("a")).toBe(true);
+    expect(isValidPinValue("7")).toBe(true);
+    expect(isValidPinValue("new")).toBe(true);
+  });
+
+  it("rejects legacy, and rejects anything outside the alphabet", () => {
+    expect(isValidPinValue("legacy")).toBe(false);
+    expect(isValidPinValue("A")).toBe(false);
+    expect(isValidPinValue("ab")).toBe(false);
+    expect(isValidPinValue("")).toBe(false);
+  });
+});
+
+describe("effectiveMintShardSet", () => {
+  it("returns set when there is no stamp", () => {
+    const r: MintShardSetResolution = { set: ["a", "b"] };
+    expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a", "b"]);
+  });
+
+  it("returns set when flippedAtMs is absent even though prevSet is present", () => {
+    const r: MintShardSetResolution = { set: ["a", "b"], prevSet: ["a"] };
+    expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a", "b"]);
+  });
+
+  it("serves prevSet inside the window and set at/after the boundary", () => {
+    const r: MintShardSetResolution = { set: ["a", "b"], prevSet: ["a"], flippedAtMs: T };
+    expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a"]);
+    expect(effectiveMintShardSet(r, T + GRACE_MS - 1, GRACE_MS)).toEqual(["a"]);
+    // Boundary is exclusive on the prev side, so every process crosses it together.
+    expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a", "b"]);
+    expect(effectiveMintShardSet(r, T + GRACE_MS + 1, GRACE_MS)).toEqual(["a", "b"]);
+  });
+
+  it("represents a graced first activation as an empty prevSet", () => {
+    const r: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T };
+    expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual([]);
+    expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a"]);
+  });
+
+  it("serves a drain through the window", () => {
+    const r: MintShardSetResolution = { set: ["a"], prevSet: ["a", "b"], flippedAtMs: T };
+    expect(effectiveMintShardSet(r, T + 1, GRACE_MS)).toEqual(["a", "b"]);
+    expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a"]);
+  });
+});
+
+describe("buildMintShardResolution", () => {
+  it("omits prevSet entirely when no flip timestamp is configured", () => {
+    // A prevSet with no timestamp can never apply, so it MUST NOT linger.
+    const r = buildMintShardResolution({ shards: "a,b", prev: "a", flippedAt: undefined });
+    expect(r).toEqual({ set: ["a", "b"], prevSet: undefined, flippedAtMs: undefined });
+  });
+
+  it("keeps an empty prevSet when a flip timestamp IS configured", () => {
+    const r = buildMintShardResolution({
+      shards: "a",
+      prev: "",
+      flippedAt: new Date(T).toISOString(),
+    });
+    expect(r).toEqual({ set: ["a"], prevSet: [], flippedAtMs: T });
+  });
+
+  it("parses the flip timestamp and sorts both lists", () => {
+    const r = buildMintShardResolution({
+      shards: "b,a",
+      prev: "c,a",
+      flippedAt: new Date(T).toISOString(),
+    });
+    expect(r).toEqual({ set: ["a", "b"], prevSet: ["a", "c"], flippedAtMs: T });
+  });
+
+  it("treats an unparseable timestamp as no stamp at all", () => {
+    const r = buildMintShardResolution({ shards: "a", prev: "b", flippedAt: "not-a-date" });
+    expect(r).toEqual({ set: ["a"], prevSet: undefined, flippedAtMs: undefined });
+  });
+});

apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts

@@ -0,0 +1,95 @@
+import type { ShardKey } from "@trigger.dev/core/v3/isomorphic";
+
+// Index 24 of a gen-2 id sits inside the pod name `runner-<id>`, and a DNS-1123 label accepts
+// lowercase only, so the alphabet is 36 keys and no wider. Core keeps its copy private;
+// mintShardGrace.test.ts pins this pattern to generateRunOpsIdV2 instead.
+export const SHARD_KEY_PATTERN = /^[a-z0-9]$/;
+
+// Neither may enter the active set: "new" already means "mint a gen-1 run-ops id" and
+// "legacy" means the cuid store, which minting never selects.
+const RESERVED_SHARD_KEYS: readonly string[] = ["new", "legacy"];
+
+// "new" IS legal as a PIN, holding one org or environment on gen-1 while the rest of the fleet
+// mints gen-2. Without it a non-empty active set moves every environment at once.
+export const GEN_1_PIN_VALUE = "new";
+
+export type MintShardSetResolution = {
+  set: string[];
+  prevSet?: string[];
+  flippedAtMs?: number;
+};
+
+export function isValidPinValue(value: unknown): value is ShardKey {
+  if (typeof value !== "string") return false;
+  return value === GEN_1_PIN_VALUE || SHARD_KEY_PATTERN.test(value);
+}
+
+// Throws rather than dropping a bad key: generateRunOpsIdV2 throws on an out-of-alphabet char,
+// so an unvalidated key must fail at boot and never at mint.
+export function parseShardCsv(raw: string | undefined | null): string[] {
+  const keys = (raw ?? "")
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+
+  const unique = new Set<string>();
+  for (const key of keys) {
+    if (RESERVED_SHARD_KEYS.includes(key)) {
+      throw new Error(`"${key}" is a reserved key and cannot be an active mint shard`);
+    }
+    if (!SHARD_KEY_PATTERN.test(key)) {
+      throw new Error(`invalid shard key "${key}": must be a single char in [a-z0-9]`);
+    }
+    unique.add(key);
+  }
+
+  // Sorted so no placement can depend on the order an operator typed the CSV in.
+  return [...unique].sort();
+}
+
+// Cutover boundary, mirroring effectiveMintKind. `nowMs` is the reader's wall clock while
+// `flippedAtMs` is operator-supplied, so this assumes NTP-synced hosts with skew << graceMs,
+// letting every process cross [flippedAtMs, flippedAtMs + graceMs) together (OLD then NEW).
+export function effectiveMintShardSet(
+  r: MintShardSetResolution,
+  nowMs: number,
+  graceMs: number
+): string[] {
+  if (r.prevSet === undefined || r.flippedAtMs === undefined) {
+    return r.set;
+  }
+  return nowMs < r.flippedAtMs + graceMs ? r.prevSet : r.set;
+}
+
+// A prevSet with no timestamp can never apply, so it is dropped. A timestamp with an EMPTY
+// prevSet is meaningful: it graces a first activation, serving no shards for the window.
+export function buildMintShardResolution(source: {
+  shards: string | undefined;
+  prev: string | undefined;
+  flippedAt: string | undefined;
+}): MintShardSetResolution {
+  const parsed = source.flippedAt !== undefined ? Date.parse(source.flippedAt) : NaN;
+  const flippedAtMs = Number.isNaN(parsed) ? undefined : parsed;
+
+  return {
+    set: parseShardCsv(source.shards),
+    prevSet: flippedAtMs === undefined ? undefined : parseShardCsv(source.prev),
+    flippedAtMs,
+  };
+}
+
+// Returns a message to log when the stamp is half-configured, otherwise undefined. Stays quiet
+// while the active set is empty, so an unconfigured deployment logs nothing at boot.
+export function mintShardStampWarning(source: {
+  shards: string | undefined;
+  prev: string | undefined;
+  flippedAt: string | undefined;
+}): string | undefined {
+  if (parseShardCsv(source.shards).length === 0) {
+    return undefined;
+  }
+  if (parseShardCsv(source.prev).length > 0 && source.flippedAt === undefined) {
+    return "RUN_OPS_MINT_SHARDS_PREV is set but RUN_OPS_MINT_SHARDS_FLIPPED_AT is not; the shard-set grace window will never apply";
+  }
+  return undefined;
+}