diff --git a/specs/automation/inngest-functions-status.t27 b/specs/automation/inngest-functions-status.t27 new file mode 100644 index 000000000..da42deb13 --- /dev/null +++ b/specs/automation/inngest-functions-status.t27 @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// specs/automation/inngest-functions-status.t27 -- the live status of every Inngest function +// of 999-multibots-telegraf, as one JSON document read by the FUNCTIONS tab on t27.ai. +// Host: 999-multibots-telegraf src/inngest_app/status/functionsStatus.ts (aggregation), +// src/inngest_app/status/inngestGraphql.ts (the runs query), +// src/api_server/routes/inngest-status.routes.ts (the route). +// Consumer: trinity apps/website/src/lib/functionsStatus.ts + src/pages/FunctionExplorer.tsx. +// Companion: specs/automation/inngest-probe-suite.t27 (what a probe run is and why it FAILS +// by design). This spec states one thing that spec leaves open: a probe run is NOT a health +// signal, so it must never colour a function red, and it must still be visible. +// phi^2 + 1/phi^2 = 3 | TRINITY + +module automation::inngest_functions_status { + + pub const KIND : str = "automation"; + pub const ID : str = "inngest-functions-status"; + pub const REPO : str = "999-multibots-telegraf"; + pub const VERSION : u8 = 2; + + // ---- Where it is served and where the runs come from ------------------------------- + pub const ROUTE : str = "GET /api/inngest/functions/status"; + pub const AUTH : str = "none"; // read-only, no secrets in the body, CORS open for t27.ai + pub const SOURCE : str = "inngest-gql:runs"; // the same GraphQL the probe suite reads + pub const WINDOW_LONG_H : u8 = 168; // runs7d + pub const WINDOW_SHORT_H : u8 = 24; // runs24h + pub const CACHE_MS : u32 = 30000; // one upstream read per half minute, whoever asks + + // ---- Which runs count as health ------------------------------------------------------ + // A run whose event name starts with INVOKED_EVENT_PREFIX was started by hand: the probe + // suite (/inngest_probe, e2e_test=true) or the Inngest dashboard "Invoke" button. Such a run + // is counted once, under `invoked`, and nowhere else: not in completed, not in failed. + pub const INVOKED_EVENT_PREFIX : str = "inngest/function.invoked"; + pub const INVOKED_COUNTED_AS : str = "invoked"; + pub const INVOKED_IN_COMPLETED : bool = false; + pub const INVOKED_IN_FAILED : bool = false; + + // ---- The two "last" fields (the change of VERSION 2) --------------------------------- + // VERSION 1 put the newest run of any kind into `lastRun`. Seventeen guarded functions FAIL + // on every probe by design (see the probe suite), so after each /inngest_probe the tab + // showed twenty red "last run FAILED" dots for functions that had done nothing wrong. + // VERSION 2 splits the field: + // lastRun = newest run that is NOT invoked (organic: event or cron) -- the health dot + // lastProbe = newest invoked run, with the manifest expectation beside it + // A function that has never run organically in the window has lastRun = null, which the + // consumer shows as "no runs", never as red. + pub const LAST_RUN_EXCLUDES_INVOKED : bool = true; + pub const LAST_PROBE_FIELD : str = "lastProbe"; + pub const LAST_ERROR_EXCLUDES_INVOKED : bool = true; + + // lastProbe.asExpected is judged from the run STATUS against `probe_expect` in the manifest. + // It is a status-level check ("FAILED-at-guard" -> FAILED, "COMPLETED" -> COMPLETED); the + // step-level verdict (did it fail AT THE GUARD) is the probe suite's job, not this route's. + pub const PROBE_EXPECT_SOURCE : str = "functions.manifest.json:probe_expect"; + pub const PROBE_JUDGE_LEVEL : str = "status"; + pub const PROBE_EXPECT_FAILED_AT_GUARD_MEANS : str = "FAILED"; + pub const PROBE_EXPECT_COMPLETED_MEANS : str = "COMPLETED"; + pub const PROBE_EXPECT_SKIP_MEANS : str = ""; // no expectation: asExpected is null + + // ---- What the consumer may conclude -------------------------------------------------- + pub const HEALTH_DOT_SOURCE : str = "lastRun.status"; + pub const PROBE_MARK_SOURCE : str = "lastProbe.asExpected"; + pub const NO_ORGANIC_RUN_TONE : str = "muted"; // not green, not red + pub const UNEXPECTED_PROBE_TONE : str = "warn"; // a probe that did not end as the manifest says + + // ---- Honesty --------------------------------------------------------------------------- + // Witness that motivated VERSION 2: production status read 2026-09-12T14:01Z, 28 functions, + // 20 with lastRun.status = FAILED, all 20 from probe runs of 2026-09-09/10; 22 of 28 had no + // organic run in the 7-day window at all. That reading is VERSION 1 output; VERSION 2 is + // written here before the host implements it. + pub const WITNESS_V1_READ_AT : str = "2026-09-12T14:01Z"; + pub const WITNESS_V1_FUNCTIONS : u8 = 28; + pub const WITNESS_V1_RED_FROM_PROBES : u8 = 20; + pub const WITNESS_V1_NO_ORGANIC_RUN_7D : u8 = 22; + + struct RunRef { + id: str, + status: str, // Inngest run status + queued_at: str, + ended_at: str, // "" while running + } + + struct LastProbe { + id: str, + status: str, + ended_at: str, + expect: str, // one of the probe suite EXPECTATIONS, "" when the manifest has none + as_expected: bool, // meaningful only when expect != "" + } + + struct FunctionHealth { + id: str, + has_last_run: bool, + last_run_status: str, // "" when has_last_run is false + has_last_probe: bool, + probe_as_expected: bool, + tone: str, // "ok" | "bad" | "muted" + } + + test "a failed probe does not colour the function red" + const f = FunctionHealth { id: "neuro-image-generate", has_last_run: false, last_run_status: "", has_last_probe: true, probe_as_expected: true, tone: "muted" }; + assert(!f.has_last_run); + assert(f.has_last_probe); + assert(f.probe_as_expected); + assert(f.tone == "muted"); + + test "an organic failure is red even when the last probe was as expected" + const f = FunctionHealth { id: "monitoring-health-check", has_last_run: true, last_run_status: "FAILED", has_last_probe: true, probe_as_expected: true, tone: "bad" }; + assert(f.has_last_run); + assert(f.last_run_status == "FAILED"); + assert(f.tone == "bad"); + + test "a guarded function that completed on a probe is flagged, not hidden" + const p = LastProbe { id: "r3", status: "COMPLETED", ended_at: "2026-09-10T03:19:18Z", expect: "FAILED-at-guard", as_expected: false }; + assert(p.expect == "FAILED-at-guard"); + assert(p.status == "COMPLETED"); + assert(!p.as_expected); +} diff --git a/specs/automation/inngest-probe-suite.t27 b/specs/automation/inngest-probe-suite.t27 index 0b2f5a5c5..7239be3dc 100644 --- a/specs/automation/inngest-probe-suite.t27 +++ b/specs/automation/inngest-probe-suite.t27 @@ -14,7 +14,7 @@ module automation::inngest_probe_suite { pub const KIND : str = "automation"; pub const ID : str = "inngest-probe-suite"; pub const REPO : str = "999-multibots-telegraf"; - pub const VERSION : u8 = 1; + pub const VERSION : u8 = 2; // 2: LIVE_RUN_STATUS witnessed; status route split, see inngest-functions-status.t27 // How the suite is started and by whom. pub const COMMAND : str = "/inngest_probe"; @@ -60,11 +60,32 @@ module automation::inngest_probe_suite { pub const EXPECT_FAILED_AT_GUARD_AT_251571C : u8 = 17; pub const EXPECT_COMPLETED_AT_251571C : u8 = 11; - // Honesty. The suite has not been run against production from this spec's commit: the - // GraphQL endpoint is private to the Railway network, so the first real run happens - // after deploy, from the admin chat. The unit tests drive the orchestrator with a fake - // Inngest (src/__tests__/inngest/probeSuite.test.ts, src/__tests__/bot/inngestProbeCommand.test.ts). - pub const LIVE_RUN_STATUS : str = "not-yet-run"; + // Honesty. VERSION 1 said "not-yet-run". Three production runs exist since. The invoked + // runs of all three are visible in the production Inngest GraphQL (read 2026-09-12); the + // verdict counts below are as recorded by the host in docs/inngest/probe-suite.md and + // docs/inngest/witness/2026-09-10-probe-suite-03-19Z.json, not re-derived here. + // 2026-09-09T19:11Z from the admin chat; the judge of that build reported 17 mismatch, + // later attributed to the judge reading Attempt 0 only + // 2026-09-10T02:17Z mirror run right after a deploy: 21 skipped with a FAILED run id + // (two judge defects, fixed in host PR #2331) + // 2026-09-10T03:19Z runProbeSuite from host main 224a87f against the production GraphQL, + // from an operator sandbox: 28 invoked, 28 match, 0 mismatch, + // 0 timeout, 0 invoke-error; 03:19:17Z - 03:20:05Z (the witness file) + // What is still not witnessed: the Telegram path of the merged judge (/inngest_probe -> + // button -> report inside the deployed bot process). Unit tests drive the orchestrator with + // a fake Inngest (src/__tests__/inngest/probeSuite.test.ts, src/__tests__/bot/inngestProbeCommand.test.ts). + pub const LIVE_RUN_STATUS : str = "witnessed-2026-09-10T03:19Z"; + pub const LIVE_RUNS : u8 = 3; + pub const LIVE_LAST_MATCH : u8 = 28; + pub const LIVE_LAST_MISMATCH : u8 = 0; + pub const TELEGRAM_PATH_WITNESSED_WITH_MERGED_JUDGE : bool = false; + + // A probe run that FAILED at its guard is the good outcome here and a red dot nowhere: + // the status route keeps such runs out of lastRun and out of the failed counters + // (specs/automation/inngest-functions-status.t27). The onFailure handler of every function + // recognises a probe event and sends no admin alert (host: isProbeFailureEvent in client.ts). + pub const PROBE_RUN_IS_HEALTH_SIGNAL : bool = false; + pub const PROBE_FAILURE_ALERTS_ADMIN : bool = false; struct ProbePlan { id: str,