gHashTag · gHashTag · Sep 13, 2026
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: Apache-2.0
+// specs/automation/inngest-functions-status.t27 -- the live status of every Inngest function
+// of 999-multibots-telegraf, as one JSON document read by the FUNCTIONS tab on t27.ai.
+// Host: 999-multibots-telegraf src/inngest_app/status/functionsStatus.ts (aggregation),
+//       src/inngest_app/status/inngestGraphql.ts (the runs query),
+//       src/api_server/routes/inngest-status.routes.ts (the route).
+// Consumer: trinity apps/website/src/lib/functionsStatus.ts + src/pages/FunctionExplorer.tsx.
+// Companion: specs/automation/inngest-probe-suite.t27 (what a probe run is and why it FAILS
+// by design). This spec states one thing that spec leaves open: a probe run is NOT a health
+// signal, so it must never colour a function red, and it must still be visible.
+// phi^2 + 1/phi^2 = 3 | TRINITY
+
+module automation::inngest_functions_status {
+
+    pub const KIND : str = "automation";
+    pub const ID : str = "inngest-functions-status";
+    pub const REPO : str = "999-multibots-telegraf";
+    pub const VERSION : u8 = 2;
+
+    // ---- Where it is served and where the runs come from -------------------------------
+    pub const ROUTE : str = "GET /api/inngest/functions/status";
+    pub const AUTH : str = "none";  // read-only, no secrets in the body, CORS open for t27.ai
+    pub const SOURCE : str = "inngest-gql:runs";  // the same GraphQL the probe suite reads
+    pub const WINDOW_LONG_H : u8 = 168;  // runs7d
+    pub const WINDOW_SHORT_H : u8 = 24;  // runs24h
+    pub const CACHE_MS : u32 = 30000;  // one upstream read per half minute, whoever asks
+
+    // ---- Which runs count as health ------------------------------------------------------
+    // A run whose event name starts with INVOKED_EVENT_PREFIX was started by hand: the probe
+    // suite (/inngest_probe, e2e_test=true) or the Inngest dashboard "Invoke" button. Such a run
+    // is counted once, under `invoked`, and nowhere else: not in completed, not in failed.
+    pub const INVOKED_EVENT_PREFIX : str = "inngest/function.invoked";
+    pub const INVOKED_COUNTED_AS : str = "invoked";
+    pub const INVOKED_IN_COMPLETED : bool = false;
+    pub const INVOKED_IN_FAILED : bool = false;
+
+    // ---- The two "last" fields (the change of VERSION 2) ---------------------------------
+    // VERSION 1 put the newest run of any kind into `lastRun`. Seventeen guarded functions FAIL
+    // on every probe by design (see the probe suite), so after each /inngest_probe the tab
+    // showed twenty red "last run FAILED" dots for functions that had done nothing wrong.
+    // VERSION 2 splits the field:
+    //   lastRun   = newest run that is NOT invoked (organic: event or cron)  -- the health dot
+    //   lastProbe = newest invoked run, with the manifest expectation beside it
+    // A function that has never run organically in the window has lastRun = null, which the
+    // consumer shows as "no runs", never as red.
+    pub const LAST_RUN_EXCLUDES_INVOKED : bool = true;
+    pub const LAST_PROBE_FIELD : str = "lastProbe";
+    pub const LAST_ERROR_EXCLUDES_INVOKED : bool = true;
+
+    // lastProbe.asExpected is judged from the run STATUS against `probe_expect` in the manifest.
+    // It is a status-level check ("FAILED-at-guard" -> FAILED, "COMPLETED" -> COMPLETED); the
+    // step-level verdict (did it fail AT THE GUARD) is the probe suite's job, not this route's.
+    pub const PROBE_EXPECT_SOURCE : str = "functions.manifest.json:probe_expect";
+    pub const PROBE_JUDGE_LEVEL : str = "status";
+    pub const PROBE_EXPECT_FAILED_AT_GUARD_MEANS : str = "FAILED";
+    pub const PROBE_EXPECT_COMPLETED_MEANS : str = "COMPLETED";
+    pub const PROBE_EXPECT_SKIP_MEANS : str = "";  // no expectation: asExpected is null
+
+    // ---- What the consumer may conclude --------------------------------------------------
+    pub const HEALTH_DOT_SOURCE : str = "lastRun.status";
+    pub const PROBE_MARK_SOURCE : str = "lastProbe.asExpected";
+    pub const NO_ORGANIC_RUN_TONE : str = "muted";  // not green, not red
+    pub const UNEXPECTED_PROBE_TONE : str = "warn";  // a probe that did not end as the manifest says
+
+    // ---- Honesty ---------------------------------------------------------------------------
+    // Witness that motivated VERSION 2: production status read 2026-09-12T14:01Z, 28 functions,
+    // 20 with lastRun.status = FAILED, all 20 from probe runs of 2026-09-09/10; 22 of 28 had no
+    // organic run in the 7-day window at all. That reading is VERSION 1 output; VERSION 2 is
+    // written here before the host implements it.
+    pub const WITNESS_V1_READ_AT : str = "2026-09-12T14:01Z";
+    pub const WITNESS_V1_FUNCTIONS : u8 = 28;
+    pub const WITNESS_V1_RED_FROM_PROBES : u8 = 20;
+    pub const WITNESS_V1_NO_ORGANIC_RUN_7D : u8 = 22;
+
+    struct RunRef {
+        id: str,
+        status: str,  // Inngest run status
+        queued_at: str,
+        ended_at: str,  // "" while running
+    }
+
+    struct LastProbe {
+        id: str,
+        status: str,
+        ended_at: str,
+        expect: str,  // one of the probe suite EXPECTATIONS, "" when the manifest has none
+        as_expected: bool,  // meaningful only when expect != ""
+    }
+
+    struct FunctionHealth {
+        id: str,
+        has_last_run: bool,
+        last_run_status: str,  // "" when has_last_run is false
+        has_last_probe: bool,
+        probe_as_expected: bool,
+        tone: str,  // "ok" | "bad" | "muted"
+    }
+
+    test "a failed probe does not colour the function red"
+        const f = FunctionHealth { id: "neuro-image-generate", has_last_run: false, last_run_status: "", has_last_probe: true, probe_as_expected: true, tone: "muted" };
+        assert(!f.has_last_run);
+        assert(f.has_last_probe);
+        assert(f.probe_as_expected);
+        assert(f.tone == "muted");
+
+    test "an organic failure is red even when the last probe was as expected"
+        const f = FunctionHealth { id: "monitoring-health-check", has_last_run: true, last_run_status: "FAILED", has_last_probe: true, probe_as_expected: true, tone: "bad" };
+        assert(f.has_last_run);
+        assert(f.last_run_status == "FAILED");
+        assert(f.tone == "bad");
+
+    test "a guarded function that completed on a probe is flagged, not hidden"
+        const p = LastProbe { id: "r3", status: "COMPLETED", ended_at: "2026-09-10T03:19:18Z", expect: "FAILED-at-guard", as_expected: false };
+        assert(p.expect == "FAILED-at-guard");
+        assert(p.status == "COMPLETED");
+        assert(!p.as_expected);
+}
@@ -14,7 +14,7 @@ module automation::inngest_probe_suite {
     pub const KIND : str = "automation";
     pub const ID : str = "inngest-probe-suite";
     pub const REPO : str = "999-multibots-telegraf";
-    pub const VERSION : u8 = 1;
+    pub const VERSION : u8 = 2;  // 2: LIVE_RUN_STATUS witnessed; status route split, see inngest-functions-status.t27
 
     // How the suite is started and by whom.
     pub const COMMAND : str = "/inngest_probe";
@@ -60,11 +60,32 @@ module automation::inngest_probe_suite {
     pub const EXPECT_FAILED_AT_GUARD_AT_251571C : u8 = 17;
     pub const EXPECT_COMPLETED_AT_251571C : u8 = 11;
 
-    // Honesty. The suite has not been run against production from this spec's commit: the
-    // GraphQL endpoint is private to the Railway network, so the first real run happens
-    // after deploy, from the admin chat. The unit tests drive the orchestrator with a fake
-    // Inngest (src/__tests__/inngest/probeSuite.test.ts, src/__tests__/bot/inngestProbeCommand.test.ts).
-    pub const LIVE_RUN_STATUS : str = "not-yet-run";
+    // Honesty. VERSION 1 said "not-yet-run". Three production runs exist since. The invoked
+    // runs of all three are visible in the production Inngest GraphQL (read 2026-09-12); the
+    // verdict counts below are as recorded by the host in docs/inngest/probe-suite.md and
+    // docs/inngest/witness/2026-09-10-probe-suite-03-19Z.json, not re-derived here.
+    //   2026-09-09T19:11Z  from the admin chat; the judge of that build reported 17 mismatch,
+    //                      later attributed to the judge reading Attempt 0 only
+    //   2026-09-10T02:17Z  mirror run right after a deploy: 21 skipped with a FAILED run id
+    //                      (two judge defects, fixed in host PR #2331)
+    //   2026-09-10T03:19Z  runProbeSuite from host main 224a87f against the production GraphQL,
+    //                      from an operator sandbox: 28 invoked, 28 match, 0 mismatch,
+    //                      0 timeout, 0 invoke-error; 03:19:17Z - 03:20:05Z (the witness file)
+    // What is still not witnessed: the Telegram path of the merged judge (/inngest_probe ->
+    // button -> report inside the deployed bot process). Unit tests drive the orchestrator with
+    // a fake Inngest (src/__tests__/inngest/probeSuite.test.ts, src/__tests__/bot/inngestProbeCommand.test.ts).
+    pub const LIVE_RUN_STATUS : str = "witnessed-2026-09-10T03:19Z";
+    pub const LIVE_RUNS : u8 = 3;
+    pub const LIVE_LAST_MATCH : u8 = 28;
+    pub const LIVE_LAST_MISMATCH : u8 = 0;
+    pub const TELEGRAM_PATH_WITNESSED_WITH_MERGED_JUDGE : bool = false;
+
+    // A probe run that FAILED at its guard is the good outcome here and a red dot nowhere:
+    // the status route keeps such runs out of lastRun and out of the failed counters
+    // (specs/automation/inngest-functions-status.t27). The onFailure handler of every function
+    // recognises a probe event and sends no admin alert (host: isProbeFailureEvent in client.ts).
+    pub const PROBE_RUN_IS_HEALTH_SIGNAL : bool = false;
+    pub const PROBE_FAILURE_ALERTS_ADMIN : bool = false;
 
     struct ProbePlan {
         id: str,