From 3abd89ee1c46ef89e7e2de5efa1d6025d3088a8e Mon Sep 17 00:00:00 2001 From: Alexey Grigorev Date: Fri, 20 Nov 2026 13:52:40 +0100 Subject: [PATCH 1/3] fix(engine): rehydrate a recovered branch step's children on redelivery A redelivered event restores a completed condition/paths step from run history as a whole, so its children never re-entered the steps context under their nested ids. dev-mailbox's verify-delivery then read route.else.post-webhook as missing and failed every redelivery even though the webhook leg had been delivered on the first attempt (runs 91acff66, 9143bacc, 08674c39 on 18-20 Nov). Re-enter the recorded branch (no predicate re-evaluation): completed children restore their own records, unfinished ones run again, like any unfinished top-level step on a redelivery. --- src/dapier/engine/logic_pkg/execution.py | 52 ++++++++++++ tests/test_branch_recovery.py | 104 +++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 tests/test_branch_recovery.py diff --git a/src/dapier/engine/logic_pkg/execution.py b/src/dapier/engine/logic_pkg/execution.py index bbb52f8e..f097d92a 100644 --- a/src/dapier/engine/logic_pkg/execution.py +++ b/src/dapier/engine/logic_pkg/execution.py @@ -55,6 +55,11 @@ def _run_step(workflow_id, step, index, event, run_action, *, step_outputs[action_id]["secret"] = True if gate.status == "filtered": return "filtered" + return _rehydrate_branch( + workflow_id, step, action_id, gate.output, event, run_action, + before_action=before_action, after_action=after_action, + on_action_error=on_action_error, scope=scope, + step_outputs=step_outputs) else: step_outputs[action_id] = {"status": "skipped"} return None @@ -146,6 +151,53 @@ def _run_step(workflow_id, step, index, event, run_action, *, return stop +def _recorded_branch(step, output): + """The (steps, id suffix) of the branch a recovered condition/paths step + took, read from its persisted output; None when it took none.""" + if not isinstance(output, dict): + return None + branch = output.get("branch") + step_type = str(step.get("type")) + if step_type == "condition" and branch in ("then", "else"): + chain = step.get(branch) + elif step_type == "paths" and branch == "default": + chain = step.get("default") + elif step_type == "paths" and branch: + chain = next((path.get("actions") for path in step.get("paths") or [] + if isinstance(path, dict) + and (str(path.get("label") or "").strip() or "path") == branch), None) + else: + return None + return (chain, branch) if isinstance(chain, list) and chain else None + + +def _rehydrate_branch(workflow_id, step, action_id, output, event, run_action, *, + before_action, after_action, on_action_error, scope=None, + step_outputs=None): + """Re-enter the branch a recovered condition/paths step recorded. + + A redelivered event recovers a completed branching step from run history + as a whole, but later steps read the branch's children by their nested + ids (``route.else.post-webhook``). Running the recorded branch again — + without re-evaluating the predicate — puts each child through the same + gate as a top-level step: completed children restore their own records, + and children that failed or were skipped run again, like any unfinished + top-level step on a redelivery. + """ + recorded = _recorded_branch(step, output) + if recorded is None: + return None + chain, branch = recorded + from .chains import run_chain # lazy: chains imports this module at its top + + return run_chain( + workflow_id, chain, event, run_action, + before_action=before_action, after_action=after_action, + on_action_error=on_action_error, prefix=f"{action_id}.{branch}", + scope=scope, step_outputs=step_outputs, + ) + + def _handle_error(workflow_id, step, action_id, exc, event, run_action, *, before_action, after_action, on_action_error, scope=None, step_outputs=None, duration_ms=None): diff --git a/tests/test_branch_recovery.py b/tests/test_branch_recovery.py new file mode 100644 index 00000000..d898fb69 --- /dev/null +++ b/tests/test_branch_recovery.py @@ -0,0 +1,104 @@ +"""A redelivered event recovers a completed branching step from run history; +its branch children must come back under their nested ids too. + +Regression for the 2026-11-19/20 dev-mailbox failures: on each queue +redelivery the ``route`` condition was restored whole, ``route.else.post-webhook`` +never re-entered ``steps``, and verify-delivery failed the run as if the +webhook leg had not been delivered. +""" +from src.dapier.engine.logic_pkg import CompletedStep, run_chain + + +def _gate(records): + """A worker-style step gate over recorded step statuses: completed steps + are restored from history, anything else runs (True).""" + def before_action(workflow_id, action_id, event, action_type=None): + record = records.get(action_id) + if record and record[0] in ("completed", "filtered"): + return CompletedStep(*record) + return True + return before_action + + +ROUTE = { + "id": "route", "type": "condition", + "when": {"steps.decide.output.route": {"equals": "hub"}}, + "then": [{"id": "notify", "type": "http_request"}], + "else": [{"id": "post-webhook", "type": "http_request"}, + {"id": "post-copy", "type": "http_request"}], +} + + +def test_recovered_condition_restores_completed_children_without_rerunning(): + records = { + "route": ("completed", {"branch": "else", "condition": "failed"}), + "route.else.post-webhook": ("completed", {"status": 202}), + "route.else.post-copy": ("completed", {"status": 200}), + } + calls = [] + + def dispatch(step, event, workflow_id, steps=None): + calls.append(step["id"]) + return {"status": 500} + + outputs = {} + run_chain("wf", [ROUTE], {"id": "evt", "data": {}}, dispatch, + before_action=_gate(records), step_outputs=outputs) + assert calls == [] + assert outputs["route"]["output"]["branch"] == "else" + assert outputs["route.else.post-webhook"] == {"status": "completed", + "output": {"status": 202}} + assert outputs["route.else.post-copy"]["status"] == "completed" + assert "route.then.notify" not in outputs + + +def test_recovered_condition_reruns_unfinished_children_on_the_recorded_branch(): + # The predicate would now pick "then" (decide is absent), but the + # recorded branch wins: the run stays on the path it already took. + records = { + "route": ("completed", {"branch": "else", "condition": "failed"}), + "route.else.post-webhook": ("completed", {"status": 202}), + "route.else.post-copy": ("failed", {}), + } + calls = [] + + def dispatch(step, event, workflow_id, steps=None): + calls.append(step["id"]) + return {"status": 200} + + outputs = {} + run_chain("wf", [ROUTE], {"id": "evt", "data": {}}, dispatch, + before_action=_gate(records), step_outputs=outputs) + assert calls == ["post-copy"] + assert outputs["route.else.post-copy"] == {"status": "completed", + "output": {"status": 200}} + + +def test_recovered_paths_step_restores_its_recorded_path_children(): + step = { + "id": "pick", "type": "paths", + "paths": [{"label": "urgent", "when": {"subject": {"contains": "!"}}, + "actions": [{"id": "page", "type": "http_request"}]}], + "default": [{"id": "log", "type": "http_request"}], + } + records = { + "pick": ("completed", {"branch": "urgent", "matched": "urgent"}), + "pick.urgent.page": ("completed", {"status": 202}), + } + outputs = {} + run_chain("wf", [step], {"id": "evt", "data": {}}, + lambda *args, **kwargs: {"status": 500}, + before_action=_gate(records), step_outputs=outputs) + assert outputs["pick.urgent.page"]["output"] == {"status": 202} + assert "pick.default.log" not in outputs + + +def test_recovered_condition_without_a_recorded_branch_runs_nothing(): + records = {"route": ("completed", {"result": {}})} + calls = [] + outputs = {} + run_chain("wf", [ROUTE], {"id": "evt", "data": {}}, + lambda step, *args, **kwargs: calls.append(step["id"]) or {}, + before_action=_gate(records), step_outputs=outputs) + assert calls == [] + assert list(outputs) == ["route"] From 00271bac493e351115b0858f2464793c7958d1a9 Mon Sep 17 00:00:00 2001 From: Alexey Grigorev Date: Fri, 20 Nov 2026 13:52:54 +0100 Subject: [PATCH 2/3] fix(mailbox): count oncall 409 as delivered; retry before queue redelivery Every dev-mailbox failure on 18-20 Nov had the same shape: Hub's oncall ingest stored the first attempt but the response timed out; each queue redelivery then resent the format-email output recovered from run history (DynamoDB map, key order lost), so Hub answered 409 delivery_id_conflict for the same X-Delivery-Id forever. The run looped through five 1800s redeliveries into the DLQ, and every redelivery tripped the 900s EventQueueAgeAlarm (all four alarms of 19-20 Nov). verify-delivery now treats an ingest 409 as already delivered (the key is the message id's hash, so Hub holds this email). A retry policy (4 attempts, 120s) re-enqueues a failed leg as a fresh message instead of waiting out the visibility timeout, so a transient Hub outage no longer ages the queue past the alarm threshold. --- tests/test_dev_mailbox_workflow.py | 33 ++++++++++++++++++++++++++++++ workflows/dev-mailbox.yaml | 20 ++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/test_dev_mailbox_workflow.py b/tests/test_dev_mailbox_workflow.py index 282aaf42..bedeb46b 100644 --- a/tests/test_dev_mailbox_workflow.py +++ b/tests/test_dev_mailbox_workflow.py @@ -490,6 +490,39 @@ def test_dev_mailbox_delivery_verifier_requires_ingest_and_routed_leg(): assert result["oncall_ingest"] == "completed" +def test_dev_mailbox_verifier_counts_oncall_409_as_already_delivered(): + """A retry whose first attempt reached Hub gets 409 delivery_id_conflict + (same X-Delivery-Id, different body digest): the email is already in + Hub's queue, so the run must not fail into 1800s queue redeliveries.""" + _, wf = _load("dev-mailbox") + verify_action = next(a for a in wf["actions"] if a.get("id") == "verify-delivery") + completed = {"status": "completed", "output": {}} + steps = { + "route": completed | {"output": {"branch": "else"}}, + "route.else.post-webhook": completed, + "post-oncall-ingest": {"status": "failed", "output": {}, + "error": "http_request returned HTTP 409"}, + } + result = run_code(verify_action, {"connector": "email", "data": {}}, steps=steps)["result"] + assert result["oncall_ingest"] == "already_delivered" + # Any other ingest failure still fails the run. + steps["post-oncall-ingest"] = {"status": "failed", "output": {}, + "error": "The read operation timed out"} + with pytest.raises(RuntimeError, match="post-oncall-ingest"): + run_code(verify_action, {"connector": "email", "data": {}}, steps=steps) + + +def test_dev_mailbox_retries_failed_delivery_before_queue_redelivery(): + from src.dapier.engine.worker import retry_policy + + _, wf = _load("dev-mailbox") + parse_workflow(yaml.safe_dump(wf)) + policy = retry_policy(wf) + assert policy["attempts"] >= 2 + # Each retry must re-enqueue well under the 900s EventQueueAgeAlarm. + assert policy["backoff_seconds"] < 900 + + @pytest.mark.parametrize("token_value", [ "secret-token", "Bearer secret-token", diff --git a/workflows/dev-mailbox.yaml b/workflows/dev-mailbox.yaml index 07219fca..695d2385 100644 --- a/workflows/dev-mailbox.yaml +++ b/workflows/dev-mailbox.yaml @@ -1,6 +1,13 @@ id: dev-mailbox enabled: true description: Developer mailbox dev@dtcdev.click -> Developer bot webhook + direct oncall ingest, or Dev sticky Hub session when GrokBot remaining usage < 1% +# A failed delivery leg (Hub briefly unreachable) retries as a fresh queue +# message two minutes later. Without a policy the event waits out the event +# queue's 1800s visibility timeout per redelivery, which trips the 900s +# EventQueueAgeAlarm on every transient outage. +retry: + attempts: 4 + backoff_seconds: 120 triggers: - connector: email event: message.received @@ -310,11 +317,20 @@ actions: failed.append("route") elif (steps.get(leg) or {}).get("status") != "completed": failed.append(leg) - if (steps.get("post-oncall-ingest") or {}).get("status") != "completed": + # Hub answers 409 delivery_id_conflict when it already holds a + # delivery under this X-Delivery-Id (sha256 of the message id) and the + # body digest differs. That happens on a retry: the first attempt + # reached Hub but its response timed out, and the retry resends the + # format-email output recovered from run history, whose key order is + # not preserved. The email is already in Hub's queue, so it counts. + ingest = steps.get("post-oncall-ingest") or {} + ingest_duplicate = (ingest.get("status") == "failed" + and "HTTP 409" in str(ingest.get("error") or "")) + if ingest.get("status") != "completed" and not ingest_duplicate: failed.append("post-oncall-ingest") if failed: raise ValueError("dev-mailbox delivery failed for: " + ", ".join(failed)) output = {"routed_to": route_output.get("branch"), "decide_reason": ((steps.get("decide-grok-route") or {}).get("output") or {}).get("result", {}).get("reason", ""), - "oncall_ingest": "completed"} + "oncall_ingest": "already_delivered" if ingest_duplicate else "completed"} From 1e05b0aac9942c113da547668fbed0f6b97890a5 Mon Sep 17 00:00:00 2001 From: Alexey Grigorev Date: Fri, 20 Nov 2026 13:52:54 +0100 Subject: [PATCH 3/3] ci(alarms): print alarm states, transitions, queue age and DLQ depth The deploy and the dispatchable inspection now report the event-queue alarms and the client-check alarm the operator topic pages on: current state, verbatim transitions over three days, oldest-message age peaks, dead-letter depth and ClientCheckFailed counts. An alarm email only says a threshold was crossed; these lines prove recovery and non-recurrence. Read-only, within the existing inspection grants (pinned by a test). --- .github/workflows/backup-inspection.yml | 3 + .github/workflows/deploy.yml | 6 ++ scripts/inspect_alarms.py | 115 ++++++++++++++++++++++++ tests/test_backup.py | 28 ++++++ 4 files changed, 152 insertions(+) create mode 100644 scripts/inspect_alarms.py diff --git a/.github/workflows/backup-inspection.yml b/.github/workflows/backup-inspection.yml index d139ba3a..cf4249f3 100644 --- a/.github/workflows/backup-inspection.yml +++ b/.github/workflows/backup-inspection.yml @@ -46,3 +46,6 @@ jobs: args="--invoke" fi uv run --with boto3 python scripts/inspect_backup.py $args + - name: Inspect event-queue and operator alarms + if: always() + run: uv run --with boto3 python scripts/inspect_alarms.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d06cb62e..591401b0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -208,5 +208,11 @@ jobs: # the next 03:00Z run). continue-on-error: true run: uv run --with boto3 python scripts/inspect_backup.py + - name: Inspect event-queue and operator alarms (read-only) + # Alarm states, transitions, queue-age peaks, DLQ depth: proof that + # a paged alarm recovered and has not come back. Same grants as the + # backup inspection (infra/backup-inspection-access.yaml). + continue-on-error: true + run: uv run --with boto3 python scripts/inspect_alarms.py - name: Inspect YouTube renewal infrastructure (read-only) run: bash scripts/inspect-youtube-renewal.sh diff --git a/scripts/inspect_alarms.py b/scripts/inspect_alarms.py new file mode 100644 index 00000000..bb047c62 --- /dev/null +++ b/scripts/inspect_alarms.py @@ -0,0 +1,115 @@ +"""Inspect the event-queue and operator alarms (read-only). + +The deploy workflow and the dispatchable backup-inspection workflow run this +after the backup checks. It prints ALARMS_* evidence lines: each alarm's +current state, its recent state transitions (the verbatim "Threshold +Crossed" datapoints), the event queue's oldest-message age peaks, the +dead-letter queue depth, and the browser client-check failure counts the +operator topic also pages on. An alarm email says when a threshold was +crossed; these lines say whether it recovered and whether it came back. + +Uses only the cloudformation/cloudwatch grants in +infra/backup-inspection-access.yaml (tests/test_backup.py keeps the two in +lockstep); a missing grant prints ALARMS_UNVERIFIED instead of failing. +""" +import json +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone + +import boto3 +from botocore.exceptions import ClientError + +# Logical alarm ids in the dapier stack, plus alarms other stacks publish to +# the same operator topic by name. +STACK_ALARMS = ('EventQueueAgeAlarm', 'EventQueueDepthAlarm', 'EventDeadLetterAlarm') +NAMED_ALARMS = ('dataops-v1-client-check-failed',) +HISTORY_DAYS = 3 + + +@contextmanager +def inspect(label): + try: + yield + except ClientError as exc: + print('ALARMS_UNVERIFIED', label, exc.response['Error']['Code']) + + +cf = boto3.client('cloudformation') +cw = boto3.client('cloudwatch') + + +def physical(logical): + return cf.describe_stack_resource(StackName='dapier', LogicalResourceId=logical)['StackResourceDetail']['PhysicalResourceId'] + + +def queue_name(logical): + # An SQS queue's physical id is its URL; metrics key on the name. + return physical(logical).rsplit('/', 1)[-1] + + +def peaks(namespace, metric, dimensions, statistic, period, floor, days=HISTORY_DAYS): + """Datapoints at or above `floor` over the last `days`, oldest first. + GetMetricStatistics caps one call at 1440 datapoints, so day windows.""" + now = datetime.now(timezone.utc) + points = [] + for offset in range(days): + end = now - timedelta(days=offset) + points.extend( + (str(point['Timestamp']), point[statistic]) + for point in cw.get_metric_statistics( + Namespace=namespace, MetricName=metric, Dimensions=dimensions, + StartTime=end - timedelta(days=1), EndTime=end, Period=period, + Statistics=[statistic])['Datapoints'] + if point[statistic] >= floor) + return sorted(points) + + +def history(name): + since = datetime.now(timezone.utc) - timedelta(days=HISTORY_DAYS) + items = [] + kwargs = {'AlarmName': name, 'HistoryItemType': 'StateUpdate', + 'StartDate': since, 'MaxRecords': 100} + while True: + page = cw.describe_alarm_history(**kwargs) + items.extend(page.get('AlarmHistoryItems', [])) + if not page.get('NextToken'): + break + kwargs['NextToken'] = page['NextToken'] + return [{'timestamp': str(item.get('Timestamp')), 'summary': item.get('HistorySummary')} + for item in sorted(items, key=lambda item: item.get('Timestamp'))] + + +def main(): + names = [] + for logical in STACK_ALARMS: + with inspect(f'resolve {logical}'): + names.append(physical(logical)) + names.extend(NAMED_ALARMS) + with inspect('alarm-states'): + found = cw.describe_alarms(AlarmNames=names)['MetricAlarms'] + print('ALARMS_STATE', json.dumps([ + {k: alarm.get(k) for k in ('AlarmName', 'StateValue', 'StateUpdatedTimestamp', 'StateReason')} + for alarm in found], default=str)) + missing = sorted(set(names) - {alarm['AlarmName'] for alarm in found}) + if missing: + print('ALARMS_NOT_VISIBLE', json.dumps(missing)) + for name in names: + with inspect(f'history {name}'): + print('ALARMS_HISTORY', json.dumps({'alarm': name, 'items': history(name)})) + with inspect('event-queue-age'): + # Five-minute maxima of the oldest message's age, only those past + # five minutes: an empty list is a queue that never backed up. + queue = [{'Name': 'QueueName', 'Value': queue_name('EventQueue')}] + print('ALARMS_QUEUE_AGE', json.dumps({'over_300s': peaks( + 'AWS/SQS', 'ApproximateAgeOfOldestMessage', queue, 'Maximum', 300, 300)})) + with inspect('dead-letter-depth'): + dlq = [{'Name': 'QueueName', 'Value': queue_name('DeadLetterQueue')}] + print('ALARMS_DLQ_VISIBLE', json.dumps({'hours_nonzero': peaks( + 'AWS/SQS', 'ApproximateNumberOfMessagesVisible', dlq, 'Maximum', 3600, 1)})) + with inspect('client-check-failures'): + print('ALARMS_CLIENT_CHECK_FAILED', json.dumps({'five_minute_sums': peaks( + 'DataOps/Client', 'ClientCheckFailed', [], 'Sum', 300, 1)})) + + +if __name__ == '__main__': + main() diff --git a/tests/test_backup.py b/tests/test_backup.py index fcbd2e28..cb0f92cc 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -613,3 +613,31 @@ def test_backup_failure_email_routes_only_to_dev_inbox(): # The backup recipient parameter itself must never default to a personal inbox. assert "alexey@datatalks.club" not in template["Parameters"]["BackupAlertEmail"]["Default"] assert "alexey@datatalks.club" not in Path("src/dapier/backup.py").read_text() + + +def test_alarm_inspection_stays_within_inspection_grants(): + """scripts/inspect_alarms.py runs in the same deploy and dispatch jobs on + the same role: every boto3 call it makes must already be granted, and it + must run read-only in both workflows.""" + import re + + script = Path("scripts/inspect_alarms.py").read_text() + clients = dict(re.findall(r"^(\w+) = boto3\.client\('([^']+)'\)\s*$", script, re.MULTILINE)) + calls = {(service, method) + for var, service in clients.items() + for method in re.findall(rf"\b{var}\.(\w+)\(", script)} + assert calls, "no boto3 calls parsed" + infra = _load_yaml("infra/backup-inspection-access.yaml") + policy = infra["Resources"]["GithubDeployBackupInspectionAccess"]["Properties"] + granted = {action + for statement in policy["PolicyDocument"]["Statement"] + for action in statement["Action"]} + for service, method in sorted(calls): + assert INSPECTION_GRANTS.get((service, method)) in granted, f"{service}.{method}" + + deploy = _load_yaml(".github/workflows/deploy.yml")["jobs"]["deploy"]["steps"] + step = next(s for s in deploy if "inspect_alarms.py" in s.get("run", "")) + assert step["run"].strip() == "uv run --with boto3 python scripts/inspect_alarms.py" + assert step.get("continue-on-error") is True + dispatch = _load_yaml(".github/workflows/backup-inspection.yml")["jobs"]["inspect"]["steps"] + assert any("scripts/inspect_alarms.py" in s.get("run", "") for s in dispatch)