Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/backup-inspection.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,6 @@ jobs:
args="--invoke"
fi
uv run --with boto3 python scripts/inspect_backup.py $args
- name: Inspect event-queue and operator alarms
if: always()
run: uv run --with boto3 python scripts/inspect_alarms.py
6 changes: 6 additions & 0 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -208,5 +208,11 @@ jobs:
# the next 03:00Z run).
continue-on-error: true
run: uv run --with boto3 python scripts/inspect_backup.py
- name: Inspect event-queue and operator alarms (read-only)
# Alarm states, transitions, queue-age peaks, DLQ depth: proof that
# a paged alarm recovered and has not come back. Same grants as the
# backup inspection (infra/backup-inspection-access.yaml).
continue-on-error: true
run: uv run --with boto3 python scripts/inspect_alarms.py
- name: Inspect YouTube renewal infrastructure (read-only)
run: bash scripts/inspect-youtube-renewal.sh
115 changes: 115 additions & 0 deletions scripts/inspect_alarms.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
"""Inspect the event-queue and operator alarms (read-only).

The deploy workflow and the dispatchable backup-inspection workflow run this
after the backup checks. It prints ALARMS_* evidence lines: each alarm's
current state, its recent state transitions (the verbatim "Threshold
Crossed" datapoints), the event queue's oldest-message age peaks, the
dead-letter queue depth, and the browser client-check failure counts the
operator topic also pages on. An alarm email says when a threshold was
crossed; these lines say whether it recovered and whether it came back.

Uses only the cloudformation/cloudwatch grants in
infra/backup-inspection-access.yaml (tests/test_backup.py keeps the two in
lockstep); a missing grant prints ALARMS_UNVERIFIED instead of failing.
"""
import json
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone

import boto3
from botocore.exceptions import ClientError

# Logical alarm ids in the dapier stack, plus alarms other stacks publish to
# the same operator topic by name.
STACK_ALARMS = ('EventQueueAgeAlarm', 'EventQueueDepthAlarm', 'EventDeadLetterAlarm')
NAMED_ALARMS = ('dataops-v1-client-check-failed',)
HISTORY_DAYS = 3


@contextmanager
def inspect(label):
try:
yield
except ClientError as exc:
print('ALARMS_UNVERIFIED', label, exc.response['Error']['Code'])


cf = boto3.client('cloudformation')
cw = boto3.client('cloudwatch')


def physical(logical):
return cf.describe_stack_resource(StackName='dapier', LogicalResourceId=logical)['StackResourceDetail']['PhysicalResourceId']


def queue_name(logical):
# An SQS queue's physical id is its URL; metrics key on the name.
return physical(logical).rsplit('/', 1)[-1]


def peaks(namespace, metric, dimensions, statistic, period, floor, days=HISTORY_DAYS):
"""Datapoints at or above `floor` over the last `days`, oldest first.
GetMetricStatistics caps one call at 1440 datapoints, so day windows."""
now = datetime.now(timezone.utc)
points = []
for offset in range(days):
end = now - timedelta(days=offset)
points.extend(
(str(point['Timestamp']), point[statistic])
for point in cw.get_metric_statistics(
Namespace=namespace, MetricName=metric, Dimensions=dimensions,
StartTime=end - timedelta(days=1), EndTime=end, Period=period,
Statistics=[statistic])['Datapoints']
if point[statistic] >= floor)
return sorted(points)


def history(name):
since = datetime.now(timezone.utc) - timedelta(days=HISTORY_DAYS)
items = []
kwargs = {'AlarmName': name, 'HistoryItemType': 'StateUpdate',
'StartDate': since, 'MaxRecords': 100}
while True:
page = cw.describe_alarm_history(**kwargs)
items.extend(page.get('AlarmHistoryItems', []))
if not page.get('NextToken'):
break
kwargs['NextToken'] = page['NextToken']
return [{'timestamp': str(item.get('Timestamp')), 'summary': item.get('HistorySummary')}
for item in sorted(items, key=lambda item: item.get('Timestamp'))]


def main():
names = []
for logical in STACK_ALARMS:
with inspect(f'resolve {logical}'):
names.append(physical(logical))
names.extend(NAMED_ALARMS)
with inspect('alarm-states'):
found = cw.describe_alarms(AlarmNames=names)['MetricAlarms']
print('ALARMS_STATE', json.dumps([
{k: alarm.get(k) for k in ('AlarmName', 'StateValue', 'StateUpdatedTimestamp', 'StateReason')}
for alarm in found], default=str))
missing = sorted(set(names) - {alarm['AlarmName'] for alarm in found})
if missing:
print('ALARMS_NOT_VISIBLE', json.dumps(missing))
for name in names:
with inspect(f'history {name}'):
print('ALARMS_HISTORY', json.dumps({'alarm': name, 'items': history(name)}))
with inspect('event-queue-age'):
# Five-minute maxima of the oldest message's age, only those past
# five minutes: an empty list is a queue that never backed up.
queue = [{'Name': 'QueueName', 'Value': queue_name('EventQueue')}]
print('ALARMS_QUEUE_AGE', json.dumps({'over_300s': peaks(
'AWS/SQS', 'ApproximateAgeOfOldestMessage', queue, 'Maximum', 300, 300)}))
with inspect('dead-letter-depth'):
dlq = [{'Name': 'QueueName', 'Value': queue_name('DeadLetterQueue')}]
print('ALARMS_DLQ_VISIBLE', json.dumps({'hours_nonzero': peaks(
'AWS/SQS', 'ApproximateNumberOfMessagesVisible', dlq, 'Maximum', 3600, 1)}))
with inspect('client-check-failures'):
print('ALARMS_CLIENT_CHECK_FAILED', json.dumps({'five_minute_sums': peaks(
'DataOps/Client', 'ClientCheckFailed', [], 'Sum', 300, 1)}))


if __name__ == '__main__':
main()
52 changes: 52 additions & 0 deletions src/dapier/engine/logic_pkg/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ def _run_step(workflow_id, step, index, event, run_action, *,
step_outputs[action_id]["secret"] = True
if gate.status == "filtered":
return "filtered"
return _rehydrate_branch(
workflow_id, step, action_id, gate.output, event, run_action,
before_action=before_action, after_action=after_action,
on_action_error=on_action_error, scope=scope,
step_outputs=step_outputs)
else:
step_outputs[action_id] = {"status": "skipped"}
return None
Expand Down Expand Up @@ -146,6 +151,53 @@ def _run_step(workflow_id, step, index, event, run_action, *,
return stop


def _recorded_branch(step, output):
"""The (steps, id suffix) of the branch a recovered condition/paths step
took, read from its persisted output; None when it took none."""
if not isinstance(output, dict):
return None
branch = output.get("branch")
step_type = str(step.get("type"))
if step_type == "condition" and branch in ("then", "else"):
chain = step.get(branch)
elif step_type == "paths" and branch == "default":
chain = step.get("default")
elif step_type == "paths" and branch:
chain = next((path.get("actions") for path in step.get("paths") or []
if isinstance(path, dict)
and (str(path.get("label") or "").strip() or "path") == branch), None)
else:
return None
return (chain, branch) if isinstance(chain, list) and chain else None


def _rehydrate_branch(workflow_id, step, action_id, output, event, run_action, *,
before_action, after_action, on_action_error, scope=None,
step_outputs=None):
"""Re-enter the branch a recovered condition/paths step recorded.

A redelivered event recovers a completed branching step from run history
as a whole, but later steps read the branch's children by their nested
ids (``route.else.post-webhook``). Running the recorded branch again —
without re-evaluating the predicate — puts each child through the same
gate as a top-level step: completed children restore their own records,
and children that failed or were skipped run again, like any unfinished
top-level step on a redelivery.
"""
recorded = _recorded_branch(step, output)
if recorded is None:
return None
chain, branch = recorded
from .chains import run_chain # lazy: chains imports this module at its top

return run_chain(
workflow_id, chain, event, run_action,
before_action=before_action, after_action=after_action,
on_action_error=on_action_error, prefix=f"{action_id}.{branch}",
scope=scope, step_outputs=step_outputs,
)


def _handle_error(workflow_id, step, action_id, exc, event, run_action, *,
before_action, after_action, on_action_error, scope=None,
step_outputs=None, duration_ms=None):
Expand Down
28 changes: 28 additions & 0 deletions tests/test_backup.py
Original file line number Diff line number Diff line change
Expand Up @@ -613,3 +613,31 @@ def test_backup_failure_email_routes_only_to_dev_inbox():
# The backup recipient parameter itself must never default to a personal inbox.
assert "alexey@datatalks.club" not in template["Parameters"]["BackupAlertEmail"]["Default"]
assert "alexey@datatalks.club" not in Path("src/dapier/backup.py").read_text()


def test_alarm_inspection_stays_within_inspection_grants():
"""scripts/inspect_alarms.py runs in the same deploy and dispatch jobs on
the same role: every boto3 call it makes must already be granted, and it
must run read-only in both workflows."""
import re

script = Path("scripts/inspect_alarms.py").read_text()
clients = dict(re.findall(r"^(\w+) = boto3\.client\('([^']+)'\)\s*$", script, re.MULTILINE))
calls = {(service, method)
for var, service in clients.items()
for method in re.findall(rf"\b{var}\.(\w+)\(", script)}
assert calls, "no boto3 calls parsed"
infra = _load_yaml("infra/backup-inspection-access.yaml")
policy = infra["Resources"]["GithubDeployBackupInspectionAccess"]["Properties"]
granted = {action
for statement in policy["PolicyDocument"]["Statement"]
for action in statement["Action"]}
for service, method in sorted(calls):
assert INSPECTION_GRANTS.get((service, method)) in granted, f"{service}.{method}"

deploy = _load_yaml(".github/workflows/deploy.yml")["jobs"]["deploy"]["steps"]
step = next(s for s in deploy if "inspect_alarms.py" in s.get("run", ""))
assert step["run"].strip() == "uv run --with boto3 python scripts/inspect_alarms.py"
assert step.get("continue-on-error") is True
dispatch = _load_yaml(".github/workflows/backup-inspection.yml")["jobs"]["inspect"]["steps"]
assert any("scripts/inspect_alarms.py" in s.get("run", "") for s in dispatch)
104 changes: 104 additions & 0 deletions tests/test_branch_recovery.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
"""A redelivered event recovers a completed branching step from run history;
its branch children must come back under their nested ids too.

Regression for the 2026-11-19/20 dev-mailbox failures: on each queue
redelivery the ``route`` condition was restored whole, ``route.else.post-webhook``
never re-entered ``steps``, and verify-delivery failed the run as if the
webhook leg had not been delivered.
"""
from src.dapier.engine.logic_pkg import CompletedStep, run_chain


def _gate(records):
"""A worker-style step gate over recorded step statuses: completed steps
are restored from history, anything else runs (True)."""
def before_action(workflow_id, action_id, event, action_type=None):
record = records.get(action_id)
if record and record[0] in ("completed", "filtered"):
return CompletedStep(*record)
return True
return before_action


ROUTE = {
"id": "route", "type": "condition",
"when": {"steps.decide.output.route": {"equals": "hub"}},
"then": [{"id": "notify", "type": "http_request"}],
"else": [{"id": "post-webhook", "type": "http_request"},
{"id": "post-copy", "type": "http_request"}],
}


def test_recovered_condition_restores_completed_children_without_rerunning():
records = {
"route": ("completed", {"branch": "else", "condition": "failed"}),
"route.else.post-webhook": ("completed", {"status": 202}),
"route.else.post-copy": ("completed", {"status": 200}),
}
calls = []

def dispatch(step, event, workflow_id, steps=None):
calls.append(step["id"])
return {"status": 500}

outputs = {}
run_chain("wf", [ROUTE], {"id": "evt", "data": {}}, dispatch,
before_action=_gate(records), step_outputs=outputs)
assert calls == []
assert outputs["route"]["output"]["branch"] == "else"
assert outputs["route.else.post-webhook"] == {"status": "completed",
"output": {"status": 202}}
assert outputs["route.else.post-copy"]["status"] == "completed"
assert "route.then.notify" not in outputs


def test_recovered_condition_reruns_unfinished_children_on_the_recorded_branch():
# The predicate would now pick "then" (decide is absent), but the
# recorded branch wins: the run stays on the path it already took.
records = {
"route": ("completed", {"branch": "else", "condition": "failed"}),
"route.else.post-webhook": ("completed", {"status": 202}),
"route.else.post-copy": ("failed", {}),
}
calls = []

def dispatch(step, event, workflow_id, steps=None):
calls.append(step["id"])
return {"status": 200}

outputs = {}
run_chain("wf", [ROUTE], {"id": "evt", "data": {}}, dispatch,
before_action=_gate(records), step_outputs=outputs)
assert calls == ["post-copy"]
assert outputs["route.else.post-copy"] == {"status": "completed",
"output": {"status": 200}}


def test_recovered_paths_step_restores_its_recorded_path_children():
step = {
"id": "pick", "type": "paths",
"paths": [{"label": "urgent", "when": {"subject": {"contains": "!"}},
"actions": [{"id": "page", "type": "http_request"}]}],
"default": [{"id": "log", "type": "http_request"}],
}
records = {
"pick": ("completed", {"branch": "urgent", "matched": "urgent"}),
"pick.urgent.page": ("completed", {"status": 202}),
}
outputs = {}
run_chain("wf", [step], {"id": "evt", "data": {}},
lambda *args, **kwargs: {"status": 500},
before_action=_gate(records), step_outputs=outputs)
assert outputs["pick.urgent.page"]["output"] == {"status": 202}
assert "pick.default.log" not in outputs


def test_recovered_condition_without_a_recorded_branch_runs_nothing():
records = {"route": ("completed", {"result": {}})}
calls = []
outputs = {}
run_chain("wf", [ROUTE], {"id": "evt", "data": {}},
lambda step, *args, **kwargs: calls.append(step["id"]) or {},
before_action=_gate(records), step_outputs=outputs)
assert calls == []
assert list(outputs) == ["route"]
33 changes: 33 additions & 0 deletions tests/test_dev_mailbox_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,39 @@ def test_dev_mailbox_delivery_verifier_requires_ingest_and_routed_leg():
assert result["oncall_ingest"] == "completed"


def test_dev_mailbox_verifier_counts_oncall_409_as_already_delivered():
"""A retry whose first attempt reached Hub gets 409 delivery_id_conflict
(same X-Delivery-Id, different body digest): the email is already in
Hub's queue, so the run must not fail into 1800s queue redeliveries."""
_, wf = _load("dev-mailbox")
verify_action = next(a for a in wf["actions"] if a.get("id") == "verify-delivery")
completed = {"status": "completed", "output": {}}
steps = {
"route": completed | {"output": {"branch": "else"}},
"route.else.post-webhook": completed,
"post-oncall-ingest": {"status": "failed", "output": {},
"error": "http_request returned HTTP 409"},
}
result = run_code(verify_action, {"connector": "email", "data": {}}, steps=steps)["result"]
assert result["oncall_ingest"] == "already_delivered"
# Any other ingest failure still fails the run.
steps["post-oncall-ingest"] = {"status": "failed", "output": {},
"error": "The read operation timed out"}
with pytest.raises(RuntimeError, match="post-oncall-ingest"):
run_code(verify_action, {"connector": "email", "data": {}}, steps=steps)


def test_dev_mailbox_retries_failed_delivery_before_queue_redelivery():
from src.dapier.engine.worker import retry_policy

_, wf = _load("dev-mailbox")
parse_workflow(yaml.safe_dump(wf))
policy = retry_policy(wf)
assert policy["attempts"] >= 2
# Each retry must re-enqueue well under the 900s EventQueueAgeAlarm.
assert policy["backoff_seconds"] < 900


@pytest.mark.parametrize("token_value", [
"secret-token",
"Bearer secret-token",
Expand Down
Loading
Loading