From 51ba63ca6ace6ce4c23b984d9eae5b36b183bd6e Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 1 Aug 2026 10:34:51 +0200 Subject: [PATCH] feat(ops): add Azure temporary VM cost guard --- .github/workflows/azure-cost-guard.yml | 49 ++++ ops/azure/README.md | 91 ++++++++ scripts/azure_cost_guard.py | 295 +++++++++++++++++++++++++ tests/test_azure_cost_guard.py | 113 ++++++++++ 4 files changed, 548 insertions(+) create mode 100644 .github/workflows/azure-cost-guard.yml create mode 100644 ops/azure/README.md create mode 100644 scripts/azure_cost_guard.py create mode 100644 tests/test_azure_cost_guard.py diff --git a/.github/workflows/azure-cost-guard.yml b/.github/workflows/azure-cost-guard.yml new file mode 100644 index 0000000..17ee1ef --- /dev/null +++ b/.github/workflows/azure-cost-guard.yml @@ -0,0 +1,49 @@ +name: Azure temporary VM cost guard + +on: + workflow_dispatch: + schedule: + - cron: '17 14 * * *' + +permissions: + contents: read + +concurrency: + group: azure-cost-guard + cancel-in-progress: false + +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - id: credentials + name: Detect Azure credentials + env: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + run: | + if [ -n "$AZURE_CREDENTIALS" ]; then + echo 'available=true' >> "$GITHUB_OUTPUT" + else + echo 'available=false' >> "$GITHUB_OUTPUT" + fi + + - if: ${{ steps.credentials.outputs.available == 'true' }} + uses: Azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - id: guard + name: Report temporary VM cost + if: ${{ steps.credentials.outputs.available == 'true' }} + continue-on-error: true + run: >- + python scripts/azure_cost_guard.py + --subscription 78add6c6-c92a-4a53-b751-eb644ac77e59 + --alert-daily-cad 5.00 + --github-output "$GITHUB_OUTPUT" + + - name: Fail when the guard detects unprotected daily cost + if: ${{ steps.credentials.outputs.available == 'true' && (steps.guard.outcome == 'failure' || steps.guard.outputs.alert == 'true') }} + run: exit 1 diff --git a/ops/azure/README.md b/ops/azure/README.md new file mode 100644 index 0000000..f628289 --- /dev/null +++ b/ops/azure/README.md @@ -0,0 +1,91 @@ +# Azure temporary-VM cost guard + +This guard protects subscription `78add6c6-c92a-4a53-b751-eb644ac77e59`. +It is report-only by default. It never deletes an Azure resource. + +## Run it + +```bash +python scripts/azure_cost_guard.py +``` + +The guard reports each running VM that has either: + +- `temporary:true`; or +- an `autostop` tag on the VM or its resource group. + +It reports the resource ID, age in hours, power state, SKU, owner, lease expiry, and +the latest available daily compute cost from Azure Cost Management. The default +cost threshold is CAD 5.00 per day. Exit status `2` means that a running +temporary/autostop VM has no active owner lease. The report also states whether +its known cost reached the threshold. This fails visibly even when Azure cost +data arrives late. + +## Scheduled monitor setup + +`.github/workflows/azure-cost-guard.yml` runs once each day and only reports. +It becomes active when the repository secret `AZURE_CREDENTIALS` exists. + +Use a service-principal JSON credential for the target subscription with these +minimum roles: + +- `Reader` for resource state and tags. +- `Cost Management Reader` for the daily cost estimate. + +Do not grant `Contributor` while the workflow stays report-only. The workflow +does not pass `--action deallocate`, `--apply`, or `--confirm-deallocate`. +The current repository has no `AZURE_CREDENTIALS` secret, so the scheduled job +will skip until this secret is added. + +## Lease protection + +Use both tags for an active qualification job: + +```text +owner=qualification-openemr +lease_expires_at=2026-08-02T18:00:00Z +``` + +The guard treats a VM as protected only when both tags exist and the expiry is +in the future. A missing, malformed, or expired lease is not protected. + +## Explicit deallocation + +The guard can deallocate an unprotected candidate. It never deletes a VM, +disk, IP address, snapshot, or resource group. + +```bash +python scripts/azure_cost_guard.py \ + --action deallocate --apply --confirm-deallocate \ + --recovery-snapshot-id /subscriptions/78add6c6-c92a-4a53-b751-eb644ac77e59/resourceGroups/openadapt-qualification-temp-20260726/providers/Microsoft.Compute/snapshots/openemr-qual-20260726-predeallocate-20260801 +``` + +Use this only after the workflow owner confirms that the job ended. A VM can +still incur disk and static-IP cost after deallocation. + +## OpenEMR qualification recovery note + +The current temporary VM is: + +```text +openemr-qual-20260726 +openadapt-qualification-temp-20260726 +``` + +Before any planned deallocation, verify the recovery snapshot: + +```text +openemr-qual-20260726-predeallocate-20260801 +``` + +Azure reports this exact resource as `Succeeded`: + +```text +/subscriptions/78add6c6-c92a-4a53-b751-eb644ac77e59/resourceGroups/openadapt-qualification-temp-20260726/providers/Microsoft.Compute/snapshots/openemr-qual-20260726-predeallocate-20260801 +``` + +The explicit deallocation path verifies this ID and state before it sends a VM +deallocation request. Record the VM ID, disk ID, snapshot ID, and the qualified +bundle/report hashes in the qualification record. Deallocation preserves the +VM disk. Delete the resource group only after the evidence export and recovery +decision. diff --git a/scripts/azure_cost_guard.py b/scripts/azure_cost_guard.py new file mode 100644 index 0000000..ef59eca --- /dev/null +++ b/scripts/azure_cost_guard.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Report Azure temporary-VM cost risk, with an explicit deallocation option. + +The normal mode is report-only. The optional deallocation mode requires both +``--apply`` and ``--confirm-deallocate``. It never deletes a resource. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + + +TARGET_SUBSCRIPTION = "78add6c6-c92a-4a53-b751-eb644ac77e59" +DEFAULT_ALERT_DAILY_CAD = 5.0 +LEASE_TAGS = ("lease_expires_at", "lease-expires-at", "openadapt_lease_expires_at") + + +class AzureCommandError(RuntimeError): + """An Azure CLI command did not return usable JSON.""" + + +def az_json(arguments: list[str], *, input_json: dict[str, Any] | None = None) -> Any: + command = ["az", *arguments, "-o", "json"] + result = subprocess.run( + command, + input=json.dumps(input_json) if input_json is not None else None, + capture_output=True, + check=False, + text=True, + ) + if result.returncode: + raise AzureCommandError(result.stderr.strip() or "Azure CLI failed") + try: + return json.loads(result.stdout) + except json.JSONDecodeError as error: + raise AzureCommandError("Azure CLI did not return JSON") from error + + +def as_tags(value: Any) -> dict[str, str]: + if not isinstance(value, dict): + return {} + return {str(key).lower(): str(item) for key, item in value.items() if item is not None} + + +def tag_is_true(tags: dict[str, str], name: str) -> bool: + return tags.get(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def has_autostop_policy(tags: dict[str, str]) -> bool: + return bool(tags.get("autostop", "").strip()) + + +def parse_time(value: str) -> datetime | None: + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError: + return None + + +def age_hours(created_at: str | None, now: datetime) -> float | None: + """Return an age in hours when Azure supplied a valid creation time.""" + if not created_at: + return None + created = parse_time(created_at) + if created is None or created > now: + return None + return round((now - created).total_seconds() / 3600, 1) + + +def active_lease(tags: dict[str, str], now: datetime) -> tuple[bool, str | None]: + """Return whether a VM has an owner and an unexpired lease.""" + owner = tags.get("owner", "").strip() + expiry_text = next((tags[name] for name in LEASE_TAGS if tags.get(name)), None) + expiry = parse_time(expiry_text) if expiry_text else None + if owner and expiry and expiry > now: + return True, expiry.isoformat() + return False, expiry_text + + +def latest_cost_by_resource(subscription: str, start: datetime, end: datetime) -> dict[str, float]: + """Read the last non-zero daily VM cost from Cost Management when available.""" + payload = { + "type": "ActualCost", + "timeframe": "Custom", + "timePeriod": {"from": start.isoformat(), "to": end.isoformat()}, + "dataset": { + "granularity": "Daily", + "aggregation": {"cost": {"name": "PreTaxCost", "function": "Sum"}}, + "grouping": [{"type": "Dimension", "name": "ResourceId"}], + }, + } + response = az_json( + [ + "rest", + "--method", + "post", + "--url", + "https://management.azure.com/subscriptions/" + f"{subscription}/providers/Microsoft.CostManagement/query?api-version=2023-11-01", + "--body", + json.dumps(payload), + ] + ) + properties = response.get("properties", {}) if isinstance(response, dict) else {} + columns = properties.get("columns", []) + rows = properties.get("rows", []) + names = [column.get("name") for column in columns if isinstance(column, dict)] + try: + cost_index, date_index, resource_index = ( + names.index("PreTaxCost"), + names.index("UsageDate"), + names.index("ResourceId"), + ) + except ValueError: + return {} + latest: dict[str, tuple[int, float]] = {} + for row in rows: + if not isinstance(row, list) or len(row) <= max(cost_index, date_index, resource_index): + continue + try: + cost, date, resource = float(row[cost_index]), int(row[date_index]), str(row[resource_index]).lower() + except (TypeError, ValueError): + continue + if cost > 0 and (resource not in latest or date > latest[resource][0]): + latest[resource] = (date, cost) + return {resource: cost for resource, (_, cost) in latest.items()} + + +@dataclass(frozen=True) +class Candidate: + id: str + name: str + resource_group: str + sku: str + power_state: str + created_at: str | None + age_hours: float | None + reason: str + owner: str | None + lease_expires_at: str | None + protected: bool + daily_cost_cad: float | None + + +def find_candidates( + vms: list[dict[str, Any]], + group_tags: dict[str, dict[str, str]], + costs: dict[str, float], + now: datetime, +) -> list[Candidate]: + candidates: list[Candidate] = [] + for vm in vms: + if not isinstance(vm, dict) or str(vm.get("powerState", "")).lower() != "vm running": + continue + group = str(vm.get("resourceGroup", "")) + vm_tags = as_tags(vm.get("tags")) + combined_tags = {**group_tags.get(group.lower(), {}), **vm_tags} + temporary = tag_is_true(combined_tags, "temporary") + autostop = has_autostop_policy(combined_tags) + if not temporary and not autostop: + continue + protected, lease_expiry = active_lease(combined_tags, now) + reason = "temporary:true" if temporary else "autostop policy" + resource_id = str(vm.get("id", "")).lower() + candidates.append( + Candidate( + id=resource_id, + name=str(vm.get("name", "")), + resource_group=group, + sku=str(vm.get("hardwareProfile", {}).get("vmSize", "unknown")), + power_state=str(vm.get("powerState", "unknown")), + created_at=vm.get("timeCreated"), + age_hours=age_hours(vm.get("timeCreated"), now), + reason=reason, + owner=combined_tags.get("owner") or None, + lease_expires_at=lease_expiry, + protected=protected, + daily_cost_cad=costs.get(resource_id), + ) + ) + return candidates + + +def report(candidates: list[Candidate], threshold: float) -> tuple[dict[str, Any], bool]: + at_risk = [candidate for candidate in candidates if not candidate.protected] + known_daily_cost = sum(candidate.daily_cost_cad or 0 for candidate in at_risk) + # A running temporary VM with an expired or absent lease is itself the + # actionable signal. Cost data can arrive late, so do not make alerting + # depend on its availability. The threshold adds cost context to the alert. + alert = bool(at_risk) + return ( + { + "subscription": TARGET_SUBSCRIPTION, + "action": "report", + "threshold_daily_cad": threshold, + "estimated_unprotected_daily_cost_cad": round(known_daily_cost, 2), + "cost_threshold_exceeded": known_daily_cost >= threshold, + "alert": alert, + "candidates": [candidate.__dict__ for candidate in candidates], + }, + alert, + ) + + +def write_github_output(path: str | None, alert: bool, estimated_cost: float) -> None: + if not path: + return + Path(path).write_text( + f"alert={'true' if alert else 'false'}\n" + f"estimated_unprotected_daily_cost_cad={estimated_cost:.2f}\n", + encoding="utf-8", + ) + + +def verify_recovery_snapshot(subscription: str, snapshot_id: str) -> None: + """Require a succeeded recovery snapshot before explicit deallocation.""" + expected_prefix = f"/subscriptions/{subscription.lower()}/" + if not snapshot_id.lower().startswith(expected_prefix): + raise AzureCommandError("Recovery snapshot belongs to a different subscription") + snapshot = az_json(["snapshot", "show", "--ids", snapshot_id]) + if not isinstance(snapshot, dict) or snapshot.get("provisioningState") != "Succeeded": + raise AzureCommandError("Recovery snapshot is not in Succeeded state") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--subscription", default=TARGET_SUBSCRIPTION) + parser.add_argument("--alert-daily-cad", type=float, default=DEFAULT_ALERT_DAILY_CAD) + parser.add_argument("--action", choices=("report", "deallocate"), default="report") + parser.add_argument("--apply", action="store_true", help="Perform the selected action.") + parser.add_argument("--confirm-deallocate", action="store_true") + parser.add_argument("--recovery-snapshot-id") + parser.add_argument("--github-output") + args = parser.parse_args(argv) + if args.subscription != TARGET_SUBSCRIPTION: + parser.error(f"This guard only permits subscription {TARGET_SUBSCRIPTION}.") + if args.action == "deallocate" and ( + not args.apply or not args.confirm_deallocate or not args.recovery_snapshot_id + ): + parser.error( + "Deallocation requires --apply, --confirm-deallocate, and --recovery-snapshot-id." + ) + + now = datetime.now(timezone.utc) + groups = az_json(["group", "list", "--subscription", args.subscription]) + group_tags = { + str(group.get("name", "")).lower(): as_tags(group.get("tags")) + for group in groups + if isinstance(group, dict) + } + vms = az_json(["vm", "list", "-d", "--subscription", args.subscription]) + # Do not call Cost Management when no running temporary/autostop VM exists. + # This keeps the scheduled guard cheap while preserving cost estimates when + # they can affect an alert or an operator decision. + candidates = find_candidates(vms, group_tags, {}, now) + if candidates: + costs: dict[str, float] = {} + try: + costs = latest_cost_by_resource( + args.subscription, + now.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=8), + now, + ) + except AzureCommandError as error: + print(f"WARNING: Cost Management data is unavailable: {error}", file=sys.stderr) + candidates = find_candidates(vms, group_tags, costs, now) + payload, alert = report(candidates, args.alert_daily_cad) + payload["action"] = args.action + print(json.dumps(payload, indent=2, sort_keys=True)) + write_github_output( + args.github_output, + alert, + payload["estimated_unprotected_daily_cost_cad"], + ) + + if args.action == "deallocate" and args.apply: + verify_recovery_snapshot(args.subscription, args.recovery_snapshot_id) + for candidate in candidates: + if candidate.protected: + continue + subprocess.run( + ["az", "vm", "deallocate", "--ids", candidate.id], check=True, text=True + ) + return 2 if alert else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_azure_cost_guard.py b/tests/test_azure_cost_guard.py new file mode 100644 index 0000000..b666c68 --- /dev/null +++ b/tests/test_azure_cost_guard.py @@ -0,0 +1,113 @@ +import pathlib +import sys +from datetime import datetime, timezone + +import yaml + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from azure_cost_guard import ( # noqa: E402 + AzureCommandError, + active_lease, + find_candidates, + report, + verify_recovery_snapshot, +) + + +NOW = datetime(2026, 8, 1, 12, tzinfo=timezone.utc) +RESOURCE_ID = "/subscriptions/x/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/temp" + + +def vm(*, tags=None, power_state="VM running"): + return { + "id": RESOURCE_ID, + "name": "temp", + "resourceGroup": "rg", + "hardwareProfile": {"vmSize": "Standard_D4s_v4"}, + "powerState": power_state, + "timeCreated": "2026-07-27T02:32:08Z", + "tags": tags or {}, + } + + +def test_temporary_running_vm_reports_cost_and_alerts(): + candidates = find_candidates( + [vm(tags={"temporary": "true"})], {}, {RESOURCE_ID.lower(): 6.54}, NOW + ) + payload, alert = report(candidates, threshold=5.0) + assert alert is True + assert payload["estimated_unprotected_daily_cost_cad"] == 6.54 + assert candidates[0].reason == "temporary:true" + assert candidates[0].age_hours == 129.5 + + +def test_group_autostop_policy_selects_vm(): + candidates = find_candidates([vm()], {"rg": {"autostop": "deallocate-when-idle"}}, {}, NOW) + assert len(candidates) == 1 + assert candidates[0].reason == "autostop policy" + + +def test_active_owner_lease_protects_qualification_job(): + tags = {"owner": "qualification", "lease_expires_at": "2026-08-01T13:00:00Z"} + assert active_lease(tags, NOW) == (True, "2026-08-01T13:00:00+00:00") + candidates = find_candidates([vm(tags={"temporary": "true", **tags})], {}, {RESOURCE_ID.lower(): 6.54}, NOW) + payload, alert = report(candidates, threshold=0.01) + assert candidates[0].protected is True + assert payload["estimated_unprotected_daily_cost_cad"] == 0 + assert alert is False + + +def test_expired_lease_is_not_protected(): + active, _ = active_lease( + {"owner": "qualification", "lease_expires_at": "2026-08-01T11:59:59Z"}, NOW + ) + assert active is False + + +def test_deallocated_vm_is_not_a_candidate(): + assert find_candidates([vm(tags={"temporary": "true"}, power_state="VM deallocated")], {}, {}, NOW) == [] + + +def test_unprotected_running_vm_alerts_even_when_cost_data_is_late(): + candidates = find_candidates([vm(tags={"temporary": "true"})], {}, {}, NOW) + _, alert = report(candidates, threshold=5.0) + assert alert is True + + +def test_recovery_snapshot_must_be_in_target_subscription(): + try: + verify_recovery_snapshot( + "target", + "/subscriptions/other/resourceGroups/rg/providers/Microsoft.Compute/snapshots/s", + ) + except AzureCommandError as error: + assert "different subscription" in str(error) + else: + raise AssertionError("expected a rejected snapshot") + + +def test_recovery_snapshot_must_have_succeeded_state(mocker): + mocker.patch("azure_cost_guard.az_json", return_value={"provisioningState": "Creating"}) + try: + verify_recovery_snapshot( + "target", + "/subscriptions/target/resourceGroups/rg/providers/Microsoft.Compute/snapshots/s", + ) + except AzureCommandError as error: + assert "Succeeded" in str(error) + else: + raise AssertionError("expected an incomplete snapshot rejection") + + +def test_scheduled_guard_uses_step_level_secret_gate(): + workflow = yaml.safe_load( + (REPO_ROOT / ".github/workflows/azure-cost-guard.yml").read_text(encoding="utf-8") + ) + steps = workflow["jobs"]["report"]["steps"] + assert "if" not in workflow["jobs"]["report"] + credentials = next(step for step in steps if step.get("id") == "credentials") + assert credentials["env"]["AZURE_CREDENTIALS"] == "${{ secrets.AZURE_CREDENTIALS }}" + login = next(step for step in steps if "Azure/login@" in step.get("uses", "")) + assert login["if"] == "${{ steps.credentials.outputs.available == 'true' }}"