diff --git a/CHANGELOG.md b/CHANGELOG.md index f555fd5..36a1fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ All notable changes to this project are documented here. The format follows ### Added - `scripts/chaos.py` and a nightly workflow: kill the emulator, keepstack, or everything at once in the middle of `terraform apply` and require Terraform to converge. Documented in [how it works](docs/how-it-works.md#crash-safety). + After a power cut, resources that Terraform created but never saved (as would happen on AWS) are reported as + `orphaned_by_terraform` instead of failing the run, as long as keepstack keeps exactly the same state across a restart. ### Changed - README: leads with Terraform validation (locally and in every pull request), with persistence as the engine; diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 59b72b1..b737fa2 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -105,12 +105,12 @@ in an SQLite transaction. Every way of dying mid-apply ends in a state that matc |---|---| | **The emulator** (`docker kill`, Docker restart, OOM) | keepstack notices, starts a new emulator and replays the journal while new requests wait. The SDK's retries go through once the state is back: the apply usually finishes by itself. | | **keepstack** (`kill -9`), restarted while Terraform retries | A write that was in flight is left unconfirmed and skipped on the next start (the client never got an answer, so Terraform does not have it either). Terraform's retries then succeed. | -| **Everything at once** (power cut) | The same, plus Terraform dies: run `terraform apply` again and it converges. | +| **Everything at once** (power cut) | keepstack restores everything the emulator confirmed. Terraform itself may have died while creating a resource it never saved to its state: as on AWS, the next apply reports "already exists" for it, and `terraform import` brings it back under management. | | **The client gives up** while its request waits (restore, slow write) | The request is not executed: keepstack checks the connection before running a request that waited. Otherwise the resource would be created behind the client's back and its retry would fail with "already exists". | This is exercised by [`scripts/chaos.py`](../scripts/chaos.py), which kills one of the three at a random moment of an apply of the quickstart example and requires the re-apply to converge and the plan to stay clean after another -restart. It runs nightly in CI ([`chaos.yml`](../.github/workflows/chaos.yml)); the deterministic case of the +restart (after a power cut: that the only errors are resources Terraform lost, and that another restart changes nothing). It runs nightly in CI ([`chaos.yml`](../.github/workflows/chaos.yml)); the deterministic case of the abandoned write is an end-to-end test (`tests/test_e2e_crash.py`). ## Limitations diff --git a/scripts/chaos.py b/scripts/chaos.py index 3d996d6..fb6b68f 100644 --- a/scripts/chaos.py +++ b/scripts/chaos.py @@ -15,6 +15,7 @@ import json import os import random +import re import shutil import signal import subprocess @@ -24,6 +25,17 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) MODES = ("emulator", "keepstack", "power") +ALREADY_EXISTS = re.compile(r"already exists|AlreadyExists|ResourceInUseException|AlreadyOwnedByYou", re.IGNORECASE) +PLANNED = re.compile(r"^ # (\S+) will be", re.MULTILINE) + + +def orphans(apply_output): + """Resources a killed `terraform apply` created but never saved (as on AWS): the re-apply's errors when every + one of them says "already exists", otherwise None (a real failure).""" + errors = apply_output.split("\nError: ")[1:] + if not errors or not all(ALREADY_EXISTS.search(e.split("\n", 1)[0]) for e in errors): + return None + return sorted(set(re.findall(r"^\s+with (\S+),", apply_output, re.MULTILINE))) def run(cmd, env, cwd=None, timeout=900): @@ -79,7 +91,7 @@ def ks(*a): reapplies.append(rc) if rc == 0: break - plan_rc, _ = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root) + plan_rc, plan = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root) ks("down") ks(*up) restart_rc, restart_plan = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root) @@ -90,6 +102,12 @@ def ks(*a): "unconfirmed_skipped": log.count("was never confirmed"), "abandoned": log.count("client gave up"), "replay_failed": log.count("REPLAY FAILED")} result["ok"] = reapplies[-1] == 0 and plan_rc == 0 and restart_rc == 0 + lost = orphans(out) if mode == "power" and reapplies[-1] != 0 else None + if lost is not None: + # Terraform died before saving them, as it would on AWS; keepstack's part is to keep exactly the same state + # across another restart. + result["orphaned_by_terraform"] = lost + result["ok"] = PLANNED.findall(plan) == PLANNED.findall(restart_plan) if not result["ok"]: result["details"] = {"first_apply": first[-3000:], "last_reapply": out[-3000:], "plan_after_restart": restart_plan[-3000:], "keepstack_log": log[-3000:]} diff --git a/tests/test_chaos.py b/tests/test_chaos.py new file mode 100644 index 0000000..e83b885 --- /dev/null +++ b/tests/test_chaos.py @@ -0,0 +1,45 @@ +"""Unit tests for scripts/chaos.py: telling resources Terraform lost in a power cut from real failures.""" +import importlib.util +import os +import unittest + +spec = importlib.util.spec_from_file_location( + "chaos", os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "scripts", "chaos.py")) +chaos = importlib.util.module_from_spec(spec) +spec.loader.exec_module(chaos) + +# Re-apply after a power cut on Linux (#39): the table was created, Terraform died before saving it. +LOST = """aws_dynamodb_table.items: Creating... + +Error: creating AWS DynamoDB Table (items): operation error DynamoDB: CreateTable, https response error StatusCode: \ +400, RequestID: 0I74K6Z35KIT7JPLR7549KZUQ2IOL6CMF8DK69TNEGVUR8EJGCU9, ResourceInUseException: Table already exists: items + + with aws_dynamodb_table.items, + on main.tf line 51, in resource "aws_dynamodb_table" "items": + 51: resource "aws_dynamodb_table" "items" { + +""" +REAL = """ +Error: creating IAM Role (app): operation error IAM: CreateRole, https response error StatusCode: 400, \ +MalformedPolicyDocument: invalid principal + + with aws_iam_role.app, + on main.tf line 70, in resource "aws_iam_role" "app": +""" + + +class Orphans(unittest.TestCase): + def test_already_exists_errors_are_resources_terraform_lost(self): + self.assertEqual(chaos.orphans(LOST), ["aws_dynamodb_table.items"]) + + def test_any_other_error_is_a_real_failure(self): + self.assertIsNone(chaos.orphans(LOST + REAL)) + self.assertIsNone(chaos.orphans("Apply complete!")) + + def test_planned_addresses(self): + plan = " # aws_dynamodb_table.items will be created\n # aws_dynamodb_table_item.seed will be created\n" + self.assertEqual(chaos.PLANNED.findall(plan), ["aws_dynamodb_table.items", "aws_dynamodb_table_item.seed"]) + + +if __name__ == "__main__": + unittest.main()