Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ All notable changes to this project are documented here. The format follows
### Added
- `scripts/chaos.py` and a nightly workflow: kill the emulator, keepstack, or everything at once in the middle of
`terraform apply` and require Terraform to converge. Documented in [how it works](docs/how-it-works.md#crash-safety).
After a power cut, resources that Terraform created but never saved (as would happen on AWS) are reported as
`orphaned_by_terraform` instead of failing the run, as long as keepstack keeps exactly the same state across a restart.

### Changed
- README: leads with Terraform validation (locally and in every pull request), with persistence as the engine;
Expand Down
4 changes: 2 additions & 2 deletions docs/how-it-works.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,12 @@ in an SQLite transaction. Every way of dying mid-apply ends in a state that matc
|---|---|
| **The emulator** (`docker kill`, Docker restart, OOM) | keepstack notices, starts a new emulator and replays the journal while new requests wait. The SDK's retries go through once the state is back: the apply usually finishes by itself. |
| **keepstack** (`kill -9`), restarted while Terraform retries | A write that was in flight is left unconfirmed and skipped on the next start (the client never got an answer, so Terraform does not have it either). Terraform's retries then succeed. |
| **Everything at once** (power cut) | The same, plus Terraform dies: run `terraform apply` again and it converges. |
| **Everything at once** (power cut) | keepstack restores everything the emulator confirmed. Terraform itself may have died while creating a resource it never saved to its state: as on AWS, the next apply reports "already exists" for it, and `terraform import` brings it back under management. |
| **The client gives up** while its request waits (restore, slow write) | The request is not executed: keepstack checks the connection before running a request that waited. Otherwise the resource would be created behind the client's back and its retry would fail with "already exists". |

This is exercised by [`scripts/chaos.py`](../scripts/chaos.py), which kills one of the three at a random moment of
an apply of the quickstart example and requires the re-apply to converge and the plan to stay clean after another
restart. It runs nightly in CI ([`chaos.yml`](../.github/workflows/chaos.yml)); the deterministic case of the
restart (after a power cut: that the only errors are resources Terraform lost, and that another restart changes nothing). It runs nightly in CI ([`chaos.yml`](../.github/workflows/chaos.yml)); the deterministic case of the
abandoned write is an end-to-end test (`tests/test_e2e_crash.py`).

## Limitations
Expand Down
20 changes: 19 additions & 1 deletion scripts/chaos.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import json
import os
import random
import re
import shutil
import signal
import subprocess
Expand All @@ -24,6 +25,17 @@

ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
MODES = ("emulator", "keepstack", "power")
ALREADY_EXISTS = re.compile(r"already exists|AlreadyExists|ResourceInUseException|AlreadyOwnedByYou", re.IGNORECASE)
PLANNED = re.compile(r"^ # (\S+) will be", re.MULTILINE)


def orphans(apply_output):
"""Resources a killed `terraform apply` created but never saved (as on AWS): the re-apply's errors when every
one of them says "already exists", otherwise None (a real failure)."""
errors = apply_output.split("\nError: ")[1:]
if not errors or not all(ALREADY_EXISTS.search(e.split("\n", 1)[0]) for e in errors):
return None
return sorted(set(re.findall(r"^\s+with (\S+),", apply_output, re.MULTILINE)))


def run(cmd, env, cwd=None, timeout=900):
Expand Down Expand Up @@ -79,7 +91,7 @@ def ks(*a):
reapplies.append(rc)
if rc == 0:
break
plan_rc, _ = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root)
plan_rc, plan = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root)
ks("down")
ks(*up)
restart_rc, restart_plan = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root)
Expand All @@ -90,6 +102,12 @@ def ks(*a):
"unconfirmed_skipped": log.count("was never confirmed"), "abandoned": log.count("client gave up"),
"replay_failed": log.count("REPLAY FAILED")}
result["ok"] = reapplies[-1] == 0 and plan_rc == 0 and restart_rc == 0
lost = orphans(out) if mode == "power" and reapplies[-1] != 0 else None
if lost is not None:
# Terraform died before saving them, as it would on AWS; keepstack's part is to keep exactly the same state
# across another restart.
result["orphaned_by_terraform"] = lost
result["ok"] = PLANNED.findall(plan) == PLANNED.findall(restart_plan)
if not result["ok"]:
result["details"] = {"first_apply": first[-3000:], "last_reapply": out[-3000:],
"plan_after_restart": restart_plan[-3000:], "keepstack_log": log[-3000:]}
Expand Down
45 changes: 45 additions & 0 deletions tests/test_chaos.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
"""Unit tests for scripts/chaos.py: telling resources Terraform lost in a power cut from real failures."""
import importlib.util
import os
import unittest

spec = importlib.util.spec_from_file_location(
"chaos", os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "scripts", "chaos.py"))
chaos = importlib.util.module_from_spec(spec)
spec.loader.exec_module(chaos)

# Re-apply after a power cut on Linux (#39): the table was created, Terraform died before saving it.
LOST = """aws_dynamodb_table.items: Creating...

Error: creating AWS DynamoDB Table (items): operation error DynamoDB: CreateTable, https response error StatusCode: \
400, RequestID: 0I74K6Z35KIT7JPLR7549KZUQ2IOL6CMF8DK69TNEGVUR8EJGCU9, ResourceInUseException: Table already exists: items

with aws_dynamodb_table.items,
on main.tf line 51, in resource "aws_dynamodb_table" "items":
51: resource "aws_dynamodb_table" "items" {

"""
REAL = """
Error: creating IAM Role (app): operation error IAM: CreateRole, https response error StatusCode: 400, \
MalformedPolicyDocument: invalid principal

with aws_iam_role.app,
on main.tf line 70, in resource "aws_iam_role" "app":
"""


class Orphans(unittest.TestCase):
def test_already_exists_errors_are_resources_terraform_lost(self):
self.assertEqual(chaos.orphans(LOST), ["aws_dynamodb_table.items"])

def test_any_other_error_is_a_real_failure(self):
self.assertIsNone(chaos.orphans(LOST + REAL))
self.assertIsNone(chaos.orphans("Apply complete!"))

def test_planned_addresses(self):
plan = " # aws_dynamodb_table.items will be created\n # aws_dynamodb_table_item.seed will be created\n"
self.assertEqual(chaos.PLANNED.findall(plan), ["aws_dynamodb_table.items", "aws_dynamodb_table_item.seed"])


if __name__ == "__main__":
unittest.main()
Loading