From 41df72d299292baf11faffcd49e01225180c8bc8 Mon Sep 17 00:00:00 2001 From: Santi Legaristi Date: Fri, 13 Nov 2026 01:34:01 +0100 Subject: [PATCH] fix(proxy): do not execute writes whose client gave up while waiting A write that waited for the restore of the emulator (or behind a slow write) was executed even if its client had timed out and retried, creating the resource behind Terraform's back. Check the connection first. Adds scripts/chaos.py (kill the emulator, keepstack or everything mid-apply) with a nightly workflow, and an e2e regression test. Refs #34 --- .github/workflows/chaos.yml | 37 ++++++++++ .github/workflows/ci.yml | 2 +- CHANGELOG.md | 10 +++ CONTRIBUTING.md | 2 +- docs/how-it-works.md | 20 +++++- scripts/chaos.py | 140 ++++++++++++++++++++++++++++++++++++ src/keepstack/proxy.py | 27 ++++++- tests/test_e2e_crash.py | 88 +++++++++++++++++++++++ tests/test_unit.py | 45 ++++++++++++ 9 files changed, 366 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/chaos.yml create mode 100644 scripts/chaos.py create mode 100644 tests/test_e2e_crash.py diff --git a/.github/workflows/chaos.yml b/.github/workflows/chaos.yml new file mode 100644 index 0000000..54351c2 --- /dev/null +++ b/.github/workflows/chaos.yml @@ -0,0 +1,37 @@ +name: Chaos + +# Kills the emulator, keepstack, or everything at once in the middle of `terraform apply`, and requires Terraform +# to converge afterwards (scripts/chaos.py). Too slow for every pull request: nightly and on demand. +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + iterations: + description: Iterations (each mode is used in turn) + default: "9" + +permissions: + contents: read + +jobs: + chaos: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - uses: hashicorp/setup-terraform@v4 + with: + terraform_wrapper: false + - name: Chaos + env: + ITERATIONS: ${{ inputs.iterations || '9' }} + run: python scripts/chaos.py --iterations "$ITERATIONS" --report chaos-report.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: chaos-report + path: chaos-report.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf102b7..6fc5200 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,4 +75,4 @@ jobs: - name: End-to-end tests env: PYTHONPATH: src - run: python -m unittest tests.test_e2e tests.test_e2e_daemon tests.test_e2e_compact tests.test_e2e_shims tests.test_e2e_dashboard tests.test_e2e_validate -v + run: python -m unittest tests.test_e2e tests.test_e2e_daemon tests.test_e2e_compact tests.test_e2e_shims tests.test_e2e_dashboard tests.test_e2e_validate tests.test_e2e_crash -v diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a673e..4695cdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +### Fixed +- A write that waited while keepstack restored the emulator (or behind a slow write), and whose client gave up and + retried meanwhile, was still executed afterwards: the resource was created behind Terraform's back and the retry + failed with "already exists" (`BucketAlreadyExists`, `ResourceInUseException`). keepstack now checks that the client + is still connected before running a request that waited. + +### Added +- `scripts/chaos.py` and a nightly workflow: kill the emulator, keepstack, or everything at once in the middle of + `terraform apply` and require Terraform to converge. Documented in [how it works](docs/how-it-works.md#crash-safety). + ## [0.3.0] - 2026-11-13 keepstack becomes a pull-request validator for Terraform: apply every change on the official AWS Emulator, on top of diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad48a25..5a49401 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ Terraform are missing. Useful variables: | Variable | Purpose | |---|---| | `TERRAFORM` | Path to a specific Terraform binary (default: `terraform` on `PATH`) | -| `KEEPSTACK_E2E_PORT` | Port for the main end-to-end run (default `9340`; the emulator uses the next one). The other suites use 9350, 9360 and 9380 (`KEEPSTACK_E2E_DAEMON_PORT`, `KEEPSTACK_E2E_COMPACT_PORT`, `KEEPSTACK_E2E_SHIMS_PORT`, 9390 for the dashboard and 9420-9423 for `validate`, `KEEPSTACK_E2E_VALIDATE_PORT`) | +| `KEEPSTACK_E2E_PORT` | Port for the main end-to-end run (default `9340`; the emulator uses the next one). The other suites use 9350, 9360 and 9380 (`KEEPSTACK_E2E_DAEMON_PORT`, `KEEPSTACK_E2E_COMPACT_PORT`, `KEEPSTACK_E2E_SHIMS_PORT`, 9390 for the dashboard, 9420-9423 for `validate`, `KEEPSTACK_E2E_VALIDATE_PORT`, and 9470 for crash safety, `KEEPSTACK_E2E_CRASH_PORT`) | | `TF_PLUGIN_CACHE_DIR` | Reuse downloaded providers between runs | Unit tests only (no Docker): `PYTHONPATH=src python -m unittest tests.test_unit tests.test_check tests.test_doctor tests.test_daemon tests.test_snapshot tests.test_compact tests.test_shims tests.test_dashboard tests.test_validate`. diff --git a/docs/how-it-works.md b/docs/how-it-works.md index a30b992..59b72b1 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -96,6 +96,23 @@ journal (`meta['shims']`) so the recorded calls replay the same way; `--reset` f to it (versioned S3 buckets are left alone). Anything shared with a surviving resource is kept. The result is replayed in a throwaway emulator before it replaces the journal, and the previous journal is saved as `pre-compact`. +## Crash safety + +The journal is written **before** a write reaches the emulator and confirmed only when the emulator answered 2xx, +in an SQLite transaction. Every way of dying mid-apply ends in a state that matches what Terraform was told: + +| What dies in the middle of `terraform apply` | What happens | +|---|---| +| **The emulator** (`docker kill`, Docker restart, OOM) | keepstack notices, starts a new emulator and replays the journal while new requests wait. The SDK's retries go through once the state is back: the apply usually finishes by itself. | +| **keepstack** (`kill -9`), restarted while Terraform retries | A write that was in flight is left unconfirmed and skipped on the next start (the client never got an answer, so Terraform does not have it either). Terraform's retries then succeed. | +| **Everything at once** (power cut) | The same, plus Terraform dies: run `terraform apply` again and it converges. | +| **The client gives up** while its request waits (restore, slow write) | The request is not executed: keepstack checks the connection before running a request that waited. Otherwise the resource would be created behind the client's back and its retry would fail with "already exists". | + +This is exercised by [`scripts/chaos.py`](../scripts/chaos.py), which kills one of the three at a random moment of +an apply of the quickstart example and requires the re-apply to converge and the plan to stay clean after another +restart. It runs nightly in CI ([`chaos.yml`](../.github/workflows/chaos.yml)); the deterministic case of the +abandoned write is an end-to-end test (`tests/test_e2e_crash.py`). + ## Limitations - **Heuristic.** A generated value that never appears in a write response cannot be learned, and very large responses @@ -103,7 +120,8 @@ journal (`meta['shims']`) so the recorded calls replay the same way; `--reset` f - **Ordering of concurrent consumers.** `ReceiveMessage` is journaled without the write lock (so a 20-second long poll does not block other writes). With several concurrent consumers the journal order may differ from the real one. - **Interrupted writes.** If keepstack dies after the emulator applied a write but before the journal confirmed it, - that write is skipped on replay (and reported). + that write is skipped on replay (and reported); the client never got an answer, so it retries or reports an error. + A write the client abandoned after the emulator started running it cannot be undone (as on AWS). - **Side effects run again.** Lambda `Invoke`, SNS → SQS deliveries and anything the emulator runs internally happen again when the calls that caused them are replayed. - **In-memory bodies.** Requests and responses are buffered (no streaming). Tested with 20 MB multipart S3 uploads. diff --git a/scripts/chaos.py b/scripts/chaos.py new file mode 100644 index 0000000..3d996d6 --- /dev/null +++ b/scripts/chaos.py @@ -0,0 +1,140 @@ +"""Chaos test: kill things in the middle of `terraform apply` and check that keepstack recovers. + + python scripts/chaos.py --iterations 6 # TERRAFORM=/path/to/terraform if not on PATH + +Every iteration applies examples/quickstart on a fresh keepstack and, after a random delay, does one of: + + emulator `docker kill` the emulator (Docker restart, OOM): keepstack must restore it by itself + keepstack kill -9 keepstack and start it again a few seconds later, while Terraform keeps retrying + power kill Terraform, keepstack and the emulator at once, then `keepstack up` again (a power cut) + +Then it re-applies until Terraform succeeds (at most 3 times) and requires a clean plan, also after another clean +restart. Exit code 1 if any iteration does not converge. Needs Docker, Terraform and a few minutes per iteration. +""" +import argparse +import json +import os +import random +import shutil +import signal +import subprocess +import sys +import tempfile +import time + +ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +MODES = ("emulator", "keepstack", "power") + + +def run(cmd, env, cwd=None, timeout=900): + p = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, encoding="utf-8", errors="replace", + timeout=timeout, check=False) + return p.returncode, p.stdout + p.stderr + + +def kill_pid(pid): + if os.name == "nt": + subprocess.run(["taskkill", "/F", "/PID", str(pid)], capture_output=True, check=False) + else: + os.kill(pid, signal.SIGKILL) + + +def iteration(i, mode, delay, args, tf, env, work): + root = os.path.join(work, f"it{i}-{mode}") + os.makedirs(root) + shutil.copy(os.path.join(ROOT, "examples", "quickstart", "main.tf"), root) + data = os.path.join(root, "data") + + def ks(*a): + return run([sys.executable, "-m", "keepstack", *a, "--data-dir", data], env) + + up = ("up", "-d", "--shims", "--port", str(args.port), "--emulator-port", str(args.port + 1)) + rc, out = ks(*up) + assert rc == 0, out + rc, out = run([tf, "init", "-input=false", "-no-color"], env, cwd=root) + assert rc == 0, out + apply = subprocess.Popen([tf, "apply", "-auto-approve", "-input=false", "-no-color"], cwd=root, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", errors="replace") + time.sleep(delay) + with open(os.path.join(data, "run.json"), encoding="utf-8") as fh: + state = json.load(fh) + if mode == "emulator": + run(["docker", "kill", state["container"]], env) + elif mode == "keepstack": + kill_pid(state["pid"]) + time.sleep(random.uniform(2, 8)) + rc, out = ks(*up) + assert rc == 0, out + else: + apply.kill() + kill_pid(state["pid"]) + run(["docker", "kill", state["container"]], env) + first, _ = apply.communicate(timeout=900) + if mode == "power": + rc, out = ks(*up) + assert rc == 0, out + reapplies = [] + for _ in range(3): + rc, out = run([tf, "apply", "-auto-approve", "-input=false", "-no-color"], env, cwd=root) + reapplies.append(rc) + if rc == 0: + break + plan_rc, _ = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root) + ks("down") + ks(*up) + restart_rc, restart_plan = run([tf, "plan", "-detailed-exitcode", "-input=false", "-no-color"], env, cwd=root) + _, log = ks("logs") + ks("down") + result = {"iteration": i, "mode": mode, "kill_after_s": round(delay, 1), "first_apply_rc": apply.returncode, + "reapply_rcs": reapplies, "plan_rc": plan_rc, "plan_after_restart_rc": restart_rc, + "unconfirmed_skipped": log.count("was never confirmed"), "abandoned": log.count("client gave up"), + "replay_failed": log.count("REPLAY FAILED")} + result["ok"] = reapplies[-1] == 0 and plan_rc == 0 and restart_rc == 0 + if not result["ok"]: + result["details"] = {"first_apply": first[-3000:], "last_reapply": out[-3000:], + "plan_after_restart": restart_plan[-3000:], "keepstack_log": log[-3000:]} + return result + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--iterations", type=int, default=6) + p.add_argument("--modes", default=",".join(MODES), help="comma-separated, used in turn") + p.add_argument("--seed", type=int, default=None, help="for repeatable delays") + p.add_argument("--min-delay", type=float, default=1.5, help="seconds after `terraform apply` starts") + p.add_argument("--max-delay", type=float, default=14) + p.add_argument("--port", type=int, default=9480) + p.add_argument("--report", help="write the JSON results here") + args = p.parse_args() + random.seed(args.seed) + tf = os.environ.get("TERRAFORM") or shutil.which("terraform") + if not tf: + sys.exit("terraform not found (PATH or TERRAFORM env var)") + env = dict(os.environ, PYTHONPATH=os.path.join(ROOT, "src"), PYTHONIOENCODING="utf-8", TF_IN_AUTOMATION="1", + AWS_ENDPOINT_URL=f"http://127.0.0.1:{args.port}", AWS_ACCESS_KEY_ID="test", + AWS_SECRET_ACCESS_KEY="test", AWS_REGION="us-east-1") + for k in ("AWS_SESSION_TOKEN", "AWS_PROFILE"): + env.pop(k, None) + modes = args.modes.split(",") + work = tempfile.mkdtemp(prefix="keepstack-chaos-") + results = [] + try: + for i in range(args.iterations): + mode = modes[i % len(modes)] + res = iteration(i, mode, random.uniform(args.min_delay, args.max_delay), args, tf, env, work) + results.append(res) + print(json.dumps({k: v for k, v in res.items() if k != "details"}), flush=True) + if not res["ok"]: + print(json.dumps(res["details"], indent=2), flush=True) + finally: + shutil.rmtree(work, ignore_errors=True) + if args.report: + with open(args.report, "w", encoding="utf-8") as fh: + json.dump(results, fh, indent=2) + passed = sum(r["ok"] for r in results) + print(f"chaos: {passed}/{len(results)} iterations converged") + return 0 if passed == len(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/keepstack/proxy.py b/src/keepstack/proxy.py index cde0358..6f58b7f 100644 --- a/src/keepstack/proxy.py +++ b/src/keepstack/proxy.py @@ -17,7 +17,9 @@ import os import random import re +import select import shutil +import socket import sqlite3 import ssl import subprocess @@ -44,6 +46,7 @@ TIMELIKE_RE = re.compile(rb"[0-9TZEe.+-]+") # dates, times, epochs, numbers (including 1.7E9) REQUEST_ID_KEYS = {b"requestid", b"request_id", b"x-amz-request-id", b"x-amzn-requestid"} READY_TIMEOUT = 300 +CLIENT_GONE = object() # App.handle() result: the client left, nothing to answer REPLAY_RETRY_SECONDS = 30 # replay is faster than AWS: e.g. PutItem before the table is ACTIVE MAX_ALIGN_TOKENS = 4000 # difflib is quadratic: above this, match by position only @@ -361,7 +364,7 @@ def send(self, method, path, headers, body, op=None): status, _, _, data = self.forward(*self.prepare(op, method, path, headers, body)) return status, shims.on_response(self, op, status, data) - def handle(self, method, path, headers, body): + def handle(self, method, path, headers, body, client_gone=lambda: False): if path.startswith(dashboard.PREFIX): # served by keepstack itself, never forwarded or journaled return dashboard.handle(self, method, path, headers) op = operation(path, headers, body) @@ -372,6 +375,12 @@ def handle(self, method, path, headers, body): if locked: self.lock.acquire() try: + # While this request waited (a restore, a slow write), its client may have given up and retried. + # Executing it now would create the resource behind the client's back: its retry then gets + # "already exists" and the resource never reaches the Terraform state. + if write and client_gone(): + log(f"client gave up waiting; not executing {op or method + ' ' + path}") + return CLIENT_GONE seq = None if locked: # Journal first, emulator second: if SQLite fails, the client gets an error @@ -405,6 +414,16 @@ def handle(self, method, path, headers, body): self.lock.release() +def socket_closed(sock): + """True when the peer closed its side, e.g. an SDK that timed out waiting and will retry. Bytes waiting to be + read (a pipelined request) mean it is still there.""" + try: + readable, _, _ = select.select([sock], [], [], 0) + return bool(readable) and sock.recv(1, socket.MSG_PEEK) == b"" + except (OSError, ValueError): # reset or already closed + return True + + def make_handler(app): class Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -449,10 +468,14 @@ def any(self): return self.reply(400, "Bad Request", [], b"keepstack: malformed request body") headers = {k: v for k, v in self.headers.items() if k.lower() not in HOP} try: - result = app.handle(self.command, self.path, headers, body) + result = app.handle(self.command, self.path, headers, body, + client_gone=lambda: socket_closed(self.connection)) except Exception as e: # noqa: BLE001 - return 502 instead of hanging the client log(f"error forwarding {self.command} {self.path}: {e}") result = 502, "Bad Gateway", [], f"keepstack: {e}".encode() + if result is CLIENT_GONE: + self.close_connection = True + return self.reply(*result) do_GET = do_POST = do_PUT = do_DELETE = do_HEAD = do_PATCH = do_OPTIONS = any diff --git a/tests/test_e2e_crash.py b/tests/test_e2e_crash.py new file mode 100644 index 0000000..363b892 --- /dev/null +++ b/tests/test_e2e_crash.py @@ -0,0 +1,88 @@ +"""E2E for crash safety (needs Docker). Env: KEEPSTACK_E2E_CRASH_PORT (default 9470; emulator = +1). + +Regression for #34: a write that waits while keepstack restores the emulator, and is abandoned by its client +meanwhile (an SDK that timed out and will retry), must not be executed afterwards. +""" +import http.client +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import unittest + +ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +PORT = int(os.environ.get("KEEPSTACK_E2E_CRASH_PORT", "9470")) +CHILD_ENV = dict(os.environ, PYTHONPATH=os.path.join(ROOT, "src"), PYTHONIOENCODING="utf-8") +AUTH = ("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20260101/us-east-1/sqs/aws4_request, " + "SignedHeaders=host;x-amz-date, Signature=" + "0" * 64) + + +def sh(*cmd, **kw): + return subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace", check=False, + timeout=kw.pop("timeout", 600), **kw) + + +def sqs(target, body, timeout=30): + c = http.client.HTTPConnection("127.0.0.1", PORT, timeout=timeout) + try: + c.request("POST", "/", json.dumps(body), {"X-Amz-Target": f"AmazonSQS.{target}", "Authorization": AUTH, + "Content-Type": "application/x-amz-json-1.0"}) + r = c.getresponse() + return r.status, r.read() + finally: + c.close() + + +class E2ECrash(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.mkdtemp(prefix="keepstack-e2e-crash-") + cls.data = os.path.join(cls.tmp, "data") + if sh("docker", "version", "--format", "{{.Server.Version}}").returncode != 0: + shutil.rmtree(cls.tmp, ignore_errors=True) + raise unittest.SkipTest("docker is not accessible") + sh("docker", "rm", "-f", f"keepstack-{PORT}") + + @classmethod + def tearDownClass(cls): + cls.ks("down") + sh("docker", "rm", "-f", f"keepstack-{PORT}") + shutil.rmtree(cls.tmp, ignore_errors=True) + + @classmethod + def ks(cls, *args, timeout=600): + return sh(sys.executable, "-m", "keepstack", args[0], *args[1:], "--data-dir", cls.data, + cwd=ROOT, env=CHILD_ENV, timeout=timeout) + + def test_write_abandoned_during_restore_is_not_executed(self): + r = self.ks("up", "-d", "--port", str(PORT), "--emulator-port", str(PORT + 1), timeout=900) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + for i in range(200): # a journal long enough that restoring it outlasts the client's patience + self.assertEqual(sqs("CreateQueue", {"QueueName": f"filler-{i}"})[0], 200) + sh("docker", "kill", f"keepstack-{PORT}") + time.sleep(1.5) # the watcher is restoring the emulator now; writes wait + with self.assertRaises((socket.timeout, TimeoutError), msg="restore finished too fast to test"): + sqs("CreateQueue", {"QueueName": "abandoned"}, timeout=1) + deadline = time.time() + 120 + while time.time() < deadline: # wait for the restore to finish + try: + status = sqs("GetQueueUrl", {"QueueName": "filler-199"}, timeout=5)[0] + if status == 200: + break + except OSError: + pass + time.sleep(1) + status, _ = sqs("GetQueueUrl", {"QueueName": "abandoned"}) + self.assertEqual(status, 400, "the abandoned CreateQueue was executed after its client left") + log = self.ks("logs").stdout + self.assertIn("client gave up waiting; not executing CreateQueue", log) + # The client retries, as an SDK would: now it goes through normally. + self.assertEqual(sqs("CreateQueue", {"QueueName": "abandoned"})[0], 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_unit.py b/tests/test_unit.py index 1dc4277..79b8064 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -1,9 +1,14 @@ import os +import socket import sys +import tempfile import time import unittest +from argparse import Namespace +from unittest import mock sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src")) +from keepstack import proxy from keepstack.proxy import IdMap, is_write, operation @@ -114,5 +119,45 @@ def test_collision_gets_unique_alias(self): self.assertEqual(m.to_client(b"igw-0000000000000008e"), alias) # stable +class TestAbandonedRequests(unittest.TestCase): # regression #34 + def test_socket_closed(self): + a, b = socket.socketpair() + self.addCleanup(a.close) + self.assertFalse(proxy.socket_closed(a)) # open, nothing to read + b.sendall(b"GET / HTTP/1.1") + self.assertFalse(proxy.socket_closed(a)) # pipelined bytes: still there + a.recv(100) + b.close() + self.assertTrue(proxy.socket_closed(a)) # the client gave up + + def app(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + app = proxy.App(Namespace(port=0, data_dir=tmp.name)) + app.ready.set() + return app + + def rows(self, app): + with proxy.connect(app.db_path) as db: + return db.execute("select count(*) from journal").fetchone()[0] + + def test_write_abandoned_while_waiting_is_not_executed(self): + app = self.app() + headers = {"X-Amz-Target": "AmazonSQS.CreateQueue"} + with mock.patch.object(app, "forward", return_value=(200, "OK", [], b"{}")) as forward: + self.assertIs(app.handle("POST", "/", headers, b'{"QueueName":"q"}', client_gone=lambda: True), + proxy.CLIENT_GONE) + forward.assert_not_called() + self.assertEqual(self.rows(app), 0) + status = app.handle("POST", "/", headers, b'{"QueueName":"q"}', client_gone=lambda: False)[0] + self.assertEqual((status, forward.call_count, self.rows(app)), (200, 1, 1)) + + def test_reads_are_answered_anyway(self): + app = self.app() + with mock.patch.object(app, "forward", return_value=(200, "OK", [], b"{}")) as forward: + app.handle("POST", "/", {"X-Amz-Target": "AmazonSQS.ListQueues"}, b"{}", client_gone=lambda: True) + forward.assert_called_once() + + if __name__ == "__main__": unittest.main()