From 3c3381f31adc17ff80e2982cda1f2f8f848c61dd Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 22 Aug 2026 16:54:09 -0700 Subject: [PATCH] fix: harden scheduled jobs and bind mounts Relative writable bind mounts resolve inside retained release directories and can lose state during pruning. Refuse them in favor of read-only release content or absolute host paths. Bound scheduled runs, serialize them with application operations, persist and notify failures, and expose their state through status. Closes #95 Closes #97 --- cmd/ob-docgen/main.go | 2 +- docs/onebox.run-v1.schema.json | 54 +++++- e2e/server_test.go | 70 ++++++++ e2e/testdata/postgres/ob.yml.tmpl | 17 ++ internal/app/constraints.go | 6 + internal/app/defaults.go | 16 +- internal/app/errors.go | 1 + internal/app/generate_test.go | 15 ++ internal/app/jsonschema.go | 24 +++ internal/app/load_test.go | 39 ++++- internal/app/names.go | 5 + internal/app/names_test.go | 2 +- internal/app/runtime.go | 13 +- internal/app/schedule.go | 7 +- internal/app/schedule_test.go | 36 +++- internal/app/testdata/contract-verdicts.json | 55 ++++-- .../testdata/corpus/ext-authentik-managed.yml | 6 +- .../app/testdata/corpus/ext-authentik.yml | 6 +- internal/app/types.go | 23 ++- internal/app/validate.go | 39 ++++- internal/engine/bootstrap_test.go | 4 +- internal/engine/finalize_test.go | 5 +- internal/engine/lock.go | 20 +++ internal/engine/lock_test.go | 27 +++ internal/engine/schedule.go | 103 +++++++++-- internal/engine/schedule_status.go | 110 ++++++++++++ internal/engine/schedule_test.go | 165 +++++++++++++++++- internal/engine/status.go | 14 ++ internal/engine/status_snapshot.go | 16 ++ internal/engine/status_snapshot_test.go | 37 +++- internal/notify/notify.go | 44 +++-- internal/onebox/staging_secrets_test.go | 2 +- site/public/onebox.run-v1.schema.json | 54 +++++- .../content/docs/guides/schedule-a-job.mdx | 48 ++++- site/src/content/docs/reference/errors.mdx | 1 + .../docs/reference/fields/notifications.mdx | 6 +- .../docs/reference/fields/workloads.mdx | 12 +- .../content/docs/reference/project-file.mdx | 28 ++- 38 files changed, 1036 insertions(+), 96 deletions(-) create mode 100644 internal/engine/schedule_status.go diff --git a/cmd/ob-docgen/main.go b/cmd/ob-docgen/main.go index 5cebacae..2d8f6c71 100644 --- a/cmd/ob-docgen/main.go +++ b/cmd/ob-docgen/main.go @@ -307,7 +307,7 @@ var blocks = []block{ Summary: "Named container registries and the local environment variables holding their credentials.", ReadWhen: []string{"Pulling from a private registry"}}, {Key: "notifications", Title: "notifications", Order: 100, Status: statusShipped, - Summary: "Named webhooks that receive selected operation outcomes.", + Summary: "Named webhooks that receive selected operation and scheduled-job outcomes.", ReadWhen: []string{"Sending deploy outcomes to Slack, Discord or an incident tool"}}, {Key: "backup_targets", Title: "backup_targets", Order: 200, Status: statusShipped, Summary: "User-owned off-host S3-compatible repositories a protected service writes its backups to. Executable for the postgres driver; every other driver refuses a policy rather than accepting one it cannot honour.", diff --git a/docs/onebox.run-v1.schema.json b/docs/onebox.run-v1.schema.json index 0e6cbe71..098148bf 100644 --- a/docs/onebox.run-v1.schema.json +++ b/docs/onebox.run-v1.schema.json @@ -1090,7 +1090,7 @@ "type": "array" }, "webhook": { - "description": "HTTP endpoint that receives operation notifications.", + "description": "HTTP endpoint that receives outcome notifications.", "examples": [ "https://hooks.example.com/onebox" ], @@ -1099,7 +1099,7 @@ }, "type": "object" }, - "description": "Named webhooks that receive selected operation outcomes.", + "description": "Named webhooks that receive selected operation and scheduled-job outcomes.", "type": "object" }, "port": { @@ -2412,11 +2412,16 @@ }, "schedule": { "additionalProperties": false, - "description": "Host-resident recurring schedule for a job.", + "description": "Host-resident recurring schedule and run policy for a job.", "patternProperties": { "^x-": {} }, "properties": { + "catch_up": { + "default": true, + "description": "Run once after the host returns if an elapsed schedule was missed while it was offline.", + "type": "boolean" + }, "cron": { "description": "Five-field cron schedule translated to a host timer. Expects five cron fields.", "examples": [ @@ -2425,6 +2430,15 @@ "pattern": "^[-0-9*/,A-Za-z ]+$", "type": "string" }, + "timeout": { + "default": "1h", + "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "30m" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + }, "timezone": { "default": "UTC", "description": "IANA timezone used to interpret the cron schedule. Expects an IANA zone name such as UTC or Europe/Berlin.", @@ -2458,9 +2472,33 @@ "type": "string" }, "volumes": { - "description": "Managed named volumes or repository bind mounts.", + "description": "Managed named volumes or bind mounts. Relative bind sources are read-only release content; absolute sources are external host state.", "items": { "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "source": { + "pattern": "^[^/]" + } + }, + "required": [ + "source" + ] + }, + "then": { + "properties": { + "mode": { + "const": "ro" + } + }, + "required": [ + "mode" + ] + } + } + ], "anyOf": [ { "required": [ @@ -2481,7 +2519,7 @@ "properties": { "mode": { "default": "rw", - "description": "Mount access mode: rw or ro.", + "description": "Mount access mode: rw or ro. A relative bind source requires ro.", "enum": [ "rw", "ro" @@ -2505,10 +2543,14 @@ "type": "string" }, "source": { - "description": "Repository-relative source path of a bind mount.", + "description": "Bind mount source. An absolute path is external host state that outlives releases. A dot-prefixed repository path is read-only release content removed by retention. Expects an absolute host path or a dot-prefixed path inside the repository, with no colon, control character or shell metacharacter.", "examples": [ "./config" ], + "not": { + "pattern": "(^|/)\\.\\.(/|$)" + }, + "pattern": "^(/[^\\x00-\\x1f'\"$`\\\\:]*|\\.(?:/[^\\x00-\\x1f'\"$`\\\\:]*)?)$", "type": "string" } }, diff --git a/e2e/server_test.go b/e2e/server_test.go index 6f368798..294c814b 100644 --- a/e2e/server_test.go +++ b/e2e/server_test.go @@ -182,6 +182,76 @@ func TestServerLifecycle(t *testing.T) { } }) + t.Run("scheduled jobs are bounded and failures reach status", func(t *testing.T) { + s.run(t, "systemd-analyze verify "+ + "/etc/systemd/system/ob-observer-chore.service "+ + "/etc/systemd/system/ob-observer-chore.timer "+ + "/etc/systemd/system/ob-observer-timeout--chore.service "+ + "/etc/systemd/system/ob-observer-timeout--chore.timer") + + // A normal host-fired run proves the generated runner, current-release + // lookup, Docker invocation and app-wide schedule lock compose on systemd. + s.run(t, "systemctl start ob-observer-chore.service") + if result := strings.TrimSpace(s.run(t, + "systemctl show ob-observer-chore.service --property=Result --value")); result != "success" { + t.Fatalf("normal scheduled run result = %q, want success", result) + } + + // The receiver lives on the target because host-fired notifications do + // too. It accepts one POST, records the body, and exits. + receiver := `from http.server import BaseHTTPRequestHandler, HTTPServer +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + size = int(self.headers.get("Content-Length", "0")) + with open("/tmp/onebox-schedule-notify", "wb") as out: + out.write(self.rfile.read(size)) + self.send_response(204) + self.end_headers() + def log_message(self, format, *args): + pass +HTTPServer(("127.0.0.1", 18080), Handler).handle_request() +` + encoded := base64.StdEncoding.EncodeToString([]byte(receiver)) + s.run(t, strings.Join([]string{ + "set -e", + "command -v python3 >/dev/null", + "systemctl stop ob-e2e-schedule-receiver.service >/dev/null 2>&1 || true", + "systemctl reset-failed ob-e2e-schedule-receiver.service >/dev/null 2>&1 || true", + "rm -f /tmp/onebox-schedule-notify", + "printf '%s' '" + encoded + "' | base64 -d > /tmp/onebox-schedule-receiver.py", + "systemd-run --quiet --collect --unit=ob-e2e-schedule-receiver /usr/bin/python3 /tmp/onebox-schedule-receiver.py", + "for i in $(seq 1 50); do ss -ltn | grep -q '127.0.0.1:18080' && break; sleep .1; done", + "ss -ltn | grep -q '127.0.0.1:18080'", + }, "\n")) + + // systemctl returns non-zero because TimeoutStartSec terminates the job. + if err := s.try(t, "systemctl start ob-observer-timeout--chore.service"); err == nil { + t.Fatal("wedged scheduled job was not terminated by its timeout") + } + if result := strings.TrimSpace(s.run(t, + "systemctl show ob-observer-timeout--chore.service --property=Result --value")); result != "timeout" { + t.Fatalf("timed-out scheduled run result = %q, want timeout", result) + } + out, err := s.ob(t, dir, "status") + if err == nil || !strings.Contains(out, "schedule timeout-chore") || !strings.Contains(out, "last run failed: timeout") { + t.Fatalf("status did not expose the scheduled failure (err=%v):\n%s", err, out) + } + notification := strings.TrimSpace(s.run(t, "cat /tmp/onebox-schedule-notify")) + for _, want := range []string{ + `"app":"observer"`, + `"verb":"scheduled job timeout-chore"`, + `"status":"fail"`, + `"error":"operation failed; inspect trusted local diagnostics"`, + } { + if !strings.Contains(notification, want) { + t.Fatalf("scheduled failure notification is missing %q: %s", want, notification) + } + } + // Do not make the deliberately induced failure pollute later lifecycle + // assertions; systemd resets Result to success with the failed state. + s.run(t, "systemctl reset-failed ob-observer-timeout--chore.service") + }) + t.Run("preflight", func(t *testing.T) { s.mustOb(t, dir, "preflight") }) diff --git a/e2e/testdata/postgres/ob.yml.tmpl b/e2e/testdata/postgres/ob.yml.tmpl index fe690d43..f22ba03f 100644 --- a/e2e/testdata/postgres/ob.yml.tmpl +++ b/e2e/testdata/postgres/ob.yml.tmpl @@ -39,6 +39,16 @@ workloads: image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 command: ["sh", "-c", "echo chore-ran"] data_effect: none + schedule: { cron: "0 0 1 1 *", timeout: 20s, catch_up: false } + # A deliberately wedged timer run proves the host-enforced timeout becomes a + # failed systemd result and that `ob status` exposes it. Its annual timer never + # fires during the suite; the test starts the service directly. + timeout-chore: + role: job + image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 + command: ["sh", "-c", "sleep 30"] + data_effect: none + schedule: { cron: "0 0 1 1 *", timeout: 1s, catch_up: false } deployment: order: [app] services: @@ -66,5 +76,12 @@ backup_targets: access_key_entry: BACKUP_ACCESS_KEY_ID secret_key_entry: BACKUP_SECRET_ACCESS_KEY encryption: { pitr: client-side } +notifications: + # The server test starts a one-shot receiver before inducing a scheduled-job + # timeout. Because the timer fires on the host, localhost is the guest itself. + schedule-failures: + webhook: http://127.0.0.1:18080/notify + on: [failure] + format: json proxy: managed: false diff --git a/internal/app/constraints.go b/internal/app/constraints.go index f0fb2ff5..2bf27151 100644 --- a/internal/app/constraints.go +++ b/internal/app/constraints.go @@ -65,6 +65,12 @@ var ( gAbsPath = grammar{"absolute path", regexp.MustCompile("^/[^\\x00-\\x1f'\"$`\\\\]*$"), "an absolute path with no control character or shell metacharacter"} + // Bind sources are rendered with Compose's short volume syntax, where a + // colon separates source, target, and mode. Accept both supported lifetime + // forms, but never a value that can add another field to that syntax. + gBindSource = grammar{"bind mount source", regexp.MustCompile("^(/[^\\x00-\\x1f'\"$`\\\\:]*|\\.(?:/[^\\x00-\\x1f'\"$`\\\\:]*)?)$"), + "an absolute host path or a dot-prefixed path inside the repository, with no colon, control character or shell metacharacter"} + gURLPath = grammar{"url path", regexp.MustCompile("^/[^\\x00-\\x1f'\"$` \\\\]*$"), "a path beginning with /"} diff --git a/internal/app/defaults.go b/internal/app/defaults.go index 1a89c7e6..94777c4b 100644 --- a/internal/app/defaults.go +++ b/internal/app/defaults.go @@ -103,9 +103,19 @@ func applyDefaults(p *Spec, raw map[string]any, derived map[string]Origin) { w.When = "manual" mark(path + ".when") } - if w.Schedule != nil && w.Schedule.Timezone == "" { - w.Schedule.Timezone = "UTC" - mark(path + ".schedule.timezone") + if w.Schedule != nil { + if w.Schedule.Timezone == "" { + w.Schedule.Timezone = "UTC" + mark(path + ".schedule.timezone") + } + if w.Schedule.Timeout == "" { + w.Schedule.Timeout = "1h" + mark(path + ".schedule.timeout") + } + if !stated(raw, path+".schedule.catch_up") { + w.Schedule.CatchUp = true + mark(path + ".schedule.catch_up") + } } for i := range w.Routes { rp := indexed(path+".routes", i) diff --git a/internal/app/errors.go b/internal/app/errors.go index ceeb7603..002b99c3 100644 --- a/internal/app/errors.go +++ b/internal/app/errors.go @@ -62,6 +62,7 @@ var errorCodes = map[string]string{ // Repository paths. "path_absolute": "a repository path may not be absolute", "path_escapes_repository": "a path resolves outside the project directory", + "path_parent_reference": "a bind source contains a parent-directory segment", "path_unresolvable": "a path could not be resolved", // Environment overrides. diff --git a/internal/app/generate_test.go b/internal/app/generate_test.go index 5d23cf33..8cb8593f 100644 --- a/internal/app/generate_test.go +++ b/internal/app/generate_test.go @@ -149,6 +149,21 @@ func TestRenderedRuntime(t *testing.T) { } } +func TestBindMountLifetimesRenderWithoutChangingTheirScope(t *testing.T) { + y := strings.Replace(appFixture, + " volumes: [{source: /data/postgres, path: /var/lib/postgresql/data}]\n", + " volumes: [{source: /data/postgres, path: /var/lib/postgresql/data}, {source: ./postgres.conf, path: /etc/postgres.conf, mode: ro}]\n", 1) + out := string(render(t, y)) + for _, want := range []string{ + "/data/postgres:/var/lib/postgresql/data", + "./postgres.conf:/etc/postgres.conf:ro", + } { + if !strings.Contains(out, want) { + t.Errorf("rendered runtime is missing bind mount %q\n%s", want, out) + } + } +} + // TestEnvFilesAreNotProjectedIntoDaemons is the rule seven real projects forced: // a database must not receive the application's secrets. func TestEnvFilesAreNotProjectedIntoDaemons(t *testing.T) { diff --git a/internal/app/jsonschema.go b/internal/app/jsonschema.go index 6fb13b3d..56b46327 100644 --- a/internal/app/jsonschema.go +++ b/internal/app/jsonschema.go @@ -350,6 +350,7 @@ var schemaConstraints = []struct { {[]string{"workloads", "*", "routes", "items", "middlewares", "items"}, pattern(gMiddlewareRef)}, {[]string{"workloads", "*", "volumes", "items", "name"}, pattern(gIdent)}, {[]string{"workloads", "*", "volumes", "items", "path"}, pattern(gAbsPath)}, + {[]string{"workloads", "*", "volumes", "items", "source"}, bindSourceConstraint()}, {[]string{"workloads", "*", "volumes", "items", "mode"}, enum(eMountMode)}, // A named volume says where it mounts, or it is a bind pair. Either way @@ -359,6 +360,22 @@ var schemaConstraints = []struct { map[string]any{"required": []any{"name", "path"}}, map[string]any{"required": []any{"source", "path"}}, }, + "allOf": []any{ + map[string]any{ + "if": map[string]any{ + "required": []any{"source"}, + "properties": map[string]any{ + "source": map[string]any{"pattern": "^[^/]"}, + }, + }, + "then": map[string]any{ + "required": []any{"mode"}, + "properties": map[string]any{ + "mode": map[string]any{"const": "ro"}, + }, + }, + }, + }, }}, {[]string{"workloads", "*", "published_ports", "items", "host"}, portBounds()}, {[]string{"workloads", "*", "published_ports", "items", "container"}, portBounds()}, @@ -367,6 +384,7 @@ var schemaConstraints = []struct { {[]string{"workloads", "*", "needs", "items", "condition"}, enum(eNeedCondition)}, {[]string{"workloads", "*", "schedule", "cron"}, pattern(gCron)}, {[]string{"workloads", "*", "schedule", "timezone"}, pattern(gTZ)}, + {[]string{"workloads", "*", "schedule", "timeout"}, pattern(gDur)}, {[]string{"services", "*", "driver"}, pattern(gIdent)}, {[]string{"services", "*", "persistence", "mode"}, enum(ePersistence)}, @@ -563,6 +581,12 @@ func appNameConstraint() map[string]any { return out } +func bindSourceConstraint() map[string]any { + out := pattern(gBindSource) + out["not"] = map[string]any{"pattern": `(^|/)\.\.(/|$)`} + return out +} + func pattern(g grammar) map[string]any { return map[string]any{"pattern": g.pattern.String(), "description": "Expects " + g.means + "."} } diff --git a/internal/app/load_test.go b/internal/app/load_test.go index 1cf1494d..4c53c29a 100644 --- a/internal/app/load_test.go +++ b/internal/app/load_test.go @@ -54,6 +54,8 @@ func conformanceCases() []conformanceCase { {"application with when", wl("w: {image: nginx, when: manual}"), false}, {"worker with schedule", wl("w: {image: nginx, role: worker, schedule: {cron: \"0 3 * * *\"}}"), false}, {"scheduled job", wl("j: {image: nginx, role: job, data_effect: none, schedule: {cron: \"0 4 * * *\"}}"), true}, + {"scheduled job run policy", wl("j: {image: nginx, role: job, data_effect: none, schedule: {cron: \"0 4 * * *\", timeout: 45m, catch_up: false}}"), true}, + {"scheduled job invalid timeout", wl("j: {image: nginx, role: job, data_effect: none, schedule: {cron: \"0 4 * * *\", timeout: forever}}"), false}, {"daemon role", wl("db: {image: postgres:16, role: daemon}"), true}, {"routes list", wl("w: {image: nginx, routes: [{domain: x, port: 4317, protocol: tcp, scheme: h2c}]}"), true}, {"provider-qualified route middlewares", wl("w: {image: nginx, routes: [{domain: x, port: 8080, middlewares: [auth@file, rate-limit@file]}]}") + "proxy: {config: traefik}\n", true}, @@ -97,7 +99,7 @@ func conformanceCases() []conformanceCase { // a managed volume read as holding nothing — and doctor, the backup gate // and the backup gate each guessed the same wrong way. {"volumes without persistence still load", wl("w: {image: nginx, volumes: [{name: data, path: /data}]}"), true}, - {"a bind mount is not durable", wl("w: {image: nginx, volumes: [{source: ./cfg, path: /etc/app}], replicas: 3}"), true}, + {"an absolute bind mount is external", wl("w: {image: nginx, volumes: [{source: /srv/app/cfg, path: /etc/app}], replicas: 3}"), true}, // Inference must not tighten a refusal against a project that loads. {"inferred durability does not refuse replicas", wl("w: {image: nginx, volumes: [{name: data, path: /data}], replicas: 3}"), true}, {"declared durability still refuses replicas", wl("w: {image: nginx, volumes: [{name: data, path: /data}], persistence: {mode: durable}, replicas: 3}"), false}, @@ -135,7 +137,12 @@ func conformanceCases() []conformanceCase { {"persistence external", wl("w: {image: nginx, persistence: {mode: external}}"), true}, {"volume scalar without a path", wl("w: {image: nginx, volumes: [data]}"), false}, {"volume scalar with a path", wl("w: {image: nginx, volumes: [{name: data, path: /data}]}"), true}, - {"bind mount volume", wl("w: {image: nginx, volumes: [{source: ./data, path: /data}]}"), true}, + {"relative read-only bind mount", wl("w: {image: nginx, volumes: [{source: ./config, path: /config, mode: ro}]}"), true}, + {"relative writable bind mount", wl("w: {image: nginx, volumes: [{source: ./data, path: /data}]}"), false}, + {"absolute writable bind mount", wl("w: {image: nginx, volumes: [{source: /srv/app/data, path: /data}]}"), true}, + {"relative bind escaping the release", wl("w: {image: nginx, volumes: [{source: ../../data, path: /data, mode: ro}]}"), false}, + {"bare bind source that Compose would treat as a named volume", wl("w: {image: nginx, volumes: [{source: config, path: /config, mode: ro}]}"), false}, + {"bind source containing a compose separator", wl("w: {image: nginx, volumes: [{source: '/srv/app:data', path: /data}]}"), false}, {"published udp port", wl("w: {image: nginx, published_ports: [{host: 8555, container: 8555, protocol: udp}]}"), true}, // A fixed host socket cannot coexist with two rolling replicas. Recreate // remains the explicit, valid way to publish it. @@ -194,6 +201,34 @@ func TestConformance(t *testing.T) { } } +func TestWritableBindMountsCannotUseReleaseStorage(t *testing.T) { + t.Run("relative source defaults to writable and is refused", func(t *testing.T) { + _, err := LoadBytes([]byte(wl("w: {image: nginx, volumes: [{source: ./data, path: /data}]}")), "ob.yml") + if err == nil { + t.Fatal("expected relative writable bind mount to be refused") + } + for _, want := range []string{"release-scoped", "mode: ro", "absolute source"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not contain %q", err, want) + } + } + }) + + t.Run("relative source is accepted when read-only", func(t *testing.T) { + _, err := LoadBytes([]byte(wl("w: {image: nginx, volumes: [{source: ./config, path: /config, mode: ro}]}")), "ob.yml") + if err != nil { + t.Fatalf("expected relative read-only bind mount to load: %v", err) + } + }) + + t.Run("absolute source remains writable external state", func(t *testing.T) { + _, err := LoadBytes([]byte(wl("w: {image: nginx, volumes: [{source: /srv/app/data, path: /data}]}")), "ob.yml") + if err != nil { + t.Fatalf("expected absolute writable bind mount to load: %v", err) + } + }) +} + // TestDefaultsMaterialise guards the CUE rule that cost a review round: a // default on an optional field never appears in output. func TestDefaultsMaterialise(t *testing.T) { diff --git a/internal/app/names.go b/internal/app/names.go index 272314a3..676b2dc1 100644 --- a/internal/app/names.go +++ b/internal/app/names.go @@ -228,6 +228,11 @@ func (n Names) ScheduledJobUnit(job string) string { return "ob-" + runtimeName(n.App, job) } +// ScheduleRunLock serializes host-fired jobs with every operation holding the +// application deploy lock. The file is only the kernel flock rendezvous; the +// existing application lock remains the durable owner/fencing record. +func (n Names) ScheduleRunLock() string { return path.Join(n.AppDir(), "schedule.lock") } + // ScheduledJobUnitPrefixes returns the current namespace followed by the // pre-2026.8.6 spelling when the application name contains a hyphen. func (n Names) ScheduledJobUnitPrefixes() []string { diff --git a/internal/app/names_test.go b/internal/app/names_test.go index 7f976fc4..7a21fae4 100644 --- a/internal/app/names_test.go +++ b/internal/app/names_test.go @@ -19,7 +19,7 @@ workloads: routes: - {domain: ledger.example.com, port: 8080} - {domain: api.ledger.example.com, port: 8080} - volumes: [{name: uploads, path: /var/lib/ledger/uploads}, {source: ./seed, path: /seed}] + volumes: [{name: uploads, path: /var/lib/ledger/uploads}, {source: ./seed, path: /seed, mode: ro}] worker: role: worker image: nginx diff --git a/internal/app/runtime.go b/internal/app/runtime.go index 6b27dc5d..6ab076c0 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -121,11 +121,11 @@ func (w Workload) IsJob() bool { return w.Role == RoleJob } // read an absent block as "not durable". A workload with a managed named // volume holds data that outlives the release whether or not it says so. // -// A bind mount is deliberately not durable here: onebox neither created the -// host path nor can tell configuration from data by looking at it, so the -// bytes are the operator's. Counting them would demand a backup report for -// every `./config` mount, and a warning that fires on everything is one -// nobody reads. +// A bind mount is deliberately not durable here. Absolute sources are external +// host state Onebox does not own. Relative sources are read-only release +// content, so they cannot hold changing application data. Counting either +// would demand a backup report for every `./config` mount, and a warning that +// fires on everything is one nobody reads. func (w Workload) HoldsDurableData() bool { if w.Persistence != nil { return w.Persistence.Mode == "durable" @@ -138,7 +138,8 @@ func (w Workload) HoldsDurableData() bool { return false } -// HasBindMounts reports whether any volume is a host path onebox does not own. +// HasBindMounts reports whether any volume mounts a host path rather than a +// Onebox-managed named volume. func (w Workload) HasBindMounts() bool { for _, v := range w.Volumes { if v.IsBind() { diff --git a/internal/app/schedule.go b/internal/app/schedule.go index c008fc8e..807a7017 100644 --- a/internal/app/schedule.go +++ b/internal/app/schedule.go @@ -27,6 +27,8 @@ type ScheduledJob struct { Name string Cron string Timezone string + Timeout string + CatchUp bool // Calendar is the host-side expression the cron translates to. Calendar string } @@ -48,7 +50,10 @@ func (p *Spec) ScheduledJobs() ([]ScheduledJob, error) { if tz == "" { tz = "UTC" } - out = append(out, ScheduledJob{Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal}) + out = append(out, ScheduledJob{ + Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal, + Timeout: w.Schedule.Timeout, CatchUp: w.Schedule.CatchUp, + }) } return out, nil } diff --git a/internal/app/schedule_test.go b/internal/app/schedule_test.go index a9f55f67..494b1ca8 100644 --- a/internal/app/schedule_test.go +++ b/internal/app/schedule_test.go @@ -78,7 +78,41 @@ workloads: if len(jobs) != 1 || jobs[0].Name != "prune" { t.Fatalf("only scheduled jobs belong here: %#v", jobs) } - if jobs[0].Timezone != "Europe/Berlin" || jobs[0].Calendar != "*-*-* 03:00:00" { + if jobs[0].Timezone != "Europe/Berlin" || jobs[0].Calendar != "*-*-* 03:00:00" || + jobs[0].Timeout != "1h" || !jobs[0].CatchUp { t.Fatalf("schedule lost its meaning: %#v", jobs[0]) } } + +func TestScheduledJobRunPolicyIsExplicitAndValidated(t *testing.T) { + spec, err := LoadBytes([]byte(`api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + prune: + role: job + image: x:1 + data_effect: none + schedule: {cron: "0 3 * * *", timeout: 20m, catch_up: false} +`), "ob.yml") + if err != nil { + t.Fatal(err) + } + jobs, err := spec.ScheduledJobs() + if err != nil { + t.Fatal(err) + } + if len(jobs) != 1 || jobs[0].Timeout != "20m" || jobs[0].CatchUp { + t.Fatalf("authored run policy was not preserved: %#v", jobs) + } + + bad := `api_version: onebox.run/v1 +app: shop +environments: {production: {server: root@h}} +workloads: + prune: {role: job, image: x:1, data_effect: none, schedule: {cron: "0 3 * * *", timeout: forever}} +` + if _, err := LoadBytes([]byte(bad), "ob.yml"); err == nil { + t.Fatal("an invalid scheduled-job timeout was accepted") + } +} diff --git a/internal/app/testdata/contract-verdicts.json b/internal/app/testdata/contract-verdicts.json index a323153d..5e69dbcd 100644 --- a/internal/app/testdata/contract-verdicts.json +++ b/internal/app/testdata/contract-verdicts.json @@ -1,9 +1,4 @@ [ - { - "case": "conformance/a bind mount is not durable", - "loads": true, - "digest": "99a55cca6736488c1c6303c5b2240e34663aada93da745b5fd20909735861bd6" - }, { "case": "conformance/a near-miss field name", "loads": false, @@ -24,6 +19,16 @@ "loads": false, "code": "project_invalid" }, + { + "case": "conformance/absolute writable bind mount", + "loads": true, + "digest": "bea3cdb441a30477ed644e73be3e092c8112c8c2ce56caddf971b05e1fa00c8c" + }, + { + "case": "conformance/an absolute bind mount is external", + "loads": true, + "digest": "504d5a4c8c933870dc003143d612c6c4c51aca662896f92277e6b259e84588e2" + }, { "case": "conformance/app starting ob-", "loads": false, @@ -79,15 +84,20 @@ "loads": false, "code": "project_invalid" }, + { + "case": "conformance/bare bind source that Compose would treat as a named volume", + "loads": false, + "code": "project_invalid" + }, { "case": "conformance/base_path absolute", "loads": true, "digest": "041fef4c505591809641cbaf261251631cfc8e6a4772cdbfed7b77fde493b962" }, { - "case": "conformance/bind mount volume", - "loads": true, - "digest": "58675aed6f40fe771ccea794590367f0aa0d29f74069f361d15bc30eba20d137" + "case": "conformance/bind source containing a compose separator", + "loads": false, + "code": "project_invalid" }, { "case": "conformance/components is not a field", @@ -329,6 +339,11 @@ "loads": true, "digest": "5a94c95c1cd277ff2b1d354167e64a1d8d6b836d532a29a968586a675769c190" }, + { + "case": "conformance/relative bind escaping the release", + "loads": false, + "code": "project_invalid" + }, { "case": "conformance/relative compose ref", "loads": true, @@ -339,6 +354,16 @@ "loads": true, "digest": "cfceac5e1d96c958a853553ab4816fcd9a7ac250797445e562faa889053a0abb" }, + { + "case": "conformance/relative read-only bind mount", + "loads": true, + "digest": "eac09a5a22979451fb16b96d252008b3451a7122c603890dbbd8f2299684ccbb" + }, + { + "case": "conformance/relative writable bind mount", + "loads": false, + "code": "project_invalid" + }, { "case": "conformance/repeated route middleware remains ordered", "loads": true, @@ -359,6 +384,16 @@ "loads": true, "digest": "fa32a001e109ec5b66168b204cb5966b9b58bdcc09fd421bfa6d93614cdb7cae" }, + { + "case": "conformance/scheduled job invalid timeout", + "loads": false, + "code": "project_invalid" + }, + { + "case": "conformance/scheduled job run policy", + "loads": true, + "digest": "fa32a001e109ec5b66168b204cb5966b9b58bdcc09fd421bfa6d93614cdb7cae" + }, { "case": "conformance/service backup policy", "loads": true, @@ -507,12 +542,12 @@ { "case": "corpus/ext-authentik-managed.yml", "loads": true, - "digest": "7525c3c11a6ecdf36e6d2609b89f3c891d65b208f4658434bfe8ba64cb1eedc7 postgres=dc4f8448b8b82b4a redis=d2660eeb4faa49fe" + "digest": "f402d2e878a34a9e496401c7533442934d3e37c720eb137459bc87aa4e090ad6 postgres=dc4f8448b8b82b4a redis=d2660eeb4faa49fe" }, { "case": "corpus/ext-authentik.yml", "loads": true, - "digest": "8595f20b0961f6e99e1d6b9cfd250466b83b1a9589bc9081a9b1ec7e4b87ba06" + "digest": "c2564766732163ee23bc7b0e8c03893a514fa68638d4e452e5e1260863958839" }, { "case": "corpus/ext-frigate.yml", diff --git a/internal/app/testdata/corpus/ext-authentik-managed.yml b/internal/app/testdata/corpus/ext-authentik-managed.yml index facfa66f..eba704a7 100644 --- a/internal/app/testdata/corpus/ext-authentik-managed.yml +++ b/internal/app/testdata/corpus/ext-authentik-managed.yml @@ -31,7 +31,7 @@ workloads: needs: [{name: postgres, condition: healthy}, {name: redis, condition: healthy}] published_ports: [{host: 9000, container: 9000}, {host: 9443, container: 9443}] volumes: - - {source: ./media, path: /media} + - {source: /srv/authentik/media, path: /media} - {source: ./custom-templates, path: /templates, mode: ro} health: {exec: "ak healthcheck", interval: 30s, start_period: 60s, retries: 10} worker: @@ -41,8 +41,8 @@ workloads: needs: [{name: postgres, condition: healthy}, {name: redis, condition: healthy}] volumes: - {source: /var/run/docker.sock, path: /var/run/docker.sock} - - {source: ./media, path: /media} - - {source: ./certs, path: /certs} + - {source: /srv/authentik/media, path: /media} + - {source: /srv/authentik/certs, path: /certs} services: postgres: "16" redis: "7.4" diff --git a/internal/app/testdata/corpus/ext-authentik.yml b/internal/app/testdata/corpus/ext-authentik.yml index 9cfecb50..c00a9491 100644 --- a/internal/app/testdata/corpus/ext-authentik.yml +++ b/internal/app/testdata/corpus/ext-authentik.yml @@ -13,7 +13,7 @@ workloads: env_files: [.env] published_ports: [{host: 9000, container: 9000}, {host: 9443, container: 9443}] volumes: - - {source: ./data, path: /data} + - {source: /srv/authentik/data, path: /data} - {source: ./custom-templates, path: /templates, mode: ro} worker: role: worker @@ -23,8 +23,8 @@ workloads: env_files: [.env] volumes: - {source: /var/run/docker.sock, path: /var/run/docker.sock} - - {source: ./data, path: /data} - - {source: ./certs, path: /certs} + - {source: /srv/authentik/data, path: /data} + - {source: /srv/authentik/certs, path: /certs} postgresql: role: daemon image: docker.io/library/postgres:16-alpine diff --git a/internal/app/types.go b/internal/app/types.go index c804a73d..58d6ce5c 100644 --- a/internal/app/types.go +++ b/internal/app/types.go @@ -51,7 +51,7 @@ type Spec struct { envDefault []EnvFile Hooks map[string]Command `json:"hooks,omitempty" description:"Lifecycle commands keyed by seam: bootstrap, pre_release, post_release, or post_deploy."` Checks Checks `json:"checks,omitzero" description:"Assertions that must pass before a release becomes current unless marked advisory."` - Notifications map[string]Notification `json:"notifications,omitempty" description:"Named webhooks that receive selected operation outcomes."` + Notifications map[string]Notification `json:"notifications,omitempty" description:"Named webhooks that receive selected operation and scheduled-job outcomes."` Registries map[string]Registry `json:"registries,omitempty" description:"Named container registries and the environment variables holding their credentials."` Proxy Proxy `json:"proxy" description:"Ownership and configuration of the host ingress proxy."` @@ -137,7 +137,7 @@ type Workload struct { Resources *Resources `json:"resources,omitempty" description:"Container memory and CPU limits."` Env map[string]any `json:"env,omitempty" description:"Literal container environment values. Managed-service credential variables cannot be overridden."` EnvFiles []EnvFile `json:"env_files,omitempty" description:"Workload-specific ordered environment-file list. Replaces broader defaults when present."` - Volumes []Volume `json:"volumes,omitempty" description:"Managed named volumes or repository bind mounts."` + Volumes []Volume `json:"volumes,omitempty" description:"Managed named volumes or bind mounts. Relative bind sources are read-only release content; absolute sources are external host state."` PublishedPorts []PublishedPort `json:"published_ports,omitempty" description:"Host ports published outside the proxy. They bind to loopback by default. A rolling workload cannot publish one, because two replicas cannot hold the same host port during a roll: set strategy: recreate, or route through the proxy instead."` Persistence *Persistence `json:"persistence,omitempty" description:"Declares whether this workload holds data that must outlive releases."` Needs []Need `json:"needs,omitempty" description:"Workload or supporting-service prerequisites and optional connection-variable mappings."` @@ -154,9 +154,9 @@ type Workload struct { Logging *Logging `json:"logging,omitempty" description:"Container logging driver and driver-specific options."` // Job only. - When string `json:"when,omitempty" description:"When a job runs: manual, pre_release, or post_release." default:"manual"` - DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` - Schedule *Schedule `json:"schedule,omitempty" description:"Host-resident recurring schedule for a job."` + When string `json:"when,omitempty" description:"When a job runs: manual, pre_release, or post_release." default:"manual"` + DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` + Schedule *JobSchedule `json:"schedule,omitempty" description:"Host-resident recurring schedule and run policy for a job."` } type Build struct { @@ -221,9 +221,9 @@ type Resources struct { type Volume struct { Name string `json:"name,omitempty" description:"Stable logical name of a Onebox-managed volume." example:"data"` Path string `json:"path,omitempty" description:"Absolute container path where the volume or bind mount is attached." example:"/var/lib/app"` - Source string `json:"source,omitempty" description:"Repository-relative source path of a bind mount." example:"./config"` + Source string `json:"source,omitempty" description:"Bind mount source. An absolute path is external host state that outlives releases. A dot-prefixed repository path is read-only release content removed by retention." example:"./config"` - Mode string `json:"mode" description:"Mount access mode: rw or ro." default:"rw"` + Mode string `json:"mode" description:"Mount access mode: rw or ro. A relative bind source requires ro." default:"rw"` } func (v Volume) IsBind() bool { return v.Source != "" } @@ -252,6 +252,13 @@ type Schedule struct { Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` } +type JobSchedule struct { + Cron string `json:"cron" description:"Five-field cron schedule translated to a host timer." example:"0 2 * * *"` + Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` + Timeout string `json:"timeout" description:"Maximum wall time for one scheduled run before systemd terminates it and records failure." default:"1h" example:"30m"` + CatchUp bool `json:"catch_up" description:"Run once after the host returns if an elapsed schedule was missed while it was offline." default:"true"` +} + type Service struct { Driver string `json:"driver,omitempty" description:"Built-in service driver. Defaults to the service map key." example:"postgres"` Version any `json:"version" description:"Driver version or image tag to run." example:"17"` @@ -457,7 +464,7 @@ type MigrationRevs struct { } type Notification struct { - Webhook string `json:"webhook" description:"HTTP endpoint that receives operation notifications." example:"https://hooks.example.com/onebox"` + Webhook string `json:"webhook" description:"HTTP endpoint that receives outcome notifications." example:"https://hooks.example.com/onebox"` On []string `json:"on,omitempty" description:"Operation outcomes that trigger this notification." default:"success, failure"` Format string `json:"format" description:"Notification payload format." default:"text"` } diff --git a/internal/app/validate.go b/internal/app/validate.go index d1342b27..34ae86ec 100644 --- a/internal/app/validate.go +++ b/internal/app/validate.go @@ -321,6 +321,21 @@ func validateWorkload(w Workload, path string) error { return err } if v.IsBind() { + if err := gBindSource.check(vp+".source", v.Source); err != nil { + return err + } + for _, part := range strings.Split(v.Source, "/") { + if part == ".." { + return errf("path_parent_reference", vp+".source", "", + "bind source %q must not contain a parent-directory segment", v.Source) + } + } + if !strings.HasPrefix(v.Source, "/") { + if v.Mode != "ro" { + return errf("project_invalid", vp+".mode", "", + "relative bind source %q is release-scoped and removed by retention; set mode: ro for versioned release content, or use an absolute source for writable host state", v.Source) + } + } if err := gAbsPath.check(vp+".path", v.Path); err != nil { return err } @@ -399,7 +414,7 @@ func validateWorkload(w Workload, path string) error { return errf("project_invalid", path+".data_effect", "", "a job must declare its data effect: %s", strings.Join(quoteAll(eDataEffect), ", ")) } - if err := validateSchedule(w.Schedule, path+".schedule"); err != nil { + if err := validateJobSchedule(w.Schedule, path+".schedule"); err != nil { return err } } else if w.When != "" || w.DataEffect != "" || w.Schedule != nil { @@ -478,17 +493,31 @@ func validateResources(r *Resources, path string) error { return gCpus.checkOptional(path+".cpus", r.CPUs) } +func validateJobSchedule(s *JobSchedule, path string) error { + if s == nil { + return nil + } + if err := validateScheduleFields(s.Cron, s.Timezone, path); err != nil { + return err + } + return gDur.check(path+".timeout", s.Timeout) +} + func validateSchedule(s *Schedule, path string) error { if s == nil { return nil } - if err := gCron.check(path+".cron", s.Cron); err != nil { + return validateScheduleFields(s.Cron, s.Timezone, path) +} + +func validateScheduleFields(cron, timezone, path string) error { + if err := gCron.check(path+".cron", cron); err != nil { return err } - if len(strings.Fields(s.Cron)) != 5 { - return errf("project_invalid", path+".cron", "", "%q must contain exactly five cron fields", s.Cron) + if len(strings.Fields(cron)) != 5 { + return errf("project_invalid", path+".cron", "", "%q must contain exactly five cron fields", cron) } - return gTZ.checkOptional(path+".timezone", s.Timezone) + return gTZ.checkOptional(path+".timezone", timezone) } func indexed(path string, i int) string { diff --git a/internal/engine/bootstrap_test.go b/internal/engine/bootstrap_test.go index a6a4951a..a2971589 100644 --- a/internal/engine/bootstrap_test.go +++ b/internal/engine/bootstrap_test.go @@ -147,7 +147,7 @@ func TestConcurrentBootstrapDoesNotRunSecondHook(t *testing.T) { }() defer func() { _ = os.WriteFile(release, nil, 0o600) }() - deadline := time.Now().Add(3 * time.Second) + deadline := time.Now().Add(10 * time.Second) for { if _, err := os.Stat(entered); err == nil { break @@ -180,7 +180,7 @@ func TestConcurrentBootstrapDoesNotRunSecondHook(t *testing.T) { if err != nil { t.Fatalf("first bootstrap: %v", err) } - case <-time.After(3 * time.Second): + case <-time.After(10 * time.Second): t.Fatal("first bootstrap did not finish") } } diff --git a/internal/engine/finalize_test.go b/internal/engine/finalize_test.go index bf30c25f..00621d42 100644 --- a/internal/engine/finalize_test.go +++ b/internal/engine/finalize_test.go @@ -176,7 +176,10 @@ func scheduledJobFake() *transport.Fake { f := happyFake() base := f.Dynamic f.Dynamic = func(cmd string) (transport.Result, bool) { - if strings.Contains(cmd, "systemd-analyze calendar") { + switch { + case strings.Contains(cmd, "systemd-analyze calendar"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): return transport.Result{Stdout: "ok\n"}, true } return base(cmd) diff --git a/internal/engine/lock.go b/internal/engine/lock.go index b087105f..ca5e1489 100644 --- a/internal/engine/lock.go +++ b/internal/engine/lock.go @@ -65,6 +65,23 @@ func (e *Engine) AcquireLock(ctx context.Context, deployID string, force bool) ( b, _ := json.Marshal(meta) // noclobber: the remote shell refuses the redirect if the lock exists create := "set -C; echo " + q(string(b)) + " > " + q(e.lockPath()) + " 2>/dev/null" + jobs, scheduleErr := e.Spec.ScheduledJobs() + if scheduleErr != nil { + return 0, scheduleErr + } + useScheduleLock := e.hasFlock(ctx) + if len(jobs) > 0 && !useScheduleLock { + return 0, errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") + } + if useScheduleLock { + // A scheduled job holds this kernel lock for its whole container run. + // Take it around the atomic application-lock creation so neither side + // can pass its check before the other publishes ownership. Keep doing + // this after the last schedule is removed: an old unit may already be + // running while that removal deploy begins. + create = "/usr/bin/flock --exclusive --nonblock --conflict-exit-code 76 " + + q(e.names().ScheduleRunLock()) + " /bin/sh -c " + q(create) + } res, err := e.T.Run(ctx, create) if err != nil { @@ -83,6 +100,9 @@ func (e *Engine) AcquireLock(ctx context.Context, deployID string, force bool) ( } return epoch, nil } + if res.ExitCode == 76 { + return 0, fmt.Errorf("deploy lock held by a scheduled job — wait for the job to finish") + } // held — inspect holder + age hres, err := e.T.Run(ctx, "cat "+q(e.lockPath())+" 2>/dev/null || true") if err != nil { diff --git a/internal/engine/lock_test.go b/internal/engine/lock_test.go index 2b89d325..72f80e9a 100644 --- a/internal/engine/lock_test.go +++ b/internal/engine/lock_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/labstack/onebox/internal/app" "github.com/labstack/onebox/internal/proxy" "github.com/labstack/onebox/internal/release" "github.com/labstack/onebox/internal/transport" @@ -46,6 +47,32 @@ func TestAcquireLockHappyPath(t *testing.T) { } } +func TestAcquireLockSerializesWithScheduledJobs(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "/usr/bin/flock"): + return transport.Result{ExitCode: 76}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + _, err := e.AcquireLock(context.Background(), "R9", false) + if err == nil || !strings.Contains(err.Error(), "scheduled job") { + t.Fatalf("error = %v, want scheduled-job contention", err) + } + seq := strings.Join(f.Commands, "\n") + if !strings.Contains(seq, cfg.NamesFor("production").ScheduleRunLock()) || !strings.Contains(seq, "--conflict-exit-code 76") { + t.Fatalf("application lock was not created under the schedule mutex:\n%s", seq) + } +} + func TestReleaseLockRemovesOnlyOwnedToken(t *testing.T) { f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { if strings.Contains(cmd, "cat '/var/lib/ob/sample/epoch'") { diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 3c6e346a..7c215335 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/labstack/onebox/internal/app" + "github.com/labstack/onebox/internal/notify" ) // A scheduled job runs when nobody is watching, so it runs on the host's own @@ -78,6 +79,9 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { } wanted := map[string]bool{} + if len(jobs) > 0 && !e.hasFlock(ctx) { + return errors.New("scheduled jobs require flock on the target so they cannot overlap deployments; install util-linux and deploy again") + } for _, job := range jobs { unit := n.ScheduledJobUnit(job.Name) wanted[unit] = true @@ -96,8 +100,21 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { job.Name, expr, job.Cron, job.Timezone) } - service := scheduleServiceUnit(e.Spec.Name, job.Name, n.CurrentLink()) + runnerPath := "/etc/systemd/system/" + unit + ".run" + notifyPath := "/etc/systemd/system/" + unit + ".notify" + runner := scheduleRunnerScript(e.Spec.Name, job.Name, n.CurrentLink(), e.lockPath(), n.ScheduleRunLock()) + notifier, err := e.scheduleFailureNotifier(job.Name) + if err != nil { + return fmt.Errorf("job %s: cannot render its failure notifier: %w", job.Name, err) + } + service := scheduleServiceUnit(e.Spec.Name, job, runnerPath, notifyPath) timer := scheduleTimerUnit(e.Spec.Name, job) + if err := e.writeServiceFile(ctx, runnerPath, []byte(runner)); err != nil { + return fmt.Errorf("job %s: cannot install its runner: %w", job.Name, err) + } + if err := e.writeServiceFile(ctx, notifyPath, []byte(notifier)); err != nil { + return fmt.Errorf("job %s: cannot install its failure notifier: %w", job.Name, err) + } if err := e.writeServiceFile(ctx, "/etc/systemd/system/"+unit+".service", []byte(service)); err != nil { return fmt.Errorf("job %s: cannot install its unit: %w", job.Name, err) } @@ -146,25 +163,91 @@ func (e *Engine) SyncSchedules(ctx context.Context) error { // scheduleServiceUnit runs one job exactly as a release-phase job runs, through // the current release's runtime. -func scheduleServiceUnit(application, job, currentLink string) string { +func scheduleRunnerScript(application, job, currentLink, applicationLock, scheduleLock string) string { + run := "if [ -e " + q(applicationLock) + " ]; then " + + "echo 'onebox: an application operation holds the deploy lock' >&2; exit 75; fi; " + + "exec /usr/bin/docker compose -p " + q(application) + " -f " + q(currentLink+"/compose.yaml") + + " run --rm --no-deps " + q(job) + return strings.Join([]string{ + "#!/bin/sh", + "# Written by Onebox. Edits are overwritten on the next deploy.", + "set -eu", + "exec /usr/bin/flock --exclusive --nonblock --conflict-exit-code 75 " + + q(scheduleLock) + " /bin/sh -c " + q(run), + "", + }, "\n") +} + +func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, notifyPath string) string { return strings.Join([]string{ "[Unit]", - "Description=Onebox scheduled job " + job + " for " + application, + "Description=Onebox scheduled job " + job.Name + " for " + application, "# Written by Onebox. Edits are overwritten on the next deploy.", "After=docker.service", "Requires=docker.service", "", "[Service]", "Type=oneshot", - // --rm because a scheduled job leaves no container behind, and - // --no-deps because its prerequisites are already running; starting - // them here would duplicate the application beside itself. - fmt.Sprintf("ExecStart=/usr/bin/docker compose -p %s -f %s/compose.yaml run --rm --no-deps %s", - application, currentLink, job), + "ExecStart=/bin/sh " + runnerPath, + // ExecStopPost runs after start failures and timeouts and receives + // SERVICE_RESULT from systemd. The script is a no-op after success. + "ExecStopPost=/bin/sh " + notifyPath, + "TimeoutStartSec=" + job.Timeout, "", }, "\n") } +const scheduleNotificationTimestamp = "__ONEBOX_SCHEDULE_TIMESTAMP__" + +// scheduleFailureNotifier extends the existing notification contract to work +// fired directly by systemd. The generated file is mode 0600, keeping webhook +// tokens out of unit metadata, and every send is bounded and fail-open. +func (e *Engine) scheduleFailureNotifier(job string) (string, error) { + environment := e.Opts.Environment + if environment == "" { + environment = e.Spec.Env + } + lines := []string{ + "#!/bin/sh", + "# Written by Onebox. Edits are overwritten on the next deploy.", + "set -u", + `[ "${SERVICE_RESULT:-success}" = success ] && exit 0`, + } + var sends []string + for _, name := range sortedNames(e.Spec.Notifications) { + cfg := e.Spec.Notifications[name] + prepared, err := notify.Prepare(cfg, notify.Payload{ + App: e.Spec.Name, Env: environment, Host: e.T.Destination(), + Verb: "scheduled job " + job, Status: "fail", + Error: "scheduled job failed; inspect trusted host diagnostics", + TS: scheduleNotificationTimestamp, + }) + if err != nil { + return "", err + } + if prepared == nil { + continue + } + body := q(string(prepared.Body)) + if before, after, ok := strings.Cut(string(prepared.Body), scheduleNotificationTimestamp); ok { + body = q(before) + `"$ts"` + q(after) + } + curl := "curl --fail --silent --show-error --max-time 5 --request POST" + + " --header " + q("Content-Type: "+prepared.ContentType) + + " --header " + q("X-Title: "+prepared.Title) + + ` --data-binary "$body" ` + q(cfg.Webhook) + sends = append(sends, "(body="+body+"; if ! "+curl+"; then echo "+ + q("onebox: notification "+name+" failed")+" >&2; fi) &") + } + if len(sends) > 0 { + lines = append(lines, `ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')`) + lines = append(lines, sends...) + lines = append(lines, "wait || true") + } + lines = append(lines, "exit 0", "") + return strings.Join(lines, "\n"), nil +} + // calendarExpr is the one string both the host's validator and the installed // unit see, so the expression that was checked is the expression that runs. func calendarExpr(job app.ScheduledJob) string { @@ -188,7 +271,7 @@ func scheduleTimerUnit(application string, job app.ScheduledJob) string { "OnCalendar=" + calendarExpr(job), // A box that was off at 2am still runs the job when it comes back, // which is the behaviour anyone declaring a nightly job expects. - "Persistent=true", + fmt.Sprintf("Persistent=%t", job.CatchUp), "", "[Install]", "WantedBy=timers.target", @@ -283,7 +366,7 @@ func (e *Engine) RemoveSchedules(ctx context.Context) error { func (e *Engine) removeScheduleUnit(ctx context.Context, unit string) error { disable, disableErr := e.mutate(ctx, "systemctl disable --now "+unit+".timer >/dev/null") remove, removeErr := e.mutate(ctx, fmt.Sprintf( - "rm -f /etc/systemd/system/%s.timer /etc/systemd/system/%s.service", unit, unit)) + "rm -f /etc/systemd/system/%s.timer /etc/systemd/system/%s.service /etc/systemd/system/%s.run /etc/systemd/system/%s.notify", unit, unit, unit, unit)) var errs []error if disableErr != nil { errs = append(errs, fmt.Errorf("disable schedule %s: %w", unit, disableErr)) diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go new file mode 100644 index 00000000..0e26672b --- /dev/null +++ b/internal/engine/schedule_status.go @@ -0,0 +1,110 @@ +package engine + +import ( + "context" + "fmt" + "strconv" + "strings" +) + +// StatusSchedule is the host-observed state of one declared scheduled job. +// systemd keeps Result after a oneshot exits, so a failed or timed-out run stays +// visible until a later successful run clears it. +type StatusSchedule struct { + Name string `json:"name"` + Unit string `json:"unit"` + TimerState string `json:"timer_state"` + LastResult string `json:"last_result"` + LastExitStatus int `json:"last_exit_status,omitempty"` + Diverged bool `json:"diverged"` + Issues []string `json:"issues,omitempty"` +} + +type scheduleUnitObservation struct { + loadState string + activeState string + result string + exitStatus int +} + +func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) { + jobs, err := e.Spec.ScheduledJobs() + if err != nil { + return nil, err + } + if len(jobs) == 0 { + return []StatusSchedule{}, nil + } + + var commands []string + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + commands = append(commands, + "printf '%s\\n' "+q("@@"+job.Name+":service"), + "systemctl show "+q(unit+".service")+" --no-pager --property=LoadState --property=ActiveState --property=Result --property=ExecMainStatus", + "printf '%s\\n' "+q("@@"+job.Name+":timer"), + "systemctl show "+q(unit+".timer")+" --no-pager --property=LoadState --property=ActiveState", + ) + } + res, err := e.T.Run(ctx, strings.Join(commands, "\n")) + if err != nil { + return nil, err + } + if res.ExitCode != 0 { + return nil, fmt.Errorf("read scheduled-job state (exit %d): %s", res.ExitCode, strings.TrimSpace(res.Stderr)) + } + + observed := map[string]map[string]scheduleUnitObservation{} + name, kind := "", "" + values := map[string]string{} + flush := func() { + if name == "" || kind == "" { + return + } + exit, _ := strconv.Atoi(values["ExecMainStatus"]) + if observed[name] == nil { + observed[name] = map[string]scheduleUnitObservation{} + } + observed[name][kind] = scheduleUnitObservation{ + loadState: values["LoadState"], activeState: values["ActiveState"], + result: values["Result"], exitStatus: exit, + } + values = map[string]string{} + } + for _, line := range strings.Split(res.Stdout, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@@") { + flush() + marker := strings.TrimPrefix(line, "@@") + name, kind, _ = strings.Cut(marker, ":") + continue + } + if key, value, ok := strings.Cut(line, "="); ok { + values[key] = value + } + } + flush() + + statuses := make([]StatusSchedule, 0, len(jobs)) + for _, job := range jobs { + unit := e.names().ScheduledJobUnit(job.Name) + service := observed[job.Name]["service"] + timer := observed[job.Name]["timer"] + status := StatusSchedule{ + Name: job.Name, Unit: unit, TimerState: timer.activeState, + LastResult: service.result, LastExitStatus: service.exitStatus, + } + if timer.loadState != "loaded" || timer.activeState != "active" { + status.Issues = append(status.Issues, "timer is not active") + } + if service.loadState != "loaded" { + status.Issues = append(status.Issues, "service unit is not loaded") + } + if service.result != "" && service.result != "success" { + status.Issues = append(status.Issues, fmt.Sprintf("last run failed: %s (exit %d)", service.result, service.exitStatus)) + } + status.Diverged = len(status.Issues) > 0 + statuses = append(statuses, status) + } + return statuses, nil +} diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 1d308fa7..f08ec278 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -14,7 +14,7 @@ func TestSyncSchedulesRetainsManualScheduledJob(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ Role: app.RoleJob, When: "manual", DataEffect: "none", - Schedule: &app.Schedule{Cron: "0 2 * * *", Timezone: "UTC"}, + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := happyFake() base := f.Dynamic @@ -24,6 +24,8 @@ func TestSyncSchedulesRetainsManualScheduledJob(t *testing.T) { return transport.Result{}, true case strings.Contains(cmd, "systemd-analyze calendar"): return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true } return base(cmd) } @@ -37,6 +39,165 @@ func TestSyncSchedulesRetainsManualScheduledJob(t *testing.T) { t.Fatalf("manual scheduled job omitted %q:\n%s", want, seq) } } + artifacts := strings.Join(f.Inputs, "\n") + for _, want := range []string{"TimeoutStartSec=1h", "Persistent=true", "schedule.lock", "run --rm --no-deps", "nightly"} { + if !strings.Contains(artifacts, want) { + t.Errorf("installed schedule artifacts are missing %q:\n%s", want, artifacts) + } + } +} + +func TestScheduledJobUnitContract(t *testing.T) { + job := app.ScheduledJob{ + Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", + Calendar: "*-*-* 02:00:00", Timeout: "45m", CatchUp: false, + } + runner := scheduleRunnerScript("sample", job.Name, "/var/lib/ob/sample/current", + "/var/lib/ob/sample/lock", "/var/lib/ob/sample/schedule.lock") + service := scheduleServiceUnit("sample", job, + "/etc/systemd/system/ob-sample-nightly.run", + "/etc/systemd/system/ob-sample-nightly.notify") + timer := scheduleTimerUnit("sample", job) + + for _, want := range []string{ + "flock --exclusive --nonblock --conflict-exit-code 75 '/var/lib/ob/sample/schedule.lock'", + "/var/lib/ob/sample/lock", + "application operation holds the deploy lock", + "docker compose", + "/var/lib/ob/sample/current/compose.yaml", + "run --rm --no-deps", + "nightly", + } { + if !strings.Contains(runner, want) { + t.Errorf("runner is missing %q:\n%s", want, runner) + } + } + for _, want := range []string{ + "Type=oneshot", + "ExecStart=/bin/sh /etc/systemd/system/ob-sample-nightly.run", + "ExecStopPost=/bin/sh /etc/systemd/system/ob-sample-nightly.notify", + "TimeoutStartSec=45m", + } { + if !strings.Contains(service, want) { + t.Errorf("service is missing %q:\n%s", want, service) + } + } + for _, want := range []string{ + "OnCalendar=*-*-* 02:00:00 UTC", + "Persistent=false", + "WantedBy=timers.target", + } { + if !strings.Contains(timer, want) { + t.Errorf("timer is missing %q:\n%s", want, timer) + } + } + for _, artifact := range []string{runner, service, timer} { + if strings.Contains(artifact, "Restart=") { + t.Errorf("cron-shaped scheduled jobs must not retry implicitly:\n%s", artifact) + } + } +} + +func TestScheduledJobFailureNotifierUsesConfiguredWebhooks(t *testing.T) { + cfg := testConfig() + cfg.Notifications = map[string]app.Notification{ + "ops": { + Webhook: "https://hooks.example.com/secret-path", + On: []string{"failure"}, Format: "json", + }, + "success-only": { + Webhook: "https://hooks.example.com/success", + On: []string{"success"}, Format: "text", + }, + } + f := &transport.Fake{TargetName: "root@example.internal"} + e := New(cfg, testProject(t), f, Options{Environment: "production", Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleFailureNotifier("nightly") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `${SERVICE_RESULT:-success}`, + `ts=$(date -u`, + `"$ts"`, + "curl --fail --silent --show-error --max-time 5 --request POST", + "Content-Type: application/json", + "X-Title: sample scheduled job nightly", + "https://hooks.example.com/secret-path", + `"host":"root@example.internal"`, + `"status":"fail"`, + "wait || true", + } { + if !strings.Contains(script, want) { + t.Errorf("failure notifier is missing %q:\n%s", want, script) + } + } + for _, forbidden := range []string{ + "https://hooks.example.com/success", + scheduleNotificationTimestamp, + } { + if strings.Contains(script, forbidden) { + t.Errorf("failure notifier contains %q:\n%s", forbidden, script) + } + } +} + +func TestSyncSchedulesRefusesMissingFlockBeforeInstallingUnits(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "list-unit-files"): + return transport.Result{}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + err := e.SyncSchedules(context.Background()) + if err == nil || !strings.Contains(err.Error(), "install util-linux") { + t.Fatalf("error = %v, want actionable flock refusal", err) + } + if len(f.Inputs) != 0 { + t.Fatalf("unit files were written before the capability refusal: %d", len(f.Inputs)) + } +} + +func TestScheduleStatusSurfacesTheLastSystemdFailure(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@nightly:service +LoadState=loaded +ActiveState=failed +Result=timeout +ExecMainStatus=15 +@@nightly:timer +LoadState=loaded +ActiveState=active +`}, true + } + return transport.Result{}, false + }} + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + statuses, err := e.scheduleStatuses(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(statuses) != 1 || !statuses[0].Diverged || statuses[0].LastResult != "timeout" || statuses[0].LastExitStatus != 15 { + t.Fatalf("failed systemd result was not surfaced: %#v", statuses) + } + if !strings.Contains(strings.Join(statuses[0].Issues, "\n"), "last run failed") { + t.Fatalf("failure has no actionable issue: %#v", statuses[0]) + } } func TestRemoveSchedulesRemovesFilesAndReloadsAfterFailedDisable(t *testing.T) { @@ -55,7 +216,7 @@ func TestRemoveSchedulesRemovesFilesAndReloadsAfterFailedDisable(t *testing.T) { t.Fatalf("remove schedules error = %v", err) } seq := strings.Join(f.Commands, "\n") - if !strings.Contains(seq, "rm -f /etc/systemd/system/ob-sample-nightly.timer /etc/systemd/system/ob-sample-nightly.service") { + if !strings.Contains(seq, "rm -f /etc/systemd/system/ob-sample-nightly.timer /etc/systemd/system/ob-sample-nightly.service /etc/systemd/system/ob-sample-nightly.run /etc/systemd/system/ob-sample-nightly.notify") { t.Fatalf("disable failure stranded the unit files:\n%s", seq) } if !strings.Contains(seq, "systemctl daemon-reload") { diff --git a/internal/engine/status.go b/internal/engine/status.go index 1fab42f0..2f6c1d2c 100644 --- a/internal/engine/status.go +++ b/internal/engine/status.go @@ -28,6 +28,7 @@ func (e *Engine) Status(ctx context.Context) error { incS journal.Summary incFound bool px proxyRaw + schedules []StatusSchedule ) reads := []func() error{ @@ -44,6 +45,7 @@ func (e *Engine) Status(ctx context.Context) error { incS, incFound = s, true return nil }, + func() (err error) { schedules, err = e.scheduleStatuses(ctx); return }, } if managed { reads = append(reads, e.proxyReads(ctx, &px)...) @@ -144,6 +146,18 @@ func (e *Engine) Status(ctx context.Context) error { } fmt.Fprintf(e.Opts.Out, "service %-12s %s\n", acc, cs[0].health) } + for _, schedule := range schedules { + if schedule.Diverged { + diverged = true + e.ui.Println(fmt.Sprintf("schedule %-11s %s", schedule.Name, e.ui.Warn(strings.Join(schedule.Issues, "; ")+" ⚠"))) + continue + } + result := schedule.LastResult + if result == "" { + result = "not run yet" + } + fmt.Fprintf(e.Opts.Out, "schedule %-11s active; last: %s\n", schedule.Name, result) + } if managed { fmt.Fprintln(e.Opts.Out) diff --git a/internal/engine/status_snapshot.go b/internal/engine/status_snapshot.go index 2fda186c..74730fe4 100644 --- a/internal/engine/status_snapshot.go +++ b/internal/engine/status_snapshot.go @@ -27,6 +27,7 @@ type StatusSnapshot struct { RecordedRelease string `json:"recorded_release,omitempty"` Roles []StatusRole `json:"roles"` Services []StatusService `json:"services"` + Schedules []StatusSchedule `json:"schedules"` Incomplete *StatusIncomplete `json:"incomplete,omitempty"` Proxy *StatusProxy `json:"proxy,omitempty"` Diverged bool `json:"diverged"` @@ -143,6 +144,7 @@ func (e *Engine) StatusSnapshot(ctx context.Context) (StatusSnapshot, error) { Runner: e.Opts.Runner, Roles: make([]StatusRole, 0, len(e.Spec.ReleaseOrder())), Services: make([]StatusService, 0, len(e.Spec.ServiceNames())), + Schedules: []StatusSchedule{}, Complete: true, } @@ -152,6 +154,7 @@ func (e *Engine) StatusSnapshot(ctx context.Context) (StatusSnapshot, error) { inc journal.Summary incFound bool px proxyRaw + schedules []StatusSchedule ) reads := []statusSnapshotRead{ @@ -183,6 +186,13 @@ func (e *Engine) StatusSnapshot(ctx context.Context) (StatusSnapshot, error) { return nil }, }, + { + component: "schedules", + run: func() (err error) { + schedules, err = e.scheduleStatuses(ctx) + return err + }, + }, } proxyReadStart := len(reads) @@ -237,6 +247,12 @@ func (e *Engine) StatusSnapshot(ctx context.Context) (StatusSnapshot, error) { snapshot.Diverged = snapshot.Diverged || status.Diverged snapshot.Services = append(snapshot.Services, status) } + if reads[3].err == nil { + snapshot.Schedules = schedules + for _, status := range schedules { + snapshot.Diverged = snapshot.Diverged || status.Diverged + } + } if reads[2].err == nil && incFound { snapshot.Incomplete = makeStatusIncomplete(inc) diff --git a/internal/engine/status_snapshot_test.go b/internal/engine/status_snapshot_test.go index 5c0222b9..64cf30af 100644 --- a/internal/engine/status_snapshot_test.go +++ b/internal/engine/status_snapshot_test.go @@ -60,7 +60,7 @@ func TestStatusSnapshotCompleteAndJSONFriendly(t *testing.T) { if err := json.Unmarshal(b, &doc); err != nil { t.Fatalf("unmarshal: %v", err) } - for _, key := range []string{"app", "host", "captured_at", "recorded_release", "roles", "services", "diverged", "complete"} { + for _, key := range []string{"app", "host", "captured_at", "recorded_release", "roles", "services", "schedules", "diverged", "complete"} { if _, ok := doc[key]; !ok { t.Fatalf("JSON missing %q: %s", key, b) } @@ -123,6 +123,41 @@ func TestStatusSnapshotReportsObservedDivergenceAndIncompleteDeploy(t *testing.T } } +func TestStatusSnapshotIncludesScheduledJobFailure(t *testing.T) { + cfg := testConfig() + cfg.Workloads["nightly"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "none", + Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, + } + f := statusFake("R2", "R2") + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + if strings.Contains(cmd, "systemctl show") { + return transport.Result{Stdout: `@@nightly:service +LoadState=loaded +ActiveState=failed +Result=exit-code +ExecMainStatus=9 +@@nightly:timer +LoadState=loaded +ActiveState=active +`}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + snapshot, err := e.StatusSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if !snapshot.Complete || !snapshot.Diverged || len(snapshot.Schedules) != 1 { + t.Fatalf("scheduled failure was not included as observed divergence: %#v", snapshot) + } + if got := snapshot.Schedules[0]; !got.Diverged || got.LastResult != "exit-code" || got.LastExitStatus != 9 { + t.Fatalf("unexpected scheduled-job status: %#v", got) + } +} + func TestStatusSnapshotReadFailuresAreDeterministicWarnings(t *testing.T) { f := statusFake("R2", "R2") f.Err = func(cmd string) error { diff --git a/internal/notify/notify.go b/internal/notify/notify.go index 40d19502..1dff3f07 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -1,4 +1,4 @@ -// Package notify pushes operation outcomes to a webhook — the journals are +// Package notify prepares and pushes operation outcomes to a webhook — the journals are // write-only forensics; this is the page. One generic JSON POST per finished // mutating verb, carrying both structured fields and a human "text" line // (Slack-compatible; Discord/ntfy/generic consumers read the fields). @@ -37,6 +37,15 @@ type Payload struct { Text string `json:"text"` // human line, filled by Send } +// Request is the stable HTTP representation of one selected notification. +// Preparing it separately lets host-fired work reuse the exact payload contract +// without requiring a resident Onebox process. +type Request struct { + Body []byte + ContentType string + Title string +} + // event maps a payload to the config's `on` vocabulary. func (p Payload) event() string { if p.Status == "ok" { @@ -56,12 +65,11 @@ func (p Payload) text() string { return fmt.Sprintf("🚨 %s: %s FAILED on %s — %s", p.App, p.Verb, p.Host, p.Error) } -// Send fires the webhook if the payload's outcome is selected by cfg.On. An -// unset webhook and a filtered outcome are silent no-ops. Callers treat a -// returned error as a warning — never as the operation's result. -func Send(ctx context.Context, cfg app.Notification, p Payload) error { +// Prepare renders a selected payload without sending it. An unset webhook and +// a filtered outcome return nil so every caller shares the same event policy. +func Prepare(cfg app.Notification, p Payload) (*Request, error) { if cfg.Webhook == "" { - return nil + return nil, nil } selected := false for _, on := range cfg.On { @@ -71,7 +79,7 @@ func Send(ctx context.Context, cfg app.Notification, p Payload) error { } } if !selected { - return nil + return nil, nil } if p.TS == "" { p.TS = time.Now().UTC().Format(time.RFC3339) @@ -92,16 +100,30 @@ func Send(ctx context.Context, cfg app.Notification, p Payload) error { } else { b, err := json.Marshal(p) if err != nil { - return err + return nil, err } body = b } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.Webhook, bytes.NewReader(body)) + return &Request{ + Body: body, ContentType: contentType, + Title: strings.TrimSpace(p.App + " " + p.Verb), + }, nil +} + +// Send fires the webhook if the payload's outcome is selected by cfg.On. An +// unset webhook and a filtered outcome are silent no-ops. Callers treat a +// returned error as a warning — never as the operation's result. +func Send(ctx context.Context, cfg app.Notification, p Payload) error { + prepared, err := Prepare(cfg, p) + if err != nil || prepared == nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.Webhook, bytes.NewReader(prepared.Body)) if err != nil { return err } - req.Header.Set("Content-Type", contentType) - req.Header.Set("X-Title", strings.TrimSpace(p.App+" "+p.Verb)) + req.Header.Set("Content-Type", prepared.ContentType) + req.Header.Set("X-Title", prepared.Title) client := &http.Client{Timeout: timeout} res, err := client.Do(req) if err != nil { diff --git a/internal/onebox/staging_secrets_test.go b/internal/onebox/staging_secrets_test.go index e39a76ec..83868792 100644 --- a/internal/onebox/staging_secrets_test.go +++ b/internal/onebox/staging_secrets_test.go @@ -52,7 +52,7 @@ workloads: port: 3000 domain: shop.example.com volumes: - - {source: ., path: /app} + - {source: ., path: /app, mode: ro} env_files: - shared.env - {file: api.enc.env, provider: sops} diff --git a/site/public/onebox.run-v1.schema.json b/site/public/onebox.run-v1.schema.json index 0e6cbe71..098148bf 100644 --- a/site/public/onebox.run-v1.schema.json +++ b/site/public/onebox.run-v1.schema.json @@ -1090,7 +1090,7 @@ "type": "array" }, "webhook": { - "description": "HTTP endpoint that receives operation notifications.", + "description": "HTTP endpoint that receives outcome notifications.", "examples": [ "https://hooks.example.com/onebox" ], @@ -1099,7 +1099,7 @@ }, "type": "object" }, - "description": "Named webhooks that receive selected operation outcomes.", + "description": "Named webhooks that receive selected operation and scheduled-job outcomes.", "type": "object" }, "port": { @@ -2412,11 +2412,16 @@ }, "schedule": { "additionalProperties": false, - "description": "Host-resident recurring schedule for a job.", + "description": "Host-resident recurring schedule and run policy for a job.", "patternProperties": { "^x-": {} }, "properties": { + "catch_up": { + "default": true, + "description": "Run once after the host returns if an elapsed schedule was missed while it was offline.", + "type": "boolean" + }, "cron": { "description": "Five-field cron schedule translated to a host timer. Expects five cron fields.", "examples": [ @@ -2425,6 +2430,15 @@ "pattern": "^[-0-9*/,A-Za-z ]+$", "type": "string" }, + "timeout": { + "default": "1h", + "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "30m" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + }, "timezone": { "default": "UTC", "description": "IANA timezone used to interpret the cron schedule. Expects an IANA zone name such as UTC or Europe/Berlin.", @@ -2458,9 +2472,33 @@ "type": "string" }, "volumes": { - "description": "Managed named volumes or repository bind mounts.", + "description": "Managed named volumes or bind mounts. Relative bind sources are read-only release content; absolute sources are external host state.", "items": { "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "source": { + "pattern": "^[^/]" + } + }, + "required": [ + "source" + ] + }, + "then": { + "properties": { + "mode": { + "const": "ro" + } + }, + "required": [ + "mode" + ] + } + } + ], "anyOf": [ { "required": [ @@ -2481,7 +2519,7 @@ "properties": { "mode": { "default": "rw", - "description": "Mount access mode: rw or ro.", + "description": "Mount access mode: rw or ro. A relative bind source requires ro.", "enum": [ "rw", "ro" @@ -2505,10 +2543,14 @@ "type": "string" }, "source": { - "description": "Repository-relative source path of a bind mount.", + "description": "Bind mount source. An absolute path is external host state that outlives releases. A dot-prefixed repository path is read-only release content removed by retention. Expects an absolute host path or a dot-prefixed path inside the repository, with no colon, control character or shell metacharacter.", "examples": [ "./config" ], + "not": { + "pattern": "(^|/)\\.\\.(/|$)" + }, + "pattern": "^(/[^\\x00-\\x1f'\"$`\\\\:]*|\\.(?:/[^\\x00-\\x1f'\"$`\\\\:]*)?)$", "type": "string" } }, diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 5184831b..fed3a52a 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -1,7 +1,7 @@ --- title: Schedule a job -description: Host timers that survive a reboot, and the cron forms Onebox refuses. -summary: How to declare a scheduled job, why it runs from a host timer rather than a Onebox process, and which cron expressions are refused at load. +description: Bounded host timers, deploy serialization, failure status, and the cron forms Onebox refuses. +summary: How to declare a scheduled job, bound its run time, choose catch-up behavior, inspect failures, and understand which cron expressions are refused at load. sidebar: order: 4 read_when: @@ -19,7 +19,11 @@ workloads: data_effect: none needs: [postgres] volumes: [{name: backups, path: /backups}] - schedule: {cron: "0 2 * * *", timezone: Europe/Berlin} + schedule: + cron: "0 2 * * *" + timezone: Europe/Berlin + timeout: 45m + catch_up: true ``` ## It becomes a host timer @@ -32,6 +36,44 @@ That design decision has a visible consequence elsewhere: encrypted environment entries are decrypted into the release when it is staged and *stay* there, because a timer firing at 02:00 must resolve the values the deploy resolved. +Each run is a systemd oneshot with a wall-time limit. `timeout` defaults to +`1h`; when it expires, systemd terminates the run and records `timeout` as its +result. Set a longer duration for jobs that legitimately need it. + +`catch_up` defaults to `true`: if the host was off at the scheduled time, the +timer runs once after it returns. Set it to `false` for time-sensitive work that +should be skipped rather than run late. A failed run is not retried implicitly; +the next attempt is the next cron elapse. + +## It cannot overlap a deploy + +A host-fired job takes an application-wide kernel lock for its whole container +run. Every operation that establishes Onebox's fenced application lock takes +the same scheduling mutex while publishing its ownership. A timer that collides +fails visibly instead of modifying Docker beside a deploy. + +The target must provide `flock` (part of `util-linux` on supported Linux hosts). +Onebox refuses to install or run schedules when that serialization primitive is +missing. + +## Failures remain visible + +systemd retains the last oneshot result. `ob status` reads it alongside the +timer state; a non-success result or inactive timer is reported as divergence: + +```text +schedule nightly-dump last run failed: timeout (exit 15) ⚠ +``` + +Use `journalctl -u ob--.service` on the target for trusted diagnostic +output. Every configured notification selecting `failure` also receives a +bounded, fail-open POST when a host-fired run fails or times out. Onebox writes +the webhook handler root-only beside the unit, so credentials in webhook +paths do not appear in `ExecStart` or `ExecStopPost`. A delivery failure is +recorded in the unit's journal and never replaces the job's original result. +The webhook must be reachable from the target host; timer delivery uses its +network path, not the operator workstation's. + ## Cron is translated exactly, or refused ``` diff --git a/site/src/content/docs/reference/errors.mdx b/site/src/content/docs/reference/errors.mdx index bda83acc..94d8999b 100644 --- a/site/src/content/docs/reference/errors.mdx +++ b/site/src/content/docs/reference/errors.mdx @@ -70,6 +70,7 @@ command. | `override_unknown_workload` | an override names a workload the project does not declare | | `path_absolute` | a repository path may not be absolute | | `path_escapes_repository` | a path resolves outside the project directory | +| `path_parent_reference` | a bind source contains a parent-directory segment | | `path_unresolvable` | a path could not be resolved | | `preflight_env_incomplete` | an environment file is missing keys the project requires | | `prerequisite_has_no_health` | a wait for health names something with no health check | diff --git a/site/src/content/docs/reference/fields/notifications.mdx b/site/src/content/docs/reference/fields/notifications.mdx index e3a62f4c..c4ead909 100644 --- a/site/src/content/docs/reference/fields/notifications.mdx +++ b/site/src/content/docs/reference/fields/notifications.mdx @@ -1,7 +1,7 @@ --- title: "notifications" -description: "Named webhooks that receive selected operation outcomes." -summary: "Named webhooks that receive selected operation outcomes." +description: "Named webhooks that receive selected operation and scheduled-job outcomes." +summary: "Named webhooks that receive selected operation and scheduled-job outcomes." status: shipped generated: true sidebar: @@ -25,4 +25,4 @@ cannot drift from what `ob validate` accepts. | --- | --- | --- | --- | | `.format` | `text` · `json` | `text` | Notification payload format. | | `.on` | list of `success` · `failure` | `success, failure` | Operation outcomes that trigger this notification. | -| `.webhook` | string | — | HTTP endpoint that receives operation notifications. | +| `.webhook` | string | — | HTTP endpoint that receives outcome notifications. | diff --git a/site/src/content/docs/reference/fields/workloads.mdx b/site/src/content/docs/reference/fields/workloads.mdx index 57490b74..f7039573 100644 --- a/site/src/content/docs/reference/fields/workloads.mdx +++ b/site/src/content/docs/reference/fields/workloads.mdx @@ -19,7 +19,7 @@ cannot drift from what `ob validate` accepts. ## Fields on this page -`args` · `bind` · `build` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `interval` · `labels` · `logging` · `memory` · `middlewares` · `mode` · `name` · `needs` · `options` · `path` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` +`args` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `env` · `env_files` · `exec` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `image` · `init` · `interval` · `labels` · `logging` · `memory` · `middlewares` · `mode` · `name` · `needs` · `options` · `path` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retries` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` ## Reference @@ -88,18 +88,20 @@ cannot drift from what `ob validate` accepts. | `.routes[].protocol` | `http` · `tcp` | `http` | Routing protocol: http, tcp, or udp. | | `.routes[].scheme` | `http` · `https` · `h2c` | `http` | Backend connection scheme: http, https, h2c, tcp, or udp. | | `.routes[].tls` | `terminate` · `passthrough` · `none` | `terminate` | TLS handling: terminate, passthrough, or none. | -| `.schedule` | object | — | Host-resident recurring schedule for a job. | +| `.schedule` | object | — | Host-resident recurring schedule and run policy for a job. | +| `.schedule.catch_up` | boolean | `true` | Run once after the host returns if an elapsed schedule was missed while it was offline. | | `.schedule.cron` | string | — | Five-field cron schedule translated to a host timer. Expects five cron fields. | +| `.schedule.timeout` | string | `1h` | Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timezone` | string | `UTC` | IANA timezone used to interpret the cron schedule. Expects an IANA zone name such as UTC or Europe/Berlin. | | `.stdin_open` | boolean | — | Keep standard input open for the container. | | `.strategy` | `rolling` · `recreate` | — | Release strategy. Defaults to rolling only for an application workload with health; all other workloads default to recreate. | | `.tty` | boolean | — | Allocate a pseudo-TTY for the container. | | `.user` | string | — | User or UID used to run the container process. | -| `.volumes` | list | — | Managed named volumes or repository bind mounts. | -| `.volumes[].mode` | `rw` · `ro` | `rw` | Mount access mode: rw or ro. | +| `.volumes` | list | — | Managed named volumes or bind mounts. Relative bind sources are read-only release content; absolute sources are external host state. | +| `.volumes[].mode` | `rw` · `ro` | `rw` | Mount access mode: rw or ro. A relative bind source requires ro. | | `.volumes[].name` | string | — | Stable logical name of a Onebox-managed volume. Expects lower-case letters, digits and hyphens, starting with a letter, at most 40 characters. | | `.volumes[].path` | string | — | Absolute container path where the volume or bind mount is attached. Expects an absolute path with no control character or shell metacharacter. | -| `.volumes[].source` | string | — | Repository-relative source path of a bind mount. | +| `.volumes[].source` | string | — | Bind mount source. An absolute path is external host state that outlives releases. A dot-prefixed repository path is read-only release content removed by retention. Expects an absolute host path or a dot-prefixed path inside the repository, with no colon, control character or shell metacharacter. | | `.when` | `pre_release` · `post_release` · `manual` | `manual` | When a job runs: manual, pre_release, or post_release. | | `.working_dir` | string | — | Absolute working directory for the container process. Expects an absolute path with no control character or shell metacharacter. | diff --git a/site/src/content/docs/reference/project-file.mdx b/site/src/content/docs/reference/project-file.mdx index 527355ce..82713fb2 100644 --- a/site/src/content/docs/reference/project-file.mdx +++ b/site/src/content/docs/reference/project-file.mdx @@ -105,6 +105,30 @@ foreign resource already holding a derived name is refused, not adopted. **Layout** — `/var/lib/ob//releases/`, plus `current`, `journal`, and `services`. Configurable per environment with `base_path`. +### Bind-mount lifetimes + +A workload bind source has one of two lifetimes: + +```yaml +workloads: + web: + image: ghcr.io/acme/shop:1.4.0 + volumes: + # Versioned configuration shipped inside this release. + - {source: ./config, path: /etc/shop, mode: ro} + # Writable state managed outside the release store. + - {source: /srv/shop/uploads, path: /var/lib/shop/uploads} +``` + +A relative source begins with `./` (or is `.`) and resolves inside the release +directory. It must be read-only because normal `deployment.retain_releases` +cleanup removes that directory when the release expires. Use it for versioned +configuration, never changing data. + +An absolute source is external host state. Onebox mounts it but neither creates, +backs up nor removes it; provision and protect that path separately. For data +Onebox should own, prefer a managed named volume instead. + **The proxy** — if anything is routed, Onebox runs Traefik and writes its static configuration. Declare `proxy.config` to own that configuration instead. The external `proxy.network` may be changed, but `default` is reserved for the @@ -156,7 +180,9 @@ generates the runtime from it. Individual services can still be adopted with - A field is **added**, never repurposed. - A scalar form once accepted is accepted **permanently**. - A default may be added; an existing default's value **does not change**. -- A constraint is **not tightened** against a project that already loads. +- A constraint is **not tightened** against a project that already loads unless + accepting it can silently lose data or produce an ambiguous runtime. A safety + refusal names the migration required to make the project valid again. The JSON Schema published by `ob schema` is generated from the same declarations the loader enforces and is checked against the conformance corpus.