diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 9b2c046018..3fcb4eda49 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -50,6 +50,7 @@ jobs: wheels: ${{ steps.scope.outputs.wheels }} recovery: ${{ steps.scope.outputs.recovery }} ui: ${{ steps.scope.outputs.ui }} + podman: ${{ steps.scope.outputs.podman }} cold_models: ${{ steps.tiers.outputs.cold_models }} dense_models: ${{ steps.tiers.outputs.dense_models }} cache_models: ${{ steps.tiers.outputs.cache_models }} @@ -79,11 +80,12 @@ jobs: echo "clickhouse=$9" echo "recovery=${10}" echo "ui=${11}" + echo "podman=${12}" } >> "${GITHUB_OUTPUT}" } if [[ "${EVENT_NAME}" != "pull_request" && "${EVENT_NAME}" != "merge_group" ]]; then - write_scope true true true true true true true true true true true + write_scope true true true true true true true true true true true true exit 0 fi @@ -95,7 +97,7 @@ jobs: if [[ -z "${BASE_SHA}" || -z "${HEAD_SHA}" ]] || \ ! git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}" > "${changed_paths}"; then echo "Could not diff ${BASE_SHA:-}...${HEAD_SHA:-}; running every suite" >&2 - write_scope true true true true true true true true true true true + write_scope true true true true true true true true true true true true exit 0 fi @@ -144,7 +146,7 @@ jobs: && "${HEAD_REF}" == release-please--branches--* ]]; } && release_metadata_only; then echo "The change is only release metadata; skipping the suites" \ | tee -a "${GITHUB_STEP_SUMMARY}" - write_scope false false false false false false false false false false false + write_scope false false false false false false false false false false false false exit 0 fi @@ -158,6 +160,7 @@ jobs: compiler_differential=false recovery=false ui=false + podman=false broad=false changed=false # A change that only touches the website, root-level Markdown, the README demo GIFs or the @@ -204,6 +207,14 @@ jobs: recovery=true ;; esac + case "${path}" in + # The reference stack and the Docker launcher, which Podman serves through its + # Docker-compatible API. + # Every path here also selects the wheels the job installs. + deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|crates/sqlbuild-cli/src/workers/*|crates/sqlbuild-cli/src/server_config/*|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|.github/workflows/verify.yml) + podman=true + ;; + esac case "${path}" in scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|tests/goldens/*|tests/e2e/fixtures/*|website/examples/*|.github/workflows/verify.yml|Makefile) compiler_differential=true @@ -271,7 +282,7 @@ jobs: done < "${changed_paths}" if [[ "${changed}" != "true" ]]; then - write_scope true true true true true true true true true true true + write_scope true true true true true true true true true true true true exit 0 fi @@ -298,7 +309,8 @@ jobs: "${compiler_differential}" \ "${clickhouse}" \ "${recovery}" \ - "${ui}" + "${ui}" \ + "${podman}" # Pull requests and merge groups run the 3k-model compile jobs and the 1k dense job; the # nightly run and manual runs add 5k and 10k. Budgets and ratio gates are the same in both. @@ -1231,6 +1243,46 @@ jobs: - name: Verify packaged Compose deployment and operational recovery run: make test-compose-packaged + podman: + name: Packaged Compose lifecycle on Podman + needs: [changes, lockfile, wheels] + if: needs.changes.outputs.podman == 'true' && needs.changes.outputs.wheels == 'true' && needs.changes.outputs.full == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + env: + # The Docker CLI and its Compose plugin drive rootful Podman through its API socket. + DOCKER_HOST: unix:///run/podman/podman.sock + DOCKER_SOCKET: /run/podman/podman.sock + # Podman has no BuildKit API: buildx would build in its own container and never load the image. + DOCKER_BUILDKIT: "0" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-uv + with: + version: "0.12.3" + enable-cache: true + - name: Install Python test dependencies without rebuilding the packages + run: | + uv python install 3.12 + uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - uses: actions/download-artifact@v4 + with: + name: dist-wheel-linux-x86_64 + path: dist + - name: Install the same wheels used by the images + run: uv pip install --no-deps dist/*.whl + - name: Start the Podman API socket with group access for the runner + run: | + sudo systemctl start podman.socket + sudo chmod 0711 /run/podman + sudo chgrp "$(id -g)" /run/podman/podman.sock + sudo chmod 0660 /run/podman/podman.sock + podman --version + docker version --format 'Podman API {{.Server.Version}}' + docker compose version + - name: Verify packaged Compose deployment and operational recovery + run: make test-compose-packaged + ui: name: UI checks and unit tests needs: changes @@ -1296,6 +1348,7 @@ jobs: clickhouse, wheels, compose, + podman, ui, ] runs-on: ubuntu-24.04 @@ -1323,6 +1376,7 @@ jobs: CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }} WHEELS_RESULT: ${{ needs.wheels.result }} COMPOSE_RESULT: ${{ needs.compose.result }} + PODMAN_RESULT: ${{ needs.podman.result }} UI_SELECTED: ${{ needs.changes.outputs.ui }} UI_RESULT: ${{ needs.ui.result }} LABEL_ONLY: ${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify' }} @@ -1335,6 +1389,7 @@ jobs: POSTGRES_SELECTED: ${{ needs.changes.outputs.postgres }} CLICKHOUSE_SELECTED: ${{ needs.changes.outputs.clickhouse }} WHEELS_SELECTED: ${{ needs.changes.outputs.wheels }} + PODMAN_SELECTED: ${{ needs.changes.outputs.podman }} run: | if [[ "${LABEL_ONLY}" == "true" ]]; then echo "A label other than full-verify changed; this run verifies nothing" @@ -1371,7 +1426,8 @@ jobs: "${POSTGRES_RESULT}" \ "${CLICKHOUSE_RESULT}" \ "${WHEELS_RESULT}" \ - "${COMPOSE_RESULT}"; do + "${COMPOSE_RESULT}" \ + "${PODMAN_RESULT}"; do if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then echo "A selected verification suite did not pass: ${result}" >&2 exit 1 @@ -1398,7 +1454,8 @@ jobs: "${POSTGRES_SELECTED}:${POSTGRES_RESULT}" \ "${CLICKHOUSE_SELECTED}:${CLICKHOUSE_RESULT}" \ "${WHEELS_SELECTED}:${WHEELS_RESULT}" \ - "${WHEELS_SELECTED}:${COMPOSE_RESULT}"; do + "${WHEELS_SELECTED}:${COMPOSE_RESULT}" \ + "${PODMAN_SELECTED}:${PODMAN_RESULT}"; do if [[ "${pair%%:*}" == "true" && "${pair#*:}" != "success" ]]; then echo "A suite this change needs did not run to success: ${pair#*:}" >&2 exit 1 diff --git a/Cargo.lock b/Cargo.lock index 8ad8280259..2daa7a2e26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4251,6 +4251,7 @@ dependencies = [ "sqlbuild-rules", "tempfile", "tokio", + "toml", "uuid", ] diff --git a/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py new file mode 100644 index 0000000000..b419a1f814 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py @@ -0,0 +1,11 @@ +"""A job asks for more memory than its worker may have and fewer CPUs than the launcher default.""" + +from sqlbuild.orchestration import job, resources, schedule + +annual_orders = job( + "annual_orders", + select="orders", + description="Build orders.", + resources=resources(memory="2g", cpus=0.5), + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/clients/typescript/README.md b/clients/typescript/README.md index 107e957f3d..e21da1dd95 100644 --- a/clients/typescript/README.md +++ b/clients/typescript/README.md @@ -42,21 +42,21 @@ separate authentication contract, not an inferred privilege of this client. ## Authenticate through a trusted proxy Create a person with `sqb admin create-principal orders-reader`, then put the -returned `principal_id` in a server-owned JSON file. Subjects are exact ASCII -upstream identities, not inferred local usernames or email addresses: - -```json -{ - "origin": "https://control.example.com", - "trusted_peers": ["127.0.0.1"], - "identities": { - "external-orders-reader": "00000000-0000-0000-0000-000000000001" - } -} +returned `principal_id` in the `[trusted_proxy]` section of the server's +`server.toml`. Subjects are exact ASCII upstream identities, not inferred local +usernames or email addresses: + +```toml +[trusted_proxy] +origin = "https://control.example.com" +trusted_peers = ["127.0.0.1"] + +[trusted_proxy.identities] +external-orders-reader = "00000000-0000-0000-0000-000000000001" ``` Replace the example UUID with the created person's ID, then start -`sqb server --trusted-proxy-config proxy.json`. The proxy must terminate HTTPS, +`sqb server --config server.toml`. The proxy must terminate HTTPS, remove any client-supplied `X-SQB-User`, and replace it with the authenticated upstream subject. Restrict access to the HTTP backend to the configured proxy peers. Trusting loopback also trusts other local processes on that host; @@ -74,23 +74,22 @@ is inferred from the subject. Register an authorization-code client with your HTTPS identity provider, using `https://control.example.com/api/auth/oidc/callback` as its redirect URI. Create each person through the local control-store-owner administration path and put -their UUID and exact provider subject in a server-owned JSON configuration: - -```json -{ - "origin": "https://control.example.com", - "issuer": "https://identity.example.com", - "client_id": "orders-control", - "identities": { - "external-orders-reader": "00000000-0000-0000-0000-000000000001" - } -} +their UUID and exact provider subject in the `[oidc]` section of `server.toml`: + +```toml +[oidc] +origin = "https://control.example.com" +issuer = "https://identity.example.com" +client_id = "orders-control" + +[oidc.identities] +external-orders-reader = "00000000-0000-0000-0000-000000000001" ``` -Start `sqb server --oidc-config oidc.json` behind an HTTPS-terminating proxy that +Start `sqb server --config server.toml` behind an HTTPS-terminating proxy that preserves the configured Host and restricts access to the HTTP backend. For a confidential client, add `client_secret_env` naming the server environment -variable holding its secret; never put the secret in the JSON file. To trust a +variable holding its secret; never put the secret in `server.toml`. To trust a private provider CA, add `certificate_authority` pointing to a PEM bundle. Relative certificate paths resolve from the server process's working directory. TLS verification remains mandatory and upstream HTTP redirects are not followed. diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 4551cd1b55..b170804a09 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -6284,7 +6284,10 @@ export interface operations { }; read_location_health: { parameters: { - query?: never; + query?: { + /** @description Only this target's observations */ + target?: string; + }; header?: never; path: { location_id: string; diff --git a/compose.yml b/compose.yml index f9decab88a..9e80c298c6 100644 --- a/compose.yml +++ b/compose.yml @@ -2,7 +2,7 @@ name: ${COMPOSE_PROJECT_NAME:-sqlbuild} services: postgres: - image: postgres:17.6-alpine3.22 + image: docker.io/library/postgres:17.6-alpine3.22 environment: POSTGRES_USER: sqlbuild POSTGRES_DB: sqlbuild @@ -24,6 +24,8 @@ services: target: server args: PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + SQLBUILD_SOURCE: ${SQLBUILD_SOURCE:-wheels} + SQLBUILD_VERSION: ${SQLBUILD_VERSION:-} environment: - SQLBUILD_CONTROL_DATABASE_URL=${SQLBUILD_CONTROL_DATABASE_URL:?Set SQLBUILD_CONTROL_DATABASE_URL} - SQLBUILD_PLATFORM_WEBHOOK_URL @@ -31,8 +33,8 @@ services: volumes: - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock configs: - - source: launchers - target: /etc/sqlbuild/launchers.json + - source: server + target: /etc/sqlbuild/server.toml ports: - "${SQLBUILD_BIND_ADDRESS:-127.0.0.1}:${SQLBUILD_PORT:-8000}:8000" healthcheck: @@ -54,6 +56,8 @@ services: target: location args: PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + SQLBUILD_SOURCE: ${SQLBUILD_SOURCE:-wheels} + SQLBUILD_VERSION: ${SQLBUILD_VERSION:-} SQLBUILD_GIT_REVISION: ${SQLBUILD_GIT_REVISION:?Set SQLBUILD_GIT_REVISION} command: [location, serve, --server, "http://server:8000", --location-id, "${SQLBUILD_LOCATION_ID:?Set SQLBUILD_LOCATION_ID}", --target, development] environment: @@ -68,9 +72,20 @@ services: networks: [workers] configs: - launchers: + server: content: | - {"docker":[{"location_id":"${SQLBUILD_LOCATION_ID}","target":"development","server_url":"http://server:8000","max_workers":1,"socket":"/var/run/docker.sock","location_container":"${COMPOSE_PROJECT_NAME:-sqlbuild}-location","executable":"/usr/local/bin/sqb"}]} + [server] + host = "0.0.0.0" + port = 8000 + + [[launchers.docker]] + location = "orders" + target = "development" + server_url = "http://server:8000" + max_workers = 1 + socket = "/var/run/docker.sock" + location_container = "${COMPOSE_PROJECT_NAME:-sqlbuild}-location" + executable = "/usr/local/bin/sqb" volumes: control: diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index ea40fc2007..c91f5fb947 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -39,6 +39,7 @@ sqlbuild-process-ownership.workspace = true sqlbuild-recall.workspace = true sqlbuild-rules.workspace = true tokio.workspace = true +toml.workspace = true uuid.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 3a1f3946b5..fdf96c0602 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -127,7 +127,7 @@ "name": "dev", "parser": { "prog": "sqb dev", - "description": "Run the local control API, scheduler and reloadable code location. Authentication is disabled only on loopback; triggers start paused. No UI is served.", + "description": "Run the local control API, scheduler and reloadable code location. Authentication is disabled only on loopback; triggers start paused. The web UI is served at the same address.", "groups": [{"title": "positional arguments"}, {"title": "options"}], "actions": [ {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, @@ -168,8 +168,8 @@ "options": [], "dest": "location_command", "required": true, - "choices": ["publish", "serve", "runtime", "execute"], - "help": "publish one snapshot, watch and reload, or run the supervisor's stdio compiler", + "choices": ["publish", "serve", "health", "runtime", "execute"], + "help": "publish one snapshot, watch and reload, check health (exits 1 when the location is down or its last load failed), or run the supervisor's stdio compiler", "group": 0 }, { @@ -1043,15 +1043,22 @@ "--database-env" ], "dest": "control_database_env", - "default": "SQLBUILD_CONTROL_DATABASE_URL", "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", "group": 1 }, + { + "kind": "store", + "options": ["--config"], + "dest": "control_config", + "metavar": "PATH", + "help": "server.toml: [server], [retention], [oidc], [trusted_proxy] and [[launchers.docker]] or [[launchers.local]]; read at start-up, so a change needs a restart. A flag overrides the file", + "group": 1 + }, { "kind": "store", "options": ["--launchers"], "dest": "control_launchers", - "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "help": {"suppress": true}, "group": 1 }, { @@ -1060,7 +1067,6 @@ "--host" ], "dest": "control_host", - "default": "127.0.0.1", "help": "IP address to listen on (default: 127.0.0.1)", "group": 1 }, @@ -1070,7 +1076,6 @@ "--port" ], "dest": "control_port", - "default": 8000, "type": "int", "help": "TCP port, or 0 to choose a free port (default: 8000)", "group": 1 @@ -1087,7 +1092,6 @@ "options": ["--default-role"], "dest": "control_default_role", "choices": ["none", "viewer"], - "default": "none", "help": "ambient metadata access for authenticated people: none (default) or installation-wide viewer; never grants SQL, evidence, execution or administration", "group": 1 }, @@ -1097,7 +1101,6 @@ "--statistics-retention-days" ], "dest": "control_statistics_retention_days", - "default": 90, "type": "int", "help": "days of hourly warehouse statistics to keep; older samples are hidden and then deleted (default: 90)", "group": 1 @@ -1108,7 +1111,6 @@ "--audit-retention-days" ], "dest": "control_audit_retention_days", - "default": 400, "type": "int", "help": "days of security audit records to keep; older records are deleted once every configured audit webhook has received them and no retained run's retry needs them (default: 400)", "group": 1 @@ -1119,7 +1121,6 @@ "--snapshot-retention-count" ], "dest": "control_snapshot_retention_count", - "default": 10, "type": "int", "help": "published snapshots to keep per location and target; older snapshots are deleted unless a run, trigger, job or other retained record still references them (default: 10)", "group": 1 @@ -1130,7 +1131,6 @@ "--run-log-retention-days" ], "dest": "control_run_log_retention_days", - "default": 14, "type": "int", "help": "days a finished run keeps its log of events, statements, SQL and output; the run, its node results and history stay (default: 14)", "group": 1 @@ -1141,7 +1141,6 @@ "--run-log-retention-count" ], "dest": "control_run_log_retention_count", - "default": 200, "type": "int", "help": "finished runs per location and target that keep their logs; older runs lose theirs, whichever limit comes first (default: 200)", "group": 1 @@ -1150,14 +1149,14 @@ "kind": "store", "options": ["--trusted-proxy-config"], "dest": "control_trusted_proxy_config", - "help": "JSON origin, trusted_peers and explicit subject-to-person identities; proxy must replace X-SQB-User and terminate HTTPS; no provisioning or grants", + "help": {"suppress": true}, "group": 1 }, { "kind": "store", "options": ["--oidc-config"], "dest": "control_oidc_config", - "help": "JSON HTTPS origin/issuer, client_id and explicit subject-to-person identities; native code flow with PKCE; no provisioning. Beta: groups_claim names an ID-token claim and group_roles maps a group to a non-admin role, optionally in one location and target", + "help": {"suppress": true}, "group": 1 }, { @@ -1221,15 +1220,22 @@ "--database-env" ], "dest": "control_database_env", - "default": "SQLBUILD_CONTROL_DATABASE_URL", "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", "group": 1 }, + { + "kind": "store", + "options": ["--config"], + "dest": "control_config", + "metavar": "PATH", + "help": "server.toml: [server], [retention], [oidc], [trusted_proxy] and [[launchers.docker]] or [[launchers.local]]; read at start-up, so a change needs a restart. A flag overrides the file", + "group": 1 + }, { "kind": "store", "options": ["--launchers"], "dest": "control_launchers", - "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "help": {"suppress": true}, "group": 1 } ] diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index 191a753c36..6952afc14a 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -80,10 +80,10 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( expected_value: Value::Str("diff".to_owned()), }, ParsedValueTestCase { - description: "server grants no ambient role by default", + description: "server leaves the ambient role to server.toml when no option is given", argv: &["server"], attribute: "control_default_role", - expected_value: Value::Str("none".into()), + expected_value: Value::None, }, ParsedValueTestCase { description: "viewer metadata access requires an explicit server option", diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index b27b9e2f55..0df005b918 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -66,15 +66,16 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - return code; } let program: PathBuf = std::env::current_exe().unwrap_or_default(); - let Some(script) = beside(&program, PYTHON_CLI_SCRIPT) else { - report_missing_python_cli(&program); - return HANDOFF_FAILURE_EXIT_CODE; - }; + let script = beside(&program, PYTHON_CLI_SCRIPT); if let Ok(NativeParse::Invocation { namespace, .. }) = &native - && let Some(code) = run_location_invocation(namespace, &script) + && let Some(code) = run_location_invocation(namespace, script.as_deref()) { return code; } + let Some(script) = script else { + report_missing_python_cli(&program); + return HANDOFF_FAILURE_EXIT_CODE; + }; let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); if let Some((path, key)) = &record diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 6e85c29d77..e8494fd932 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -12,4 +12,5 @@ pub mod launcher; pub mod native_commands; pub mod person_login; pub mod replay; +pub(crate) mod server_config; pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 0986d05d67..73a73224a0 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -30,6 +30,7 @@ use crate::native_commands::models::{ ControlAdmin, CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, }; +use crate::server_config::models::{ServerConfig, ServerSection}; use sqlbuild_core::cost_command::constants::DESCENDING_ORDER; /// Whether `sqb review --json` failed to parse, which Python reports as a JSON document. @@ -247,19 +248,22 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result Result, ControlError> { if standalone { return Ok(None); } - let host: IpAddr = text(namespace, "control_host") - .unwrap_or_else(|| "127.0.0.1".into()) - .parse() - .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; + let host = match text(namespace, "control_host") { + Some(host) => host + .parse() + .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?, + None => server.host.unwrap_or(IpAddr::from([127, 0, 0, 1])), + }; let port = match namespace.get("control_port") { Some(Value::Int(value)) => value .parse::() .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, - _ => 8000, + _ => server.port.unwrap_or(8000), }; Ok(Some(SocketAddr::new(host, port))) } @@ -268,29 +272,53 @@ pub(crate) fn control_address( pub(crate) fn control_api_settings( namespace: &Namespace, standalone: bool, + config: &ServerConfig, ) -> Result<(ApiAuthentication, ServerFacts), ControlError> { - let statistics = match retention_setting(namespace, "statistics-retention-days")? { + let retention = &config.retention; + let statistics = match retention_setting( + namespace, + "statistics-retention-days", + ("statistics_days", retention.statistics_days), + )? { Some(days) => StatisticsRetention::days(days)?, None => StatisticsRetention::default(), }; - let audit = match retention_setting(namespace, "audit-retention-days")? { + let audit = match retention_setting( + namespace, + "audit-retention-days", + ("audit_days", retention.audit_days), + )? { Some(days) => AuditRetention::days(days)?, None => AuditRetention::default(), }; - let snapshots = match retention_setting(namespace, "snapshot-retention-count")? { + let snapshots = match retention_setting( + namespace, + "snapshot-retention-count", + ("snapshot_count", retention.snapshot_count), + )? { Some(count) => SnapshotRetention::count(count)?, None => SnapshotRetention::default(), }; let defaults = RunLogRetention::default(); let run_logs = RunLogRetention::new( - retention_setting(namespace, "run-log-retention-days")?.unwrap_or(defaults.days()), - retention_setting(namespace, "run-log-retention-count")?.unwrap_or(defaults.count()), + retention_setting( + namespace, + "run-log-retention-days", + ("run_log_days", retention.run_log_days), + )? + .unwrap_or(defaults.days()), + retention_setting( + namespace, + "run-log-retention-count", + ("run_log_count", retention.run_log_count), + )? + .unwrap_or(defaults.count()), )?; Ok(( - control_authentication(namespace)?, + control_authentication(namespace, config)?, ServerFacts { version: env!("CARGO_PKG_VERSION").into(), - scheduler: standalone || !flag(namespace, "control_no_scheduler"), + scheduler: scheduler_enabled(namespace, standalone, &config.server), audit, statistics, snapshots, @@ -299,8 +327,21 @@ pub(crate) fn control_api_settings( )) } -/// Zero and out-of-range values name the flag before any type-specific check runs. -fn retention_setting(namespace: &Namespace, flag: &str) -> Result, ControlError> { +/// `--no-scheduler` wins over `[server] scheduler`; the standalone scheduler always schedules. +pub(crate) fn scheduler_enabled( + namespace: &Namespace, + standalone: bool, + server: &ServerSection, +) -> bool { + standalone || !flag(namespace, "control_no_scheduler") && server.scheduler.unwrap_or(true) +} + +/// Zero and out-of-range values name the flag, or the `[retention]` key, before any type-specific check runs. +fn retention_setting( + namespace: &Namespace, + flag: &str, + (key, file): (&str, Option), +) -> Result, ControlError> { match namespace.get(&format!("control_{}", flag.replace('-', "_"))) { Some(Value::Int(value)) => match value.parse::() { Ok(setting) if setting > 0 => Ok(Some(setting)), @@ -308,21 +349,22 @@ fn retention_setting(namespace: &Namespace, flag: &str) -> Result, C "--{flag} must be between 1 and 65535" ))), }, - _ => Ok(None), + _ => match file { + Some(0) => Err(ControlError::Invalid(format!( + "[retention] {key} must be between 1 and 65535" + ))), + setting => Ok(setting), + }, } } -fn control_authentication(namespace: &Namespace) -> Result { - let password = text(namespace, "control_password_origin"); - let proxy = authentication_settings( - text(namespace, "control_trusted_proxy_config"), - ("trusted-proxy", "origin, trusted_peers and identities"), - )?; - let oidc = authentication_settings( - text(namespace, "control_oidc_config"), - ("OIDC", "origin, issuer, client_id and identities"), - )?; - let login = match (password, proxy, oidc) { +fn control_authentication( + namespace: &Namespace, + config: &ServerConfig, +) -> Result { + let password = text(namespace, "control_password_origin") + .or_else(|| config.server.password_origin.clone()); + let login = match (password, config.trusted_proxy.clone(), config.oidc.clone()) { (None, None, None) => HumanLogin::None, (Some(origin), None, None) => HumanLogin::Password(origin), (None, Some(settings), None) => HumanLogin::TrustedProxy(settings), @@ -342,33 +384,26 @@ fn control_authentication(namespace: &Namespace) -> Result( - path: Option, - (mechanism, fields): (&str, &str), -) -> Result, ControlError> { - path.map(|path| { - let document = std::fs::read_to_string(path).map_err(|_| { - ControlError::Invalid(format!("could not read {mechanism} configuration")) - })?; - serde_json::from_str(&document).map_err(|_| { - ControlError::Invalid(format!( - "{mechanism} configuration requires valid JSON with {fields}" - )) - }) - }) - .transpose() +/// The variable naming the control database: `--database-env`, then `[server] database_env`. +pub(crate) fn control_database_variable(namespace: &Namespace, server: &ServerSection) -> String { + text(namespace, "control_database_env") + .or_else(|| server.database_env.clone()) + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()) } -pub(crate) fn control_database(namespace: &Namespace) -> Result { - let variable = text(namespace, "control_database_env") - .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); - match std::env::var(&variable) { +pub(crate) fn control_database( + namespace: &Namespace, + variable: &str, +) -> Result { + match std::env::var(variable) { Ok(value) => Ok(value), Err(std::env::VarError::NotPresent) => Ok(PathBuf::from( text(namespace, "project_dir").unwrap_or_else(|| ".".into()), diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index ad0141c041..9fd9f3af51 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -61,7 +61,9 @@ use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; -use crate::native_commands::_helpers::decision::{control_database, flag, text}; +use crate::native_commands::_helpers::decision::{ + control_database, control_database_variable, text, +}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; use crate::native_commands::_helpers::paths::{location_database_files, project_location}; @@ -87,11 +89,12 @@ use crate::native_commands::models::{ MetadataReply, NativeCommand, NativeInvocation, ProjectLocation, ScanToken, }; use crate::native_commands::types::LocationPublication; +use crate::server_config::models::ServerSection; use crate::workers::main::docker_image_sources::docker_image_sources; use crate::workers::main::open_development_workers::open_development_workers; use crate::workers::main::open_workers::open_workers; use crate::workers::main::terminate_child::terminate_child; -use crate::workers::models::Workers; +use crate::workers::models::{Launchers, Workers}; /// Own the API, scheduler and outbound location as one development process tree. pub(crate) async fn run_development_roles( @@ -970,8 +973,12 @@ pub(crate) async fn run_control_action( /// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. pub(crate) async fn open_control_store( namespace: &Namespace, + server: Option<&ServerSection>, ) -> Result { - let url = control_database(namespace)?; + let url = control_database( + namespace, + &control_database_variable(namespace, server.unwrap_or(&ServerSection::default())), + )?; set_release_version(env!("CARGO_PKG_VERSION")); drop(write_output(&[( Stream::Stderr, @@ -983,16 +990,14 @@ pub(crate) async fn open_control_store( Ok(store) } -/// Decode privileged CLI configuration before handing process ownership to the workers domain. -pub(crate) fn open_control_workers( - namespace: &Namespace, +/// Hand the configured launchers to the workers domain, which resolves their locations in the store. +pub(crate) async fn open_control_workers( + (store, namespace): (&ControlStore, &Namespace), + (server, launchers): (&ServerSection, Launchers), + scheduler_enabled: bool, ) -> Result<(Workers, RunHandoff), ControlError> { - let workers = open_workers( - text(namespace, "control_launchers").as_deref(), - text(namespace, "control_database_env") - .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), - !flag(namespace, "control_no_scheduler"), - )?; + let variable = control_database_variable(namespace, server); + let workers = open_workers(store, launchers, (variable, scheduler_enabled)).await?; let images = docker_image_sources(&workers); Ok((workers, (images, adhoc_planner()))) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index 5eb26c8aeb..ad5721c6c9 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -31,7 +31,7 @@ use crate::native_commands::models::ControlAdmin; pub(crate) async fn run_admin(namespace: &Namespace) -> Result { let action = control_admin(namespace)?; - let store = open_control_store(namespace).await?; + let store = open_control_store(namespace, None).await?; let context = AuditContext { actor: "local-control-store-owner".into(), granting_role: "control-store-owner".into(), diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 32fdc07306..29f8802b4b 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -5,7 +5,9 @@ use sqlbuild_orchestration::platform::models::RetentionPolicies; use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; use sqlbuild_orchestration::scheduler::models::SchedulerState; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; use tokio::sync::watch; +use tokio::task::JoinSet; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; @@ -14,6 +16,8 @@ use crate::native_commands::_helpers::execution::{ open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; use crate::native_commands::_helpers::terminal::write_output; +use crate::server_config::main::read_server_config::read_server_config; +use crate::workers::models::Workers; async fn stopped(mut shutdown: watch::Receiver) { while !*shutdown.borrow_and_update() { @@ -27,12 +31,18 @@ pub(crate) async fn run_control( namespace: &Namespace, standalone: bool, ) -> Result<(), ControlError> { - let address = control_address(namespace, standalone)?; - let (authentication, facts) = control_api_settings(namespace, standalone)?; - let store = open_control_store(namespace).await?; - let (workers, handoff) = open_control_workers(namespace)?; + let config = read_server_config(namespace)?; + let address = control_address(namespace, standalone, &config.server)?; + let (authentication, facts) = control_api_settings(namespace, standalone, &config)?; + let store = open_control_store(namespace, Some(&config.server)).await?; + let (workers, handoff) = open_control_workers( + (&store, namespace), + (&config.server, config.launchers), + facts.scheduler, + ) + .await?; let (stop, shutdown) = watch::channel(false); - let mut tasks = tokio::task::JoinSet::new(); + let mut tasks = JoinSet::new(); if let Some(address) = address { drop(write_output(&[( Stream::Stderr, @@ -66,21 +76,29 @@ pub(crate) async fn run_control( )?; } if facts.scheduler { - drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); - let scheduler_store = store.clone(); - tasks.spawn(async move { - run_scheduler( - &scheduler_store, - SchedulerState::new(Uuid::now_v7()), - shutdown, - workers, - ) - .await - }); - drop(write_output(&[( - Stream::Stderr, - "Scheduler loop started; leadership is lease-guarded\n", - )])); + start_scheduler(&mut tasks, store, shutdown, workers); } supervise_control_roles(tasks, stop).await } + +fn start_scheduler( + tasks: &mut JoinSet>, + store: ControlStore, + shutdown: watch::Receiver, + workers: Workers, +) { + drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); + tasks.spawn(async move { + run_scheduler( + &store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + workers, + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Scheduler loop started; leadership is lease-guarded\n", + )])); +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs index 8570106c85..2b3c162bba 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs @@ -18,12 +18,13 @@ use crate::native_commands::_helpers::decision::{control_address, flag, text}; use crate::native_commands::_helpers::execution::run_development_roles; use crate::native_commands::_helpers::project_facts::development_target; use crate::native_commands::_helpers::terminal::write_output; +use crate::server_config::models::ServerSection; pub(super) async fn run_dev(namespace: &Namespace) -> Result<(), ControlError> { let project = std::fs::canonicalize(text(namespace, "project_dir").unwrap_or_else(|| ".".into()))?; let target = development_target(&project, text(namespace, "target"))?; - let address = control_address(namespace, false)? + let address = control_address(namespace, false, &ServerSection::default())? .ok_or_else(|| ControlError::Invalid("development address is required".into()))?; if !address.ip().is_loopback() { return Err(ControlError::Invalid( diff --git a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs index 8c2056d976..4e6c21d633 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs @@ -33,7 +33,10 @@ pub(crate) fn parse_native_invocation(arguments: &[OsString]) -> Result Option { +/// Health needs no Python CLI; serving and development return `None` without one so the caller reports it. +pub(crate) fn run_location_invocation(namespace: &Namespace, script: Option<&Path>) -> Option { let development = text(namespace, "command").as_deref() == Some("dev"); - if !development - && (text(namespace, "command").as_deref() != Some("location") - || text(namespace, "location_command").as_deref() != Some("serve")) - { + let location_command = text(namespace, "location_command"); + if !development && text(namespace, "command").as_deref() != Some("location") { + return None; + } + if !development && location_command.as_deref() == Some("health") { + return Some(location_health(namespace)); + } + if !development && location_command.as_deref() != Some("serve") { return None; } + let script = script?; let result = (|| { let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -63,3 +75,60 @@ pub(crate) fn run_location_invocation(namespace: &Namespace, script: &Path) -> O } }) } + +/// `sqb location health`: 0 when healthy or degraded, 1 when down or falling back, 2 when unanswered. +fn location_health(namespace: &Namespace) -> i32 { + drop(write_output(&[( + Stream::Stderr, + "Checking location health...\n", + )])); + match read_state(namespace) { + Ok((location_id, target, state)) => { + let document = json!({"location_id": location_id, "target": target, "state": state}); + drop(write_output(&[ + (Stream::Stdout, &format!("{document}\n")), + (Stream::Stderr, "Location health checked successfully\n"), + ])); + i32::from(matches!( + state, + LocationState::Broken | LocationState::FallingBack + )) + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Location health check failed: {error}\n"), + )])); + 2 + } + } +} + +fn read_state(namespace: &Namespace) -> Result<(Uuid, String, LocationState), String> { + let location_id: Uuid = text(namespace, "location_id") + .unwrap_or_default() + .parse() + .map_err(|_| "location ID must be a UUID".to_owned())?; + let target = text(namespace, "target").unwrap_or_default(); + if target.trim().is_empty() { + return Err("target must not be blank".into()); + } + let bearer = match std::env::var("SQLBUILD_LOCATION_TOKEN") { + Ok(bearer) if !bearer.trim().is_empty() => bearer, + _ => return Err("SQLBUILD_LOCATION_TOKEN must contain the location's credential".into()), + }; + let server_url = text(namespace, "location_server_url").unwrap_or_default(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| error.to_string())?; + let observations = runtime + .block_on(async { + connect_location_work((&server_url, &bearer), location_id)? + .recent_health(&target) + .await + }) + .map_err(|error| error.to_string())?; + let state = LocationState::of(&observations, (&target, Utc::now())); + Ok((location_id, target, state)) +} diff --git a/crates/sqlbuild-cli/src/server_config/constants.rs b/crates/sqlbuild-cli/src/server_config/constants.rs new file mode 100644 index 0000000000..92f703b757 --- /dev/null +++ b/crates/sqlbuild-cli/src/server_config/constants.rs @@ -0,0 +1,15 @@ +//! Flags the configuration file replaced, each with the section that now holds its settings. + +pub(super) const REPLACED_SERVER_FLAGS: [(&str, &str, &str); 3] = [ + ( + "control_launchers", + "--launchers", + "[[launchers.docker]] or [[launchers.local]]", + ), + ("control_oidc_config", "--oidc-config", "[oidc]"), + ( + "control_trusted_proxy_config", + "--trusted-proxy-config", + "[trusted_proxy]", + ), +]; diff --git a/crates/sqlbuild-cli/src/server_config/main/mod.rs b/crates/sqlbuild-cli/src/server_config/main/mod.rs new file mode 100644 index 0000000000..6a04bf0627 --- /dev/null +++ b/crates/sqlbuild-cli/src/server_config/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod read_server_config; diff --git a/crates/sqlbuild-cli/src/server_config/main/read_server_config.rs b/crates/sqlbuild-cli/src/server_config/main/read_server_config.rs new file mode 100644 index 0000000000..7c978eb72f --- /dev/null +++ b/crates/sqlbuild-cli/src/server_config/main/read_server_config.rs @@ -0,0 +1,40 @@ +//! Read `--config` once before any store opens; the replaced flags fail with where their settings went. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::argv::models::{Namespace, Value}; +use crate::server_config::constants::REPLACED_SERVER_FLAGS; +use crate::server_config::models::ServerConfig; + +pub(crate) fn read_server_config(namespace: &Namespace) -> Result { + if let Some((_, flag, section)) = REPLACED_SERVER_FLAGS + .iter() + .find(|(destination, _, _)| text(namespace, destination).is_some()) + { + return Err(ControlError::Invalid(format!( + "{flag} was replaced by --config: put its settings in the {section} section of \ + server.toml and start the server with --config " + ))); + } + let Some(path) = text(namespace, "control_config") else { + return Ok(ServerConfig::default()); + }; + let document = std::fs::read_to_string(&path) + .map_err(|error| ControlError::Invalid(format!("could not read {path}: {error}")))?; + toml::from_str(&document).map_err(|error: toml::de::Error| { + let line = error + .span() + .map_or(1, |span| document[..span.start].matches('\n').count() + 1); + ControlError::Invalid(format!( + "{path} is not a valid server configuration at line {line}: {}", + error.message().trim_end() + )) + }) +} + +fn text(namespace: &Namespace, name: &str) -> Option { + match namespace.get(name) { + Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), + _ => None, + } +} diff --git a/crates/sqlbuild-cli/src/server_config/mod.rs b/crates/sqlbuild-cli/src/server_config/mod.rs new file mode 100644 index 0000000000..8ce3540380 --- /dev/null +++ b/crates/sqlbuild-cli/src/server_config/mod.rs @@ -0,0 +1,5 @@ +//! The installation's `server.toml`: listener, retention, sign-in and launchers, read once at start-up. + +mod constants; +pub(crate) mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/server_config/models.rs b/crates/sqlbuild-cli/src/server_config/models.rs new file mode 100644 index 0000000000..0ad72f77de --- /dev/null +++ b/crates/sqlbuild-cli/src/server_config/models.rs @@ -0,0 +1,43 @@ +//! Each section of `server.toml`; a flag given on the command line overrides the file. + +use std::net::IpAddr; + +use sqlbuild_orchestration::api::models::{OidcAuthentication, TrustedProxyAuthentication}; + +use crate::workers::models::Launchers; + +#[derive(Debug, Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ServerConfig { + #[serde(default)] + pub(crate) server: ServerSection, + #[serde(default)] + pub(crate) retention: RetentionSection, + pub(crate) oidc: Option, + pub(crate) trusted_proxy: Option, + #[serde(default)] + pub(crate) launchers: Launchers, +} + +/// The listener, control database variable, password sign-in, default role and scheduler. +#[derive(Debug, Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ServerSection { + pub(crate) host: Option, + pub(crate) port: Option, + pub(crate) database_env: Option, + pub(crate) password_origin: Option, + pub(crate) default_role: Option, + pub(crate) scheduler: Option, +} + +/// How long the server keeps statistics, audit records, snapshots and run logs. +#[derive(Debug, Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RetentionSection { + pub(crate) statistics_days: Option, + pub(crate) audit_days: Option, + pub(crate) snapshot_count: Option, + pub(crate) run_log_days: Option, + pub(crate) run_log_count: Option, +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs index 0027b59f29..dea4e3ef0b 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -2,7 +2,9 @@ use std::collections::BTreeSet; +use sqlbuild_orchestration::credentials::main::locations::location_named::location_named; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::workers::models::{Launcher, LauncherScope, Launchers}; @@ -44,25 +46,41 @@ impl Launchers { } } -pub(in crate::workers) fn read_configuration( - path: Option<&str>, +/// Each launcher names one existing location; the server reads them once, so a new location needs a restart. +pub(in crate::workers) async fn resolve_locations( + store: &ControlStore, + configuration: &mut Launchers, +) -> Result<(), ControlError> { + let names = configuration + .local + .iter_mut() + .map(|launcher| (&launcher.location, &mut launcher.location_id)) + .chain( + configuration + .docker + .iter_mut() + .map(|launcher| (&launcher.location, &mut launcher.location_id)), + ); + for (name, location_id) in names { + *location_id = location_named(store, name).await?.ok_or_else(|| { + ControlError::Invalid(format!( + "launcher location '{name}' does not exist; create it with \ + `sqb admin create-location {name}`, then restart the server" + )) + })?; + } + Ok(()) +} + +pub(in crate::workers) fn validate_configuration( + configuration: &Launchers, scheduler_enabled: bool, -) -> Result { - let configuration: Launchers = match path { - Some(path) => serde_json::from_slice(&std::fs::read(path)?) - .map_err(|error| ControlError::Invalid(format!("launcher configuration: {error}")))?, - None => Launchers { - local: Vec::new(), - docker: Vec::new(), - concurrency_limits: Default::default(), - }, - }; +) -> Result<(), ControlError> { if !scheduler_enabled && configuration.locations().next().is_some() { return Err(ControlError::Invalid( "launchers require the embedded scheduler".into(), )); } - let mut scopes: BTreeSet<(Uuid, &str)> = BTreeSet::new(); for location in &configuration.local { if !location.project_dir.is_absolute() || !location.project_dir.is_dir() @@ -72,20 +90,54 @@ pub(in crate::workers) fn read_configuration( return Err(ControlError::Invalid("local launchers require unique location/target scopes and existing absolute project/executable paths".into())); } } - for location in configuration.locations() { - let scope = location.scope(); - if scope.target.trim().is_empty() || !scopes.insert((scope.location_id, scope.target)) { + let mut scopes: BTreeSet<(&str, &str)> = BTreeSet::new(); + let named = configuration + .local + .iter() + .map(|launcher| (launcher.location.as_str(), launcher.target.as_str())) + .chain( + configuration + .docker + .iter() + .map(|launcher| (launcher.location.as_str(), launcher.target.as_str())), + ); + for (name, target) in named { + if name.trim().is_empty() || target.trim().is_empty() || !scopes.insert((name, target)) { return Err(ControlError::Invalid( - "launchers require unique location/target scopes".into(), + "launchers require a location name and unique location/target scopes".into(), )); } - if let Launcher::Docker(location) = location - && (!location.socket.is_absolute() - || location.location_container.trim().is_empty() - || location.executable.trim().is_empty()) + } + for location in &configuration.docker { + if !location.socket.is_absolute() + || location.location_container.trim().is_empty() + || location.executable.trim().is_empty() { return Err(ControlError::Invalid("Docker launchers require an absolute socket path, location container and executable".into())); } + let run = &location.run; + if run + .cpus + .is_some_and(|cpus| !cpus.is_normal() || cpus.is_sign_negative()) + || run.pids == Some(0) + || run + .cgroup_parent + .as_deref() + .is_some_and(|path| path.trim().is_empty()) + { + return Err(ControlError::Invalid(format!( + "launcher location '{}' run settings require positive cpus and pids and a non-blank cgroup_parent", + location.location + ))); + } + if let (Some(memory), Some(ceiling)) = (run.memory, run.max_memory) + && memory > ceiling + { + return Err(ControlError::Invalid(format!( + "launcher location '{}' run memory exceeds its max_memory", + location.location + ))); + } } - Ok(configuration) + Ok(()) } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs index 746ddf0c8e..341a274922 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs @@ -4,12 +4,15 @@ use std::path::Path; use reqwest::{Method, StatusCode}; use serde_json::{Value, json}; -use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::declarations::main::job_resources::job_resources; +use sqlbuild_orchestration::declarations::models::RunResources; +use sqlbuild_orchestration::runs::models::{RunClaim, RunKind}; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; use crate::workers::_helpers::invocation::{arguments, environment, excluded}; use crate::workers::constants::IMAGE_ID_HEX_LENGTH; -use crate::workers::models::{DockerLauncher, DockerWorker, Launcher, WorkerExit}; +use crate::workers::models::{DockerLauncher, DockerWorker, Launcher, RunDefaults, WorkerExit}; impl DockerWorker { pub(in crate::workers) fn prepare( @@ -40,11 +43,17 @@ impl DockerWorker { pub(in crate::workers) async fn launch( &self, - location: &DockerLauncher, + (store, location): (&ControlStore, &DockerLauncher), claim: &RunClaim, bearer: &str, control_variable: &str, ) -> Result<(), ControlError> { + let resources = match (&claim.request.kind, &claim.request.job_key) { + (RunKind::Job, Some(job_key)) => { + job_resources(store, (claim.request.snapshot_id, job_key)).await? + } + _ => None, + }; let parent: Value = self .request( Method::GET, @@ -85,13 +94,7 @@ impl DockerWorker { "Image": self.image_id, "Entrypoint": [location.executable], "Cmd": command, "WorkingDir": project, "User": parent["Config"]["User"], "Env": variables, "Labels": {"io.sqlbuild.claim-id": self.claim_id.to_string()}, - "HostConfig": { - "NetworkMode": parent["HostConfig"]["NetworkMode"], - "VolumesFrom": [location.location_container], - "Memory": parent["HostConfig"]["Memory"], "MemorySwap": parent["HostConfig"]["MemorySwap"], - "NanoCpus": parent["HostConfig"]["NanoCpus"], "PidsLimit": parent["HostConfig"]["PidsLimit"], - "ReadonlyRootfs": parent["HostConfig"]["ReadonlyRootfs"], - }, + "HostConfig": host_config(&parent, location, resources.unwrap_or_default()), }); self.request( Method::POST, @@ -183,6 +186,79 @@ impl DockerWorker { } } +/// The job's resources, then launcher run defaults, then the location container's limits. A job never exceeds `max_memory`, or without it the location container's limits. Only positive limits are sent because Podman applies a zero; the container's unlimited `-1` passes through. +fn host_config(container: &Value, location: &DockerLauncher, job: RunResources) -> Value { + let parent = &container["HostConfig"]; + let RunDefaults { + memory, + cpus, + pids, + cgroup_parent, + max_memory, + } = &location.run; + let inherited = |key: &str| parent[key].as_u64().filter(|limit| *limit > 0); + let unlimited_or_inherited = |key: &str| parent[key].as_i64().filter(|limit| *limit != 0); + let nanos = |cpus: f64| Some((cpus * 1e9).round() as u64).filter(|nanos| *nanos > 0); + let own_memory = inherited("Memory"); + let max_memory = max_memory.map(|size| size.0); + let memory = job + .memory + .map(|size| at_most(size.0, max_memory.or(own_memory))) + .or(memory.map(|size| size.0)) + .or(own_memory + .or(max_memory) + .map(|bytes| at_most(bytes, max_memory))); + let swap = if memory == own_memory { + unlimited_or_inherited("MemorySwap").map(Value::from) + } else { + memory.map(Value::from) + }; + let own_cpus = inherited("NanoCpus"); + let nano_cpus = job + .cpus + .and_then(nanos) + .map(|nanos| at_most(nanos, own_cpus)) + .or_else(|| cpus.and_then(nanos)) + .or(own_cpus); + let pids_limit = job + .pids + .filter(|limit| *limit > 0) + .map(|limit| at_most(limit, inherited("PidsLimit"))) + .or(*pids) + .map(Value::from) + .or_else(|| unlimited_or_inherited("PidsLimit").map(Value::from)); + json!({ + "NetworkMode": network_mode(container), + "VolumesFrom": [location.location_container], + "Memory": memory, "MemorySwap": swap, "NanoCpus": nano_cpus, "PidsLimit": pids_limit, + "CgroupParent": cgroup_parent.as_deref().or_else(|| parent["CgroupParent"].as_str()).filter(|path| !path.is_empty()), + "ReadonlyRootfs": parent["ReadonlyRootfs"], + "CapDrop": parent["CapDrop"], "SecurityOpt": parent["SecurityOpt"], + }) +} + +fn at_most(value: u64, ceiling: Option) -> u64 { + ceiling.map_or(value, |ceiling| value.min(ceiling)) +} + +/// The location's network; Podman reports a container on a named network as `bridge`, so take the network's name instead. +fn network_mode(container: &Value) -> Value { + let mode = &container["HostConfig"]["NetworkMode"]; + if !matches!(mode.as_str(), Some("bridge" | "default")) { + return mode.clone(); + } + let named: Vec<&String> = container["NetworkSettings"]["Networks"] + .as_object() + .into_iter() + .flat_map(serde_json::Map::keys) + .filter(|name| !matches!(name.as_str(), "bridge" | "podman")) + .collect(); + match named.as_slice() { + [name] => json!(name), + _ => mode.clone(), + } +} + fn docker_error(error: reqwest::Error) -> ControlError { ControlError::Invalid(format!("Docker daemon request failed: {error}")) } diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs index ecfff4859a..80d4e295dc 100644 --- a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -282,7 +282,12 @@ impl Workers { if let (Launcher::Docker(location), Some(WorkerChild::Docker(child))) = (location, &worker.child) && let Err(error) = child - .launch(location, claim, &credential.bearer, &self.control_variable) + .launch( + (store, location), + claim, + &credential.bearer, + &self.control_variable, + ) .await { worker.launch_failure = Some("docker_launch_failed"); diff --git a/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs index 846eadcc68..57f00d7b3c 100644 --- a/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs +++ b/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs @@ -16,6 +16,7 @@ pub(crate) fn open_development_workers( Ok(Workers { configuration: Launchers { local: vec![LocalLauncher { + location: String::new(), location_id, target, server_url, diff --git a/crates/sqlbuild-cli/src/workers/main/open_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_workers.rs index 544dae4fc5..c0330ffdf0 100644 --- a/crates/sqlbuild-cli/src/workers/main/open_workers.rs +++ b/crates/sqlbuild-cli/src/workers/main/open_workers.rs @@ -1,17 +1,21 @@ //! Open the installation's process owner before accepting scheduler work. use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; -use crate::workers::_helpers::configuration::read_configuration; -use crate::workers::models::Workers; +use crate::workers::_helpers::configuration::{resolve_locations, validate_configuration}; +use crate::workers::models::{Launchers, Workers}; -pub(crate) fn open_workers( - configuration_path: Option<&str>, - control_variable: String, - scheduler_enabled: bool, +/// Validate the configured launchers, then resolve each location name in the opened store. +pub(crate) async fn open_workers( + store: &ControlStore, + mut configuration: Launchers, + (control_variable, scheduler_enabled): (String, bool), ) -> Result { + validate_configuration(&configuration, scheduler_enabled)?; + resolve_locations(store, &mut configuration).await?; Ok(Workers { - configuration: read_configuration(configuration_path, scheduler_enabled)?, + configuration, active: Vec::new(), control_variable, recovered_epoch: None, diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs index 852f344fde..277eb31c1c 100644 --- a/crates/sqlbuild-cli/src/workers/models.rs +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -5,6 +5,7 @@ use std::num::{NonZeroU32, NonZeroUsize}; use std::path::PathBuf; use std::time::Instant; +use sqlbuild_orchestration::declarations::models::MemorySize; use sqlbuild_orchestration::runs::models::RunClaim; use tokio::process::Child; use uuid::Uuid; @@ -30,9 +31,10 @@ pub(super) enum WorkerExit { Absent, } -#[derive(Debug, serde::Deserialize)] +/// The `[launchers]` section of `server.toml`. +#[derive(Debug, Default, serde::Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct Launchers { +pub(crate) struct Launchers { #[serde(default)] pub(super) local: Vec, #[serde(default)] @@ -53,6 +55,9 @@ pub(super) struct LauncherScope<'a> { #[derive(Debug, serde::Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct LocalLauncher { + /// The location's name; start-up resolves it to `location_id`. + pub(super) location: String, + #[serde(skip)] pub(super) location_id: Uuid, pub(super) target: String, pub(super) server_url: String, @@ -65,6 +70,9 @@ pub(super) struct LocalLauncher { #[derive(Debug, serde::Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct DockerLauncher { + /// The location's name; start-up resolves it to `location_id`. + pub(super) location: String, + #[serde(skip)] pub(super) location_id: Uuid, pub(super) target: String, pub(super) server_url: String, @@ -72,6 +80,19 @@ pub(super) struct DockerLauncher { pub(super) socket: PathBuf, pub(super) location_container: String, pub(super) executable: String, + #[serde(default)] + pub(super) run: RunDefaults, +} + +/// Worker container limits for every run of a launcher; a job's `resources` override them up to `max_memory`. +#[derive(Debug, Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RunDefaults { + pub(super) memory: Option, + pub(super) cpus: Option, + pub(super) pids: Option, + pub(super) cgroup_parent: Option, + pub(super) max_memory: Option, } #[derive(Clone, Copy, Debug)] diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs index 163c973a81..dc4e620f9f 100644 --- a/crates/sqlbuild-cli/tests/native_commands.rs +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -6,6 +6,8 @@ mod helpers; mod test_admin; #[path = "native_commands/test_control.rs"] mod test_control; +#[path = "native_commands/test_location_health.rs"] +mod test_location_health; #[path = "native_commands/admin/test_passwords.rs"] mod test_passwords; #[path = "native_commands/test_proxy.rs"] diff --git a/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs b/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs new file mode 100644 index 0000000000..4d32b485a4 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs @@ -0,0 +1,170 @@ +//! `sqb location health` reads a target's state with the location credential and fails a container health check when it is down or falling back. + +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use uuid::Uuid; + +use crate::admin_helpers::administer; +use crate::helpers::{api_ready, start_control}; +use crate::test_types::LocationHealthTestCase; + +#[test] +fn given_location_health_reports_when_checking_health_then_down_or_falling_back_locations_exit_nonzero() + { + let test_cases = [ + LocationHealthTestCase { + description: "no instance has checked in", + reports: &[], + other_target_connections: 0, + expected_state: "broken", + expected_exit: 1, + }, + LocationHealthTestCase { + description: "a recent heartbeat", + reports: &[r#"{"kind":"heartbeat"}"#], + other_target_connections: 0, + expected_state: "healthy", + expected_exit: 0, + }, + LocationHealthTestCase { + description: "the newest load failed after a good connection check", + reports: &[ + r#"{"kind":"connection","generation":2,"healthy":true}"#, + r#"{"kind":"compile_failed","generation":3}"#, + ], + other_target_connections: 0, + expected_state: "falling_back", + expected_exit: 1, + }, + LocationHealthTestCase { + description: "a later load published and connected after an earlier failure", + reports: &[ + r#"{"kind":"compile_failed","generation":3}"#, + r#"{"kind":"connection","generation":4,"healthy":true}"#, + ], + other_target_connections: 0, + expected_state: "healthy", + expected_exit: 0, + }, + LocationHealthTestCase { + description: "the newest load failed while another target sent more connection reports than one page holds", + reports: &[ + r#"{"kind":"connection","generation":2,"healthy":true}"#, + r#"{"kind":"compile_failed","generation":3}"#, + ], + other_target_connections: 120, + expected_state: "falling_back", + expected_exit: 1, + }, + LocationHealthTestCase { + description: "the warehouse connection check failed", + reports: &[r#"{"kind":"connection","generation":2,"healthy":false}"#], + other_target_connections: 0, + expected_state: "degraded", + expected_exit: 0, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let client = reqwest::Client::new(); + for test_case in test_cases { + let location = administer( + &control, + &["create-location", &format!("orders-{}", Uuid::now_v7())], + ); + let location_id = location["location_id"].as_str().expect("location UUID").to_owned(); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &["issue-location-token", &location_id, "--expires-at", &expiry], + ); + let bearer = credential["bearer"].as_str().expect("location bearer").to_owned(); + // Connection reports name a generation the location has published. + for (target, generation) in ["development", "production"] + .into_iter() + .flat_map(|target| (1..=4).map(move |generation| (target, generation))) + { + let snapshot = SnapshotInput { + location_id: location_id.parse().expect("location UUID"), + target: target.into(), + generation, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "model:orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::new(), + }; + publish_remote_snapshot(&url, &bearer, &snapshot) + .await + .expect("location publishes"); + } + let session_id = Uuid::now_v7(); + for report in test_case.reports { + let health: Value = serde_json::from_str(report).expect("health JSON"); + let response = client + .post(format!("{url}/api/locations/{location_id}/health")) + .bearer_auth(&bearer) + .json(&json!({"event_id": Uuid::now_v7(), "session_id": session_id, "target": "development", "health": health})) + .send() + .await + .expect("health report"); + let status = response.status(); + assert!( + status.is_success(), + "{}: {status} {}", + test_case.description, + response.text().await.unwrap_or_default() + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + for _ in 0..test_case.other_target_connections { + let response = client + .post(format!("{url}/api/locations/{location_id}/health")) + .bearer_auth(&bearer) + .json(&json!({"event_id": Uuid::now_v7(), "session_id": session_id, "target": "production", "health": {"kind": "connection", "generation": 4, "healthy": true}})) + .send() + .await + .expect("other target connection report"); + assert!(response.status().is_success(), "{}", test_case.description); + } + let output = std::process::Command::new(control.directory.path().join(format!("sqb{}", std::env::consts::EXE_SUFFIX))) + .args(["location", "health", "--server", &url, "--location-id", &location_id, "--target", "development"]) + .current_dir(control.directory.path()) + .env("PATH", "") + .env("SQLBUILD_LOCATION_TOKEN", &bearer) + .output() + .expect("location health process"); + let document: Value = serde_json::from_slice(&output.stdout).expect("JSON-only stdout"); + assert_eq!( + (document["state"].as_str(), output.status.code()), + (Some(test_case.expected_state), Some(test_case.expected_exit)), + "{}: {}", + test_case.description, + String::from_utf8_lossy(&output.stderr) + ); + } + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs index db54e35bb1..9d33b2a1b9 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs @@ -1,10 +1,10 @@ -//! A standalone server authenticates each request from explicit proxy bindings and rejects ambiguous modes. +//! A standalone server reads `server.toml`, authenticates each request from explicit proxy bindings and rejects ambiguous modes. use reqwest::{Client, StatusCode, header}; use serde_json::{Value, json}; use crate::helpers::{api_ready, start_control_in, wait_for_exit}; -use crate::test_types::{AuthenticationConfigurationTestCase, ProxyCliTestCase}; +use crate::test_types::{ProxyCliTestCase, ServerConfigurationTestCase}; #[test] fn given_an_owner_configured_proxy_when_requests_carry_a_subject_then_only_explicit_people_resolve_without_sessions_or_machine_authority() @@ -32,10 +32,15 @@ fn given_an_owner_configured_proxy_when_requests_carry_a_subject_then_only_expli assert!(provisioned.status.success()); let person: Value = serde_json::from_slice(&provisioned.stdout).expect("person JSON"); let principal = person["principal_id"].as_str().expect("person UUID"); - let configuration = json!({"origin": "https://control.example.com", "trusted_peers": ["127.0.0.1"], "identities": {test_case.subject: principal}}); - std::fs::write(directory.path().join("proxy.json"), configuration.to_string()).expect("server-owned identity bindings"); - let mut control = start_control_in(directory, &["server", "--port", "0", "--no-scheduler", "--trusted-proxy-config", "proxy.json"]); + let configuration = format!( + "[server]\nhost = \"203.0.113.9\"\nscheduler = true\n\n[trusted_proxy]\norigin = \"https://control.example.com\"\ntrusted_peers = [\"127.0.0.1\"]\n\n[trusted_proxy.identities]\n{} = \"{principal}\"\n", + json!(test_case.subject) + ); + std::fs::write(directory.path().join("server.toml"), configuration).expect("server-owned identity bindings"); + let mut control = start_control_in(directory, &["server", "--host", "127.0.0.1", "--port", "0", "--no-scheduler", "--config", "server.toml"]); let url = api_ready(&mut control).await; + let log = std::fs::read_to_string(&control.stderr).expect("startup log"); + assert!(!log.contains("Starting scheduler"), "{}: --no-scheduler overrides [server] scheduler", test_case.description); let client = Client::builder().timeout(std::time::Duration::from_secs(5)).build().expect("HTTP client"); let runs = format!("{url}/api/control/locations/{}/targets/development/runs", uuid::Uuid::now_v7()); let read = || client.get(&runs).header(header::HOST, "control.example.com"); @@ -59,100 +64,113 @@ fn given_an_owner_configured_proxy_when_requests_carry_a_subject_then_only_expli } #[test] -fn given_invalid_or_conflicting_identity_configuration_when_a_standalone_server_starts_then_it_fails_closed() +fn given_invalid_or_conflicting_server_configuration_when_a_standalone_server_starts_then_it_fails_closed() { + let proxy = "[trusted_proxy]\norigin = \"https://control.example.com\"\ntrusted_peers = [\"127.0.0.1\"]\nidentities = {}\n"; + let oidc = "[oidc]\norigin = \"https://control.example.com\"\nissuer = \"https://identity.example.com\"\nclient_id = \"orders-control\"\nidentities = {}\n"; let test_cases = [ - AuthenticationConfigurationTestCase { - description: "malformed JSON", - option: "--trusted-proxy-config", - configuration: "{", + ServerConfigurationTestCase { + description: "malformed TOML", + configuration: "[trusted_proxy", + arguments: &[], + expected_error: "server.toml is not a valid server configuration at line 1", + }, + ServerConfigurationTestCase { + description: "unknown section", + configuration: "[orders]\nsynthetic = true\n", arguments: &[], - expected_error: "trusted-proxy configuration requires valid JSON", + expected_error: "server.toml is not a valid server configuration at line 1", }, - AuthenticationConfigurationTestCase { - description: "unknown configuration fields", - option: "--trusted-proxy-config", - configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{},"unknown":"synthetic-unrecognized-value"}"#, + ServerConfigurationTestCase { + description: "unknown trusted-proxy field", + configuration: "[trusted_proxy]\norigin = \"https://control.example.com\"\ntrusted_peers = [\"127.0.0.1\"]\nidentities = {}\nunknown = \"synthetic-unrecognized-value\"\n", arguments: &[], - expected_error: "trusted-proxy configuration requires valid JSON", + expected_error: "server.toml is not a valid server configuration at line 5: unknown field `unknown`", }, - AuthenticationConfigurationTestCase { + ServerConfigurationTestCase { description: "no trusted peer boundary", - option: "--trusted-proxy-config", - configuration: r#"{"origin":"https://control.example.com","trusted_peers":[],"identities":{}}"#, + configuration: "[trusted_proxy]\norigin = \"https://control.example.com\"\ntrusted_peers = []\nidentities = {}\n", arguments: &[], expected_error: "requires explicit trusted peer IP addresses", }, - AuthenticationConfigurationTestCase { + ServerConfigurationTestCase { description: "non-HTTPS external origin", - option: "--trusted-proxy-config", - configuration: r#"{"origin":"http://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, + configuration: "[trusted_proxy]\norigin = \"http://control.example.com\"\ntrusted_peers = [\"127.0.0.1\"]\nidentities = {}\n", arguments: &[], expected_error: "trusted-proxy authentication requires an HTTPS origin", }, - AuthenticationConfigurationTestCase { + ServerConfigurationTestCase { description: "subject cannot be represented by the header transport", - option: "--trusted-proxy-config", - configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{"external-café":"00000000-0000-0000-0000-000000000001"}}"#, + configuration: "[trusted_proxy]\norigin = \"https://control.example.com\"\ntrusted_peers = [\"127.0.0.1\"]\n\n[trusted_proxy.identities]\n\"external-café\" = \"00000000-0000-0000-0000-000000000001\"\n", arguments: &[], expected_error: "subjects must be nonempty, unpadded ASCII header values", }, - AuthenticationConfigurationTestCase { - description: "ambiguous password/proxy modes", - option: "--trusted-proxy-config", - configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, + ServerConfigurationTestCase { + description: "a password-origin flag conflicts with the file's trusted proxy", + configuration: proxy, arguments: &["--password-origin", "https://control.example.com"], expected_error: "trusted-proxy authentication cannot be combined", }, - AuthenticationConfigurationTestCase { - description: "OIDC malformed configuration", - option: "--oidc-config", - configuration: "{", - arguments: &[], - expected_error: "OIDC configuration requires valid JSON", - }, - AuthenticationConfigurationTestCase { - description: "OIDC unknown configuration fields", - option: "--oidc-config", - configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"unknown":"synthetic-unrecognized-value"}"#, - arguments: &[], - expected_error: "OIDC configuration requires valid JSON", - }, - AuthenticationConfigurationTestCase { + ServerConfigurationTestCase { description: "OIDC rejects plaintext issuer", - option: "--oidc-config", - configuration: r#"{"origin":"https://control.example.com","issuer":"http://identity.example.com","client_id":"orders-control","identities":{}}"#, + configuration: "[oidc]\norigin = \"https://control.example.com\"\nissuer = \"http://identity.example.com\"\nclient_id = \"orders-control\"\nidentities = {}\n", arguments: &[], expected_error: "OIDC endpoints require HTTPS URLs without credentials", }, - AuthenticationConfigurationTestCase { + ServerConfigurationTestCase { description: "OIDC rejects plaintext browser origin", - option: "--oidc-config", - configuration: r#"{"origin":"http://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{}}"#, + configuration: "[oidc]\norigin = \"http://control.example.com\"\nissuer = \"https://identity.example.com\"\nclient_id = \"orders-control\"\nidentities = {}\n", arguments: &[], expected_error: "OIDC authentication requires an HTTPS origin", }, - AuthenticationConfigurationTestCase { + ServerConfigurationTestCase { description: "OIDC rejects missing client secret environment variable", - option: "--oidc-config", - configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"client_secret_env":"SQLBUILD_SYNTHETIC_OIDC_MISSING"}"#, + configuration: "[oidc]\norigin = \"https://control.example.com\"\nissuer = \"https://identity.example.com\"\nclient_id = \"orders-control\"\nidentities = {}\nclient_secret_env = \"SQLBUILD_SYNTHETIC_OIDC_MISSING\"\n", arguments: &[], expected_error: "OIDC client-secret environment variable is unavailable", }, - AuthenticationConfigurationTestCase { + ServerConfigurationTestCase { description: "OIDC rejects missing certificate authority", - option: "--oidc-config", - configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"certificate_authority":"absent-orders-ca.pem"}"#, + configuration: "[oidc]\norigin = \"https://control.example.com\"\nissuer = \"https://identity.example.com\"\nclient_id = \"orders-control\"\nidentities = {}\ncertificate_authority = \"absent-orders-ca.pem\"\n", arguments: &[], expected_error: "could not read OIDC certificate authority", }, - AuthenticationConfigurationTestCase { - description: "OIDC conflicts with password authentication", - option: "--oidc-config", - configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{}}"#, + ServerConfigurationTestCase { + description: "the file's OIDC conflicts with password authentication", + configuration: oidc, arguments: &["--password-origin", "https://control.example.com"], expected_error: "OIDC authentication cannot be combined", }, + ServerConfigurationTestCase { + description: "a zero retention in the file names its key", + configuration: "[retention]\nstatistics_days = 0\n", + arguments: &[], + expected_error: "[retention] statistics_days must be between 1 and 65535", + }, + ServerConfigurationTestCase { + description: "the replaced trusted-proxy flag points to the file", + configuration: "", + arguments: &["--trusted-proxy-config", "proxy.json"], + expected_error: "--trusted-proxy-config was replaced by --config: put its settings in the [trusted_proxy] section", + }, + ServerConfigurationTestCase { + description: "the replaced OIDC flag points to the file", + configuration: "", + arguments: &["--oidc-config", "oidc.json"], + expected_error: "--oidc-config was replaced by --config: put its settings in the [oidc] section", + }, + ServerConfigurationTestCase { + description: "the replaced launchers flag points to the file", + configuration: "", + arguments: &["--launchers", "launchers.json"], + expected_error: "--launchers was replaced by --config: put its settings in the [[launchers.docker]] or [[launchers.local]] section", + }, + ServerConfigurationTestCase { + description: "a launcher names a location the store does not have", + configuration: "[[launchers.docker]]\nlocation = \"orders\"\ntarget = \"development\"\nserver_url = \"http://127.0.0.1:1\"\nmax_workers = 1\nsocket = \"/var/run/docker.sock\"\nlocation_container = \"orders-location\"\nexecutable = \"sqb\"\n", + arguments: &[], + expected_error: "launcher location 'orders' does not exist; create it with `sqb admin create-location orders`", + }, ]; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -160,20 +178,13 @@ fn given_invalid_or_conflicting_identity_configuration_when_a_standalone_server_ .expect("runtime"); for test_case in test_cases { runtime.block_on(async { - let directory = tempfile::tempdir().expect("invalid identity installation"); + let directory = tempfile::tempdir().expect("invalid server installation"); std::fs::write( - directory.path().join("identity.json"), + directory.path().join("server.toml"), test_case.configuration, ) .expect("configuration input"); - let mut arguments = vec![ - "server", - "--port", - "0", - "--no-scheduler", - test_case.option, - "identity.json", - ]; + let mut arguments = vec!["server", "--port", "0", "--config", "server.toml"]; arguments.extend_from_slice(test_case.arguments); let mut control = start_control_in(directory, &arguments); let status = wait_for_exit(&mut control).await; @@ -181,11 +192,11 @@ fn given_invalid_or_conflicting_identity_configuration_when_a_standalone_server_ let log = std::fs::read_to_string(&control.stderr).expect("startup failure"); assert!( log.contains(test_case.expected_error), - "{}", + "{}: {log}", test_case.description ); assert!( - !log.contains(test_case.configuration), + test_case.configuration.is_empty() || !log.contains(test_case.configuration), "configuration contents must not be echoed" ); assert!( diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index b0cefc8a6c..689f38a303 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -52,9 +52,8 @@ pub(crate) struct ProxyCliTestCase { pub(crate) expected_statuses: [reqwest::StatusCode; 3], } -pub(crate) struct AuthenticationConfigurationTestCase { +pub(crate) struct ServerConfigurationTestCase { pub(crate) description: &'static str, - pub(crate) option: &'static str, pub(crate) configuration: &'static str, pub(crate) arguments: &'static [&'static str], pub(crate) expected_error: &'static str, @@ -72,3 +71,13 @@ impl Drop for TestControl { let _ = self.child.wait(); } } + +pub(crate) struct LocationHealthTestCase { + pub(crate) description: &'static str, + /// Health reports the location sends before the check, oldest first, as JSON `health` values. + pub(crate) reports: &'static [&'static str], + /// Connection reports another target of the same location sends afterwards. + pub(crate) other_target_connections: usize, + pub(crate) expected_state: &'static str, + pub(crate) expected_exit: i32, +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs index 31a14b3721..b8ee74341f 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -290,23 +290,24 @@ pub(crate) fn disconnect_worker(_worker: WorkerControl, control: &mut TestContro pub(crate) fn start_recovery_scheduler( control: &TestControl, url: &str, - location_id: Uuid, + location: &str, ) -> TestControl { let directory = tempfile::tempdir().expect("recovery scheduler logs"); - let configuration = directory.path().join("launchers.json"); - let launcher = serde_json::json!({"docker": [{ - "location_id": location_id, "target": "development", "server_url": url, - "max_workers": 1, "socket": directory.path().join("docker.sock"), - "location_container": "orders-location", "executable": "sqb" - }]}); - std::fs::write(&configuration, launcher.to_string()).expect("Docker scope configuration"); + let configuration = directory.path().join("server.toml"); + let launcher = format!( + "[[launchers.docker]]\nlocation = {}\ntarget = \"development\"\nserver_url = {}\nmax_workers = 1\nsocket = {}\nlocation_container = \"orders-location\"\nexecutable = \"sqb\"\n", + serde_json::json!(location), + serde_json::json!(url), + serde_json::json!(directory.path().join("docker.sock")), + ); + std::fs::write(&configuration, launcher).expect("Docker scope configuration"); let stderr = directory.path().join("scheduler.log"); let child = Command::new(isolated_binary(directory.path())) .args([ "scheduler", "--database-env", "SQLBUILD_RECOVERY_TEST_STORE", - "--launchers", + "--config", ]) .arg(configuration) .current_dir(directory.path()) diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs index aeb5ae95ec..eb7a66fb93 100644 --- a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -46,7 +46,7 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ runtime.block_on(async { let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); let url = api_ready(&mut control).await; - let (worker, baseline, _) = claim_worker_run(&control, test_case.image_id).await; + let (worker, _, _) = claim_worker_run(&control, test_case.image_id).await; let connection = rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) .expect("observe durable ownership"); @@ -58,7 +58,7 @@ fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_ CREATE TRIGGER reject_recovery_publication BEFORE INSERT ON recall_events WHEN NEW.event_type='{}' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", test_case.failure_event )).expect("expire the departed leader and interrupt durable publication"); - let mut replacement = start_recovery_scheduler(&control, &url, baseline.location_id); + let mut replacement = start_recovery_scheduler(&control, &url, "orders"); wait_for_worker_log(&mut replacement, "Retrying worker shutdown publication").await; let held: (i64, i64) = connection.query_row( "SELECT (SELECT COUNT(*) FROM sqb_write_claims), (SELECT COUNT(*) FROM sqb_runs WHERE worker_instance IS NOT NULL)", diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs index adeed404f4..8496ff6700 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs @@ -1,6 +1,6 @@ //! Scope both observations and reads before exposing any location history. -use axum::extract::{Path, State}; +use axum::extract::{Path, Query, State}; use axum::{Extension, Json}; use uuid::Uuid; @@ -10,7 +10,9 @@ use crate::api::models::MutationResponse; use crate::credentials::models::TokenIdentity; use crate::platform::main::location_health::location_health; use crate::platform::main::report_location_health::report_location_health; -use crate::platform::models::{LocationHealthObservation, LocationHealthReport}; +use crate::platform::models::{ + LocationHealthObservation, LocationHealthQuery, LocationHealthReport, +}; use crate::store::models::ControlStore; #[utoipa::path(post, path = "/api/locations/{location_id}/health", operation_id = "report_location_health", @@ -31,15 +33,18 @@ pub(crate) async fn report( } #[utoipa::path(get, path = "/api/locations/{location_id}/health", operation_id = "read_location_health", - params(("location_id" = Uuid, Path)), + params(("location_id" = Uuid, Path), + ("target" = Option, Query, description = "Only this target's observations")), responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), security(("bearer" = [])))] pub(crate) async fn recent( State(store): State, Extension(identity): Extension, - Path(location_id): Path, + (Path(location_id), Query(options)): (Path, Query), ) -> Result>, ApiError> { require_location(&identity, location_id)?; - Ok(Json(location_health(&store, (location_id, None)).await?)) + Ok(Json( + location_health(&store, (location_id, options.target.as_deref())).await?, + )) } diff --git a/crates/sqlbuild-orchestration/src/credentials/main/locations/location_named.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/location_named.rs new file mode 100644 index 0000000000..453e769325 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/locations/location_named.rs @@ -0,0 +1,30 @@ +//! The installation's launcher configuration names a location; the store holds its identity. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The active location with this name, or `None` when none exists or it is archived. +pub async fn location_named( + store: &ControlStore, + name: &str, +) -> Result, ControlError> { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + "SELECT location_id FROM sqb_locations WHERE name=$1 AND archived_at IS NULL", + vec![Parameter::Text(name.into())], + ) + .await?; + commit(transaction).await?; + rows.first() + .map(|row| { + row["location_id"] + .as_str() + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Schema("invalid location identity".into())) + }) + .transpose() +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs index 23453c831d..960e62ed4c 100644 --- a/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs +++ b/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs @@ -1,2 +1,3 @@ pub mod create_location; pub mod development_location; +pub mod location_named; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/job_resources.rs b/crates/sqlbuild-orchestration/src/declarations/main/job_resources.rs new file mode 100644 index 0000000000..e4310f7649 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/job_resources.rs @@ -0,0 +1,28 @@ +//! The container limits a job asked for in the snapshot its run is pinned to. + +use uuid::Uuid; + +use crate::declarations::models::{JobDefinition, RunResources}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// `None` when the job asks for nothing or the snapshot no longer holds it. +pub async fn job_resources( + store: &ControlStore, + (snapshot_id, job_key): (Uuid, &str), +) -> Result, ControlError> { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + "SELECT definition_json FROM sqb_jobs WHERE snapshot_id=$1 AND job_key=$2", + vec![Parameter::Id(snapshot_id), Parameter::Text(job_key.into())], + ) + .await?; + commit(transaction).await?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + .map_err(|error| ControlError::Schema(error.to_string()))?; + Ok(definition.resources) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index 8f0738398f..085e543d5b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -2,6 +2,7 @@ pub mod change_person_trigger; pub mod change_trigger; pub(crate) mod frozen_definitions; pub mod group_dependencies; +pub mod job_resources; pub mod list_person_jobs; pub mod list_person_trigger_changes; pub mod list_person_triggers; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index b0a475a46d..96428acafa 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -52,6 +52,61 @@ pub struct JobDefinition { pub overlap: Overlap, #[serde(default, skip_serializing_if = "Option::is_none")] pub on_failure: Option, + /// The worker container's limits for this job; the launcher's `max_memory` caps memory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resources: Option, +} + +/// Container limits a job asks for; each unset limit falls back to the launcher's. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RunResources { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpus: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pids: Option, +} + +/// Bytes, written as an integer or with a `k`, `m` or `g` suffix (powers of 1024), as Docker's `--memory` reads it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct MemorySize(pub u64); + +impl MemorySize { + /// `"512m"`, `"2g"` or `"1073741824"`; the size must be at least one byte. + pub fn parse(text: &str) -> Result { + let text = text.trim().to_ascii_lowercase(); + let (digits, scale) = match text.strip_suffix('b').unwrap_or(&text) { + value if value.ends_with('k') => (&value[..value.len() - 1], 1u64 << 10), + value if value.ends_with('m') => (&value[..value.len() - 1], 1u64 << 20), + value if value.ends_with('g') => (&value[..value.len() - 1], 1u64 << 30), + value => (value, 1), + }; + match digits.parse::().map(|count| count.checked_mul(scale)) { + Ok(Some(bytes)) if bytes > 0 => Ok(Self(bytes)), + _ => Err(format!( + "memory must be a positive size such as \"512m\" or \"2g\", not {text:?}" + )), + } + } +} + +impl<'de> Deserialize<'de> for MemorySize { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Written { + Bytes(u64), + Text(String), + } + match Written::deserialize(deserializer)? { + Written::Bytes(0) => Err(serde::de::Error::custom("memory must be at least one byte")), + Written::Bytes(bytes) => Ok(Self(bytes)), + Written::Text(text) => Self::parse(&text).map_err(serde::de::Error::custom), + } + } } /// What a schedule tick or sensor request does while a run of the same job is active; people's requests always queue. diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index a44a93c5b3..cf739e204d 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -6,8 +6,8 @@ use crate::api::errors::ClientError; use crate::api::main::client_response::client_response; use crate::api::models::MutationResponse; use crate::platform::models::{ - LocationHealthReport, LocationWorkClaim, LocationWorkClient, LocationWorkPoll, - LocationWorkResult, + LocationHealthObservation, LocationHealthReport, LocationWorkClaim, LocationWorkClient, + LocationWorkPoll, LocationWorkResult, }; use crate::warehouse_metadata::models::{ MetadataBatch, PreviewCopy, PreviewCopyRead, StatisticsDueRead, StatisticsRead, @@ -36,6 +36,21 @@ impl LocationWorkClient { Ok(response.applied) } + /// The newest health observations of one target of this location, newest first. + pub async fn recent_health( + &self, + target: &str, + ) -> Result, ClientError> { + let mut endpoint = self.connection.endpoint(&[ + "api", + "locations", + &self.location_id.to_string(), + "health", + ])?; + endpoint.query_pairs_mut().append_pair("target", target); + client_response(self.connection.client.get(endpoint)).await + } + pub async fn metadata_watermark( &self, target: &str, diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index b34c046cdb..efcacecef3 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -73,6 +73,13 @@ pub struct LocationHealthReport { pub health: LocationHealth, } +/// Narrows a location's health history to one target, so other targets' traffic cannot crowd it out. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocationHealthQuery { + pub target: Option, +} + /// Receipt time belongs to the server; consumers derive freshness rather than storing availability. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct LocationHealthObservation { @@ -80,6 +87,68 @@ pub struct LocationHealthObservation { pub report: LocationHealthReport, } +/// One target's state from its newest health observations, as the web UI shows it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LocationState { + Healthy, + /// The warehouse connection check after the last load failed. + Degraded, + /// The last load failed, so runs use the previous snapshot. + FallingBack, + /// No instance has checked in for 90 seconds. + Broken, +} + +impl LocationState { + /// `target`'s state at `now` from observations that may cover several targets: down without a recent check-in, then a failed load, then a failed connection. + #[must_use] + pub fn of( + observations: &[LocationHealthObservation], + (target, now): (&str, DateTime), + ) -> Self { + let own: Vec<&LocationHealthObservation> = observations + .iter() + .filter(|observation| observation.report.target == target) + .collect(); + let silence = + chrono::TimeDelta::seconds(crate::platform::constants::LOCATION_SILENCE_SECONDS); + if !own + .iter() + .any(|observation| now - observation.received_at <= silence) + { + return Self::Broken; + } + let mut failed: Option<(DateTime, i64)> = None; + let mut connection: Option<(DateTime, i64, bool)> = None; + for observation in own { + let seen = observation.received_at; + match observation.report.health { + LocationHealth::CompileFailed { generation } + if failed.is_none_or(|(at, _)| seen > at) => + { + failed = Some((seen, generation)); + } + LocationHealth::Connection { + generation, + healthy, + } if connection.is_none_or(|(at, _, _)| seen > at) => { + connection = Some((seen, generation, healthy)); + } + _ => {} + } + } + match (failed, connection) { + (Some((_, failed)), Some((_, connected, _))) if failed >= connected => { + Self::FallingBack + } + (Some(_), None) => Self::FallingBack, + (_, Some((_, _, false))) => Self::Degraded, + _ => Self::Healthy, + } + } +} + /// Only named location operations cross this boundary, never arbitrary SQL or Python. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index a63c08cd12..c761476387 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -21,8 +21,8 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, module_jobs, - run_controls, + AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, job_resources, + module_jobs, run_controls, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ compile_calendars, compile_cron, compile_sensors, @@ -128,6 +128,10 @@ pub(crate) fn compile_declarations( job.on_failure.as_ref().map(|policy| policy.bind(py)), ), )?; + let resources = job_resources( + &job.name, + job.resources.as_ref().map(|resources| resources.bind(py)), + )?; let targets: BTreeSet = job.targets.into_iter().collect(); if let Some(schedule) = job.schedule { triggers.insert( @@ -153,6 +157,7 @@ pub(crate) fn compile_declarations( owner: controls.owner, overlap: controls.overlap, on_failure: controls.on_failure, + resources, }, ); } @@ -216,6 +221,7 @@ pub(crate) fn compile_declarations( owner: controls.owner, overlap: controls.overlap, on_failure: controls.on_failure, + resources: None, }, jobs: dependencies, }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 89f8f159be..dcf1dfd9bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -11,7 +11,7 @@ use serde_json::Value; use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_discovery::project_validation::models::OrchestrationNameFact; use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_orchestration::declarations::models::{DeclaredAt, OnFailure, Overlap}; +use sqlbuild_orchestration::declarations::models::{DeclaredAt, OnFailure, Overlap, RunResources}; use crate::bindings::_helpers::native_host::constants::{ AUTHORING_MODULE, CALENDAR_LABEL, CONSTRUCTING_MODULES, GROUP_LABEL, JOB_LABEL, MAX_RETRIES, @@ -37,6 +37,7 @@ pub(crate) struct AuthoredJob { pub(crate) owner: Option, pub(crate) overlap: String, pub(crate) on_failure: Option>, + pub(crate) resources: Option>, } #[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] @@ -212,6 +213,36 @@ pub(crate) fn run_controls( }) } +/// Refuse a memory size Docker cannot read and CPUs or a process limit that is not positive. +pub(crate) fn job_resources( + name: &str, + resources: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + let Some(resources) = resources else { + return Ok(None); + }; + let mut written = serde_json::Map::new(); + for field in ["memory", "cpus", "pids"] { + let value = authored_setting(&resources.getattr(field)?)?; + if !value.is_null() { + written.insert(field.into(), value); + } + } + let invalid = |reason: String| PyValueError::new_err(format!("job {name} resources: {reason}")); + let resources: RunResources = serde_json::from_value(Value::Object(written)) + .map_err(|error| invalid(error.to_string()))?; + if resources + .cpus + .is_some_and(|cpus| !cpus.is_normal() || cpus.is_sign_negative()) + { + return Err(invalid("cpus must be a positive number".into())); + } + if resources.pids == Some(0) { + return Err(invalid("pids must be a positive whole number".into())); + } + Ok(Some(resources)) +} + /// Only module execution crosses into Python; discovery and collection stay native. pub(crate) fn module_jobs<'py>( py: Python<'py>, diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 5ecf300a8a..f07f0561cd 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,9 +1,16 @@ -# Build wheels from this checkout before building these images; see deploy/README.md. +# Install a released SQLBuild (SQLBUILD_SOURCE=release) or this checkout's wheels; see deploy/README.md. ARG PYTHON_VERSION=3.12 -FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime +ARG SQLBUILD_SOURCE=wheels +FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS install-release +ARG SQLBUILD_VERSION +RUN test -n "$SQLBUILD_VERSION" && python -m pip install --no-cache-dir "sqlbuild==${SQLBUILD_VERSION}" + +FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS install-wheels COPY dist/*.whl /wheels/ -RUN python -m pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels \ - && useradd --uid 10001 --create-home sqlbuild \ +RUN python -m pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels + +FROM install-${SQLBUILD_SOURCE} AS runtime +RUN useradd --uid 10001 --create-home sqlbuild \ && mkdir -p /project/target /project/logs /warehouse \ && chown sqlbuild:sqlbuild /project /project/target /project/logs /warehouse ENV PYTHONUNBUFFERED=1 @@ -18,11 +25,11 @@ ENV SQLBUILD_GIT_REVISION=${SQLBUILD_GIT_REVISION} COPY --chown=10001:10001 deploy/project/ /project/ # The control plane has no Python interpreter, project, adapter, or warehouse credentials. -FROM debian:bookworm-slim AS server +FROM docker.io/library/debian:bookworm-slim AS server RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl \ && rm -rf /var/lib/apt/lists/* \ && useradd --uid 10001 --create-home sqlbuild COPY --from=runtime /usr/local/bin/sqb /usr/local/bin/sqb USER 10001:10001 ENTRYPOINT ["sqb"] -CMD ["server", "--host", "0.0.0.0", "--port", "8000", "--launchers", "/etc/sqlbuild/launchers.json"] +CMD ["server", "--config", "/etc/sqlbuild/server.toml"] diff --git a/deploy/README.md b/deploy/README.md index eb85bcf12b..531a9ff768 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -4,30 +4,39 @@ This Linux reference runs PostgreSQL, the native SQLBuild server/scheduler, and Python code location. Workers use Docker only. The server image contains no Python or project. The location image includes the example orders project and its Python dependencies. -Use Docker Engine with a Unix socket and Docker Compose 2.23.1 or newer, Python 3.12, `uv`, -a Rust toolchain, and `jq`. Build and test from a trusted checkout. Docker socket access is -host-administrator authority: only the server gets that socket and control-store credentials. +Use Docker Engine with a Unix socket and Docker Compose 2.23.1 or newer, or Podman 5 (see +[Run on Podman](#run-on-podman)), and `jq`. Docker socket access is host-administrator authority: +only the server gets that socket and control-store credentials. Locations and workers get warehouse access, but no control-store credentials or inbound ports. The default server port binds to loopback, not the public network. ## Build the images -Build matching wheels from this checkout before building the images: +The images install a released SQLBuild from PyPI. Copy the example settings and set +`SQLBUILD_VERSION` in the ignored `.env` to the release you deploy: + +```sh +cp deploy/compose.env.example .env +``` + +To test unreleased code from a trusted checkout instead, set `SQLBUILD_SOURCE=wheels` and build +matching wheels first, with Python 3.12, `uv` and a Rust toolchain: ```sh uv sync --locked --all-extras uv run --no-sync maturin build --release --compatibility linux --out dist uv run --no-sync maturin build --manifest-path crates/sqlbuild-cli/Cargo.toml --release --compatibility linux --out dist -cp deploy/compose.env.example .env ``` -Set the ignored `.env` values. Use a unique PostgreSQL password and put its URL-encoded value +Keep only the matching SQLBuild and CLI wheels in `dist/`; their Python version and architecture +must match the image's `PYTHON_VERSION` build argument and Docker host. + +Set the other `.env` values. Use a unique PostgreSQL password and put its URL-encoded value in `SQLBUILD_CONTROL_DATABASE_URL`, for example `postgresql://sqlbuild:@postgres:5432/sqlbuild`. Set `DOCKER_GID` to `stat -c %g /var/run/docker.sock` and `SQLBUILD_GIT_REVISION` to -`git rev-parse HEAD`. Keep only the matching SQLBuild and CLI wheels in `dist/`; their Python -version and architecture must match the image's `PYTHON_VERSION` build argument and Docker host. -The default image uses Python 3.12. Do not put credentials in build arguments or project files. +`git rev-parse HEAD`. The default image uses Python 3.12. Do not put credentials in build +arguments or project files. The Docker build context includes only wheels and the example project. ```sh @@ -44,6 +53,11 @@ direct-store administration path, not an HTTP or person-role grant: docker compose run --rm --no-deps server admin create-location orders ``` +The server reads `/etc/sqlbuild/server.toml`, which `compose.yml` writes from its `server` config. +Its `[[launchers.docker]]` entry starts workers for the location named `orders`, so create the +location before the server starts. Edit that config and recreate the server to change launchers, +retention or sign-in; see `sqb server` in the documentation for every setting. + Replace `.env`'s placeholder `SQLBUILD_LOCATION_ID` with the returned `location_id`. Issue a bounded credential, using a future RFC 3339 expiry you choose: @@ -66,6 +80,18 @@ running container's actual Docker image ID at publication; no `--image-id` or Do needed in the location. Workers recompile in that pinned image and verify environment and node hashes before writing. The warehouse persists in its named volume. +Check the location the way a container health check would; it exits 1 when the location is down +or its last load failed: + +```sh +docker compose exec location sqb location health --server http://server:8000 --location-id "$SQLBUILD_LOCATION_ID" --target development +``` + +Workers get the location container's memory, CPU and process limits (`cpus: 2` and +`mem_limit: 1g` in `compose.yml`), dropped capabilities and security options. Add a +`[launchers.docker.run]` table to the `server` config for different worker limits, and give a job +`resources` to ask for more or less, up to the table's `max_memory`. + For your project, build the `runtime` target as a base, then extend it with `COPY --chown=10001:10001` of your project to `/project`, install its locked dependencies, and set `SQLBUILD_GIT_REVISION`. Set `SQLBUILD_LOCATION_IMAGE` to that project image. Give location UID 10001 access to its @@ -104,6 +130,42 @@ changing the host source tree's ownership. Use the same overlay for `down --volu these development caches and logs. Workers reject source drift against a pinned snapshot; the bind mount is not an immutable deployment and does not provide old-source fallback. +## Run on Podman + +The stack runs on Podman 5 through its Docker-compatible API socket. Workers start, report their +exit, share the location's volumes and network, and run the image ID the location published, as +they do on Docker. + +For rootful Podman, start the system socket, `/run/podman/podman.sock`: + +```sh +sudo systemctl enable --now podman.socket +``` + +The socket belongs to `root:root` with mode `0660`. Give it a dedicated group instead, so the +server joins that group rather than `root`: + +```sh +sudo groupadd --system sqlbuild-socket +sudo systemctl edit podman.socket # add the two lines below under [Socket] +# SocketGroup=sqlbuild-socket +# SocketMode=0660 +sudo systemctl restart podman.socket +``` + +For rootless Podman, start your user's socket, `$XDG_RUNTIME_DIR/podman/podman.sock`, with +`systemctl --user enable --now podman.socket`. It belongs to your user, and containers run in your +user's namespace. + +Install Docker Compose 2.23.1 or newer as a Docker CLI plugin; `podman compose` runs it against +the Podman socket. `podman-compose` cannot run this stack, because it does not create the +server's `server.toml` from `compose.yml`. + +In `.env`, set `DOCKER_SOCKET` to the socket path and `DOCKER_GID` to +`stat -c %g "$DOCKER_SOCKET"`. Then run each `docker compose` command in this guide as +`podman compose`. Check that it uses Docker Compose: `podman compose version` prints +`Docker Compose version`. + ## Credentials, identity, and platform events Rotate a location credential with `docker compose run --rm --no-deps server admin rotate-token @@ -111,9 +173,9 @@ TOKEN_UUID`, store its one-time replacement bearer, and recreate the location. T expiry stay unchanged, and the old credential is revoked atomically. Service-account principal tokens are separate from location/run tokens. Person identities do not become machine bearers. -For OIDC sign-in, mount a server-owned JSON file with `origin`, `issuer`, -`client_id`, and explicit `identities` subject-to-existing-person UUID bindings; append -`--oidc-config /etc/sqlbuild/oidc.json` to the server command. Both origin and issuer require +For OIDC sign-in, add an `[oidc]` section to the `server` config with `origin`, `issuer`, +`client_id`, and explicit `identities` subject-to-existing-person UUID bindings, and name the +client secret's environment variable in `client_secret_env`. Both origin and issuer require HTTPS. Terminate HTTPS at a trusted proxy, preserve the external Host, restrict backend access, and register `/api/auth/oidc/callback` at the issuer. The callback verifies an explicitly bound, enabled person and issues a one-hour browser session. With a trusted proxy, every request @@ -178,7 +240,7 @@ outside this stack too: a stopped server cannot report its own outage. ## Back up and upgrade Back up PostgreSQL with `pg_dump` using your normal protected database backup tooling. Also -back up warehouse data, the restricted `.env`/identity configuration, and referenced project +back up warehouse data, the restricted `.env`, the `server` config, and referenced project images. Keep backup credentials and output outside the repository. Test restoration before an upgrade. Ordinary `docker compose down` preserves named volumes; `down --volumes` destroys them. diff --git a/deploy/compose.env.example b/deploy/compose.env.example index a667e912e0..e8a4543d1c 100644 --- a/deploy/compose.env.example +++ b/deploy/compose.env.example @@ -1,6 +1,10 @@ # Copy to the ignored root .env; set the password and matching URL before starting PostgreSQL. +# The SQLBuild release the images install; set SQLBUILD_SOURCE=wheels to use checkout wheels in dist/. +SQLBUILD_SOURCE=release +SQLBUILD_VERSION= POSTGRES_PASSWORD= SQLBUILD_CONTROL_DATABASE_URL= +DOCKER_SOCKET=/var/run/docker.sock DOCKER_GID= SQLBUILD_GIT_REVISION= # Replace this placeholder with create-location's assigned identity before starting the location. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index ade6e96003..6b4a04a621 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -161,4 +161,6 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb dag](cli/dag.md) (`cli/dag`) - Generate the static DAG artifact for Dagster and other integrations. - [sqb dbt](cli/dbt.md) (`cli/dbt`) - Coordinate dbt and SQLBuild projects. - [sqb server](cli/server.md) (`cli/server`) - Run the control service and scheduler. +- [sqb location](cli/location.md) (`cli/location`) - Run a code location that publishes a project to the server, and check its health. +- [sqb dev](cli/dev.md) (`cli/dev`) - Run the server, scheduler and a reloading code location for one project on your machine. - [sqb login and logout](cli/login.md) (`cli/login`) - Authorize a bounded CLI session through your server's browser sign-in. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dev.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dev.md new file mode 100644 index 0000000000..7003b2301e --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dev.md @@ -0,0 +1,51 @@ + + +# sqb dev + +> Run the server, scheduler and a reloading code location for one project on your machine. + +Online: https://sqlbuild.com/docs/cli/dev/ + +Run the control API, the scheduler and a code location for the project in the current directory, +and open the web UI at the address it prints: + +```bash +sqb dev +``` + +```text +Development API ready: 127.0.0.1:8000 +Development location: 01a1d7ff-0b3a-7631-ba35-e44d44d072f2 +Authentication disabled on loopback; triggers start paused +``` + +The location compiles the project and publishes it to the server. When you save a model, +orchestration file or other project input, it compiles and publishes again, so the UI and the +next run use the new version. + +## Options + +| Option | Default | Effect | +| --- | --- | --- | +| `--port` | `8000` | Port to listen on, on `127.0.0.1`. `0` picks a free port | +| `--target` | The configured target | Target to compile and run | +| `--control-store-url` | `.sqlbuild/control.sqlite3` in the project | Control store URL or path | +| `--no-scheduler` | Off | Serve and reload without running scheduled work | +| `--triggers` | Off | Keep each trigger's default status and your changes to it, instead of pausing every trigger at start-up | + +## Run scheduled jobs locally + +Triggers start paused, so a schedule never runs on your machine by accident. Resume a trigger in +the UI to let it run, or start with `--triggers` to keep the status each trigger was declared with: + +```bash +sqb dev --triggers +``` + +## Details + +- `sqb dev` listens only on loopback and needs no sign-in. Use [`sqb server`](server.md) + for a server other people or machines connect to. +- The control store keeps runs, trigger changes and history between sessions. Delete + `.sqlbuild/control.sqlite3` to start again from nothing. +- Stop it with Ctrl+C; it prints `Development services stopped successfully`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md new file mode 100644 index 0000000000..e9e56aa8f3 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md @@ -0,0 +1,53 @@ + + +# sqb location + +> Run a code location that publishes a project to the server, and check its health. + +Online: https://sqlbuild.com/docs/cli/location/ + +Run a code location: it compiles the project in its own Python environment, publishes it to the +server, and publishes again when the project changes: + +```bash +export SQLBUILD_LOCATION_TOKEN=... +sqb location serve --server https://control.example.com --location-id "$LOCATION_ID" --target production +``` + +Create the location and its token with `sqb admin create-location orders` and +`sqb admin issue-location-token` on the server's host. The location reads the token from +`SQLBUILD_LOCATION_TOKEN` and never opens the control database. + +## Check a location's health + +Ask the server how a location and target are doing, for example from a container health check: + +```bash +sqb location health --server https://control.example.com --location-id "$LOCATION_ID" --target production +``` + +```json +{"location_id":"01a1d7ff-0b3a-7631-ba35-e44d44d072f2","target":"production","state":"healthy"} +``` + +| `state` | Meaning | Exit code | +| --- | --- | --- | +| `healthy` | An instance checked in within 90 seconds and its last load published | `0` | +| `degraded` | The warehouse connection check after the last load failed | `0` | +| `falling_back` | The last load failed, so runs use the previous snapshot | `1` | +| `broken` | No instance has checked in for 90 seconds | `1` | + +The command exits `2` when it cannot read the state: the server is unreachable, or the token is +missing, expired or for another location. It needs the location's token, not a person's sign-in. + +A Compose health check for the location service: + +```yaml +healthcheck: + test: [CMD, sqb, location, health, --server, "http://server:8000", --location-id, "${SQLBUILD_LOCATION_ID}", --target, development] + interval: 30s + start_period: 60s +``` + +A `falling_back` location still runs scheduled jobs from its last good snapshot. Restarting it +does not fix the failed load; fix the project and deploy it again. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 7cc1efbdff..8251f31acc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -8,6 +8,7 @@ Online: https://sqlbuild.com/docs/cli/server/ ## Contents +- Configure the server - Serve over HTTPS - Receive platform events - Upgrade @@ -16,17 +17,71 @@ Online: https://sqlbuild.com/docs/cli/server/ - Inspect published SQL - Sync from the warehouse now - Skip overlapping runs and retry failures +- Limit worker containers - Run ad-hoc commands - Plan builds - Define project roles -Run the control API and scheduler without loading project Python: +Run the control API and scheduler without loading project Python, with its settings in +`server.toml`: ```bash -sqb server +sqb server --config server.toml ``` -Use `sqb server --help` for service, authentication and launcher options. +## Configure the server + +`server.toml` holds the listener, retention, sign-in and the launchers that start workers. The +server reads it once at start-up, so restart the server after you change it: + +```toml +[server] +host = "0.0.0.0" +port = 8000 +default_role = "viewer" + +[retention] +run_log_days = 7 + +[trusted_proxy] +origin = "https://control.example.com" +trusted_peers = ["10.0.0.5"] + +[trusted_proxy.identities] +orders-person = "0190c3c1-5a4e-7b8e-9f00-2a7d1c9e4b11" + +[[launchers.docker]] +location = "orders" +target = "development" +server_url = "http://server:8000" +max_workers = 2 +socket = "/var/run/docker.sock" +location_container = "orders-location" +executable = "/usr/local/bin/sqb" +``` + +| Section | Keys | +| --- | --- | +| `[server]` | `host` (default `127.0.0.1`), `port` (`8000`), `database_env` (`SQLBUILD_CONTROL_DATABASE_URL`), `password_origin`, `default_role` (`none`), `scheduler` (`true`) | +| `[retention]` | `statistics_days`, `audit_days`, `snapshot_count`, `run_log_days`, `run_log_count`; see [Set retention](#set-retention) | +| `[oidc]` | `origin`, `issuer`, `client_id`, `identities`, and optionally `client_secret_env`, `certificate_authority`, `groups_claim` and `group_roles` | +| `[trusted_proxy]` | `origin`, `trusted_peers` and `identities` | +| `[[launchers.docker]]` | `location`, `target`, `server_url`, `max_workers`, `socket`, `location_container`, `executable`, and an optional `run` table; see [Limit worker containers](#limit-worker-containers) | +| `[[launchers.local]]` | `location`, `target`, `server_url`, `max_workers`, and absolute `project_dir` and `executable` paths | +| `[launchers.concurrency_limits]` | The most runs at once for each job `concurrency_groups` name, such as `orders_pool = 2` | + +- A flag overrides the same setting in the file: `sqb server --config server.toml --port 9000` + listens on port 9000. +- Keep secrets in environment variables. The file names the variable, as `database_env` and + `client_secret_env` do, never the secret itself. +- Each launcher names a location by the name you gave `sqb admin create-location`. Create the + location before you start the server; an unknown name stops it. +- `sqb scheduler --config server.toml` reads the same file for its control database and launchers. +- `sqb admin` doesn't read the file. When `database_env` names another variable, pass the same + name to `sqb admin --database-env` so it changes the server's store. +- An unknown section or key stops the server with the line it is on. + +Use `sqb server --help` for the remaining options. ## Serve over HTTPS @@ -116,21 +171,25 @@ the control server is older than this worker and refused its run report; upgrade ## Set retention -Set how much history the server keeps: +Set how much history the server keeps in the `[retention]` section of `server.toml`: -```bash -sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 \ - --run-log-retention-days 7 --run-log-retention-count 100 +```toml +[retention] +statistics_days = 30 +audit_days = 730 +snapshot_count = 20 +run_log_days = 7 +run_log_count = 100 ``` -| Data | Option | Default | Range | -| --- | --- | --- | --- | -| Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | -| Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | -| Published snapshots per location and target | `--snapshot-retention-count` | `10` | `1`–`65535` | -| Logs of finished runs | `--run-log-retention-days` | `14` | `1`–`65535` | -| Finished runs per location and target that keep their logs | `--run-log-retention-count` | `200` | `1`–`65535` | -| Skipped and covered schedule ticks | None | 30 days | Fixed | +| Data | `[retention]` key | Option | Default | Range | +| --- | --- | --- | --- | --- | +| Hourly warehouse statistics | `statistics_days` | `--statistics-retention-days` | `90` | `1`–`65535` | +| Security audit records | `audit_days` | `--audit-retention-days` | `400` | `1`–`65535` | +| Published snapshots per location and target | `snapshot_count` | `--snapshot-retention-count` | `10` | `1`–`65535` | +| Logs of finished runs | `run_log_days` | `--run-log-retention-days` | `14` | `1`–`65535` | +| Finished runs per location and target that keep their logs | `run_log_count` | `--run-log-retention-count` | `200` | `1`–`65535` | +| Skipped and covered schedule ticks | None | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then deleted in the background. @@ -152,7 +211,7 @@ deletes its node definitions and SQL text when no other snapshot shares them. Re interrupting this cleanup is safe. A finished run loses its log, the events, statements, SQL and output its worker recorded, when it -is older than `--run-log-retention-days` or when more than `--run-log-retention-count` newer runs +is older than `run_log_days` or when more than `run_log_count` newer runs have finished in its location and target, whichever comes first. The run itself, its node results, and the run history the scheduler recorded stay, so run lists, node status and history are unchanged; only the run's log, statements and SQL are no longer shown. A running run keeps its log. @@ -172,20 +231,26 @@ webhook, so give them the same values and webhook. ## Grant roles to OIDC groups (beta) With OIDC sign-in, members of an identity-provider group can get a role without a per-person -assignment. Add `groups_claim` and `group_roles` to the JSON file passed to `--oidc-config`: +assignment. Add `groups_claim` and `group_roles` to the `[oidc]` section of `server.toml`: -```json -{ - "origin": "https://control.example.com", - "issuer": "https://login.example.com", - "client_id": "orders-control", - "identities": {"orders-person": "0190c3c1-5a4e-7b8e-9f00-2a7d1c9e4b11"}, - "groups_claim": "groups", - "group_roles": [ - {"group": "orders-operators", "role": "operator", "location_id": "0190c3c1-6b2f-7c3d-8e44-5f1a2b3c4d5e", "target": null}, - {"group": "orders-readers", "role": "viewer", "location_id": null, "target": null} - ] -} +```toml +[oidc] +origin = "https://control.example.com" +issuer = "https://login.example.com" +client_id = "orders-control" +groups_claim = "groups" + +[oidc.identities] +orders-person = "0190c3c1-5a4e-7b8e-9f00-2a7d1c9e4b11" + +[[oidc.group_roles]] +group = "orders-operators" +role = "operator" +location_id = "0190c3c1-6b2f-7c3d-8e44-5f1a2b3c4d5e" + +[[oidc.group_roles]] +group = "orders-readers" +role = "viewer" ``` | Key | Meaning | @@ -348,6 +413,58 @@ Details: - An invalid value fails publication with the job's name, such as `job nightly_orders overlap must be queue or skip, not coalesce`. +## Limit worker containers + +A Docker worker starts with its location container's memory, CPU and process limits. Set limits +for every run of a launcher in its `run` table, and for one job with `resources`: + +```toml +[[launchers.docker]] +location = "orders" +target = "development" +server_url = "http://server:8000" +max_workers = 2 +socket = "/var/run/docker.sock" +location_container = "orders-location" +executable = "/usr/local/bin/sqb" + +[launchers.docker.run] +memory = "1g" +cpus = 1 +pids = 512 +max_memory = "4g" +``` + +```python +from sqlbuild.orchestration import job, resources + +nightly_orders = job( + "nightly_orders", + description="Orders and their daily totals", + select="+order_totals", + resources=resources(memory="3g", cpus=2), +) +``` + +| Setting | `run` table | `resources` | Value | +| --- | --- | --- | --- | +| `memory` | Yes | Yes | Bytes, or a size such as `"512m"` or `"2g"` | +| `cpus` | Yes | Yes | A positive number, such as `0.5` | +| `pids` | Yes | Yes | The most processes, a positive whole number | +| `cgroup_parent` | Yes | No | The cgroup workers start under | +| `max_memory` | Yes | No | The most memory any worker of the launcher gets | + +- A job's `resources` win over the launcher's `run` table, which wins over the location + container's limits. +- A worker never gets more than `max_memory`: a job that asks for more gets `max_memory`. + Without `max_memory`, a job's `resources` can't go above the location container's memory, CPU + or process limits; set `max_memory` to let jobs ask for more memory than the location has. +- Workers keep the location container's dropped capabilities and security options, such as + `no-new-privileges`. +- A job in a group uses its own `resources`. Local launchers ignore `resources`. +- An invalid value fails publication with the job's name, such as + `job nightly_orders resources: cpus must be a positive number`. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 265d16d608..1b042a5e27 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -16,6 +16,7 @@ from sqlbuild.orchestration_declarations.models.authoring import ( PlatformEventCondition as platform_event, ) +from sqlbuild.orchestration_declarations.models.authoring import Resources as resources from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor @@ -43,6 +44,7 @@ "node_event", "on_failure", "platform_event", + "resources", "run_event", "schedule", "sensor", diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index dbc3f4e8c1..f127ece7e5 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -29,6 +29,15 @@ class OnFailure: retry_delay_seconds: int = 0 +@dataclass(frozen=True, kw_only=True) +class Resources: + """Worker container limits for a job; memory is bytes or a size such as "512m" or "2g".""" + + memory: int | str | None = None + cpus: float | None = None + pids: int | None = None + + @dataclass(frozen=True, eq=False) class Job: """A named selection to execute, optionally on a wall-clock schedule.""" @@ -45,6 +54,7 @@ class Job: owner: str | None = field(default=None, kw_only=True) overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) on_failure: OnFailure | None = field(default=None, kw_only=True) + resources: Resources | None = field(default=None, kw_only=True) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b9664a8885..a217b17815 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -122,6 +122,9 @@ class WorkerExecutionTestCase: store_edit: Callable[[sqlite3.Connection], None] = lambda _: None concurrency_limits: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" + # A Docker launcher's [launchers.docker.run] settings and the worker HostConfig they produce. + launcher_settings: dict[str, Any] = field(default_factory=dict) + expected_host_config: dict[str, Any] = field(default_factory=dict) warehouse_setup: Callable[[Path], None] = lambda _: None # Each run node as (key, status, error code, skipped because of, error message). expected_run_nodes: list[tuple[str, str, str | None, str | None, str | None]] = field( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py index 74b846395e..b8121c3fae 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py @@ -29,7 +29,9 @@ def __init__(self, directory: Path) -> None: self.environment = os.environ | { "COMPOSE_PROJECT_NAME": self.name, "SQLBUILD_PORT": "0", - "DOCKER_GID": str(Path("/var/run/docker.sock").stat().st_gid), + "DOCKER_GID": str( + Path(os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")).stat().st_gid + ), "POSTGRES_PASSWORD": password, "SQLBUILD_CONTROL_DATABASE_URL": f"postgresql://sqlbuild:{password}@postgres:5432/sqlbuild", "SQLBUILD_SERVER_IMAGE": os.environ.get( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py index e85570e8ee..559649c228 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -81,6 +81,12 @@ def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeploy name, "--network", "host", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--memory", + "768m", "--user", f"{os.getuid()}:{os.getgid()}", "--workdir", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index ce40130425..2b37e4fa84 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -724,6 +724,33 @@ def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: return result +def toml_table(value: dict[str, Any]) -> str: + """Inline TOML table; every key is quoted so any name survives.""" + return ( + "{" + + ", ".join(f"{json.dumps(str(key))} = {toml_value(item)}" for key, item in value.items()) + + "}" + ) + + +def toml_array(value: list[Any]) -> str: + """Inline TOML array.""" + return "[" + ", ".join(toml_value(item) for item in value) + "]" + + +def toml_value(value: Any) -> str: + """One-line TOML for JSON-shaped settings; scalars share JSON's spelling.""" + writers: dict[type, Callable[[Any], str]] = {dict: toml_table, list: toml_array} + return writers.get(type(value), json.dumps)(value) + + +def write_server_config(path: Path, sections: dict[str, Any]) -> None: + """Write `server.toml` with each section as an inline table.""" + _ = path.write_text( + "".join(f"{name} = {toml_value(value)}\n" for name, value in sections.items()) + ) + + def native_launcher(directory: Path, executable: str) -> AbstractContextManager[WorkerDeployment]: """Use the installed native launcher without a synthetic child.""" return nullcontext( @@ -824,20 +851,25 @@ def scheduled_worker( ) assert published.returncode == 0, published.stderr write_files(project_dir=project, files=test_case.files_after_publication) - configuration: Path = directory / "launchers.json" - launcher: dict[str, Any] = { - "location_id": api.location_id, - "target": "development", - "server_url": api.url, - "max_workers": 1, - } | deployment.configuration - _ = configuration.write_text( - json.dumps( - { + configuration: Path = directory / "server.toml" + launcher: dict[str, Any] = ( + { + "location": "orders", + "target": "development", + "server_url": api.url, + "max_workers": 1, + } + | deployment.configuration + | test_case.launcher_settings + ) + write_server_config( + configuration, + { + "launchers": { deployment.kind: [launcher], "concurrency_limits": test_case.concurrency_limits, } - ) + }, ) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: @@ -859,7 +891,7 @@ def scheduled_worker( "scheduler", "--database-env", "SQLBUILD_SCHEDULER_TEST_STORE", - "--launchers", + "--config", str(configuration), ], log, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py index f242d18c37..878a46f425 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py @@ -20,6 +20,10 @@ from urllib.parse import SplitResult, parse_qs, urlencode, urlsplit from urllib.request import HTTPCookieProcessor, HTTPSHandler, Request, build_opener +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + write_server_config, +) + @contextmanager def gateway(root: Path) -> Iterator[tuple[str, Path, dict[str, str]]]: @@ -179,14 +183,14 @@ def authentication_arguments( client_id="orders-control", certificate_authority=str(certificate), ) - flag = "--oidc-config" + section = "oidc" else: assert mode == "proxy" configuration["trusted_peers"] = ["127.0.0.1"] - flag = "--trusted-proxy-config" - path: Path = certificate.parent / "authentication.json" - path.write_text(json.dumps(configuration)) - return flag, str(path) + section = "trusted_proxy" + path: Path = certificate.parent / "server.toml" + write_server_config(path, {section: configuration}) + return "--config", str(path) def sign_in(client: Any, origin: str, mode: str, username: str, password: str) -> str: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py index 30736d5b25..1d61339dcc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py @@ -94,12 +94,13 @@ def test_given_packaged_services_when_redeployed_and_restarted_then_pinned_runs_ stack.docker("inspect", "--format", "{{json .Mounts}}", worker).stdout ) assert {mount["Destination"] for mount in mounts} == {"/warehouse"} - assert ( + # Podman reports a named network's NetworkMode as `bridge`, so check the attachment. + networks: dict[str, Any] = json.loads( stack.docker( - "inspect", "--format", "{{.HostConfig.NetworkMode}}", worker - ).stdout.strip() - == f"{stack.name}_workers" + "inspect", "--format", "{{json .NetworkSettings.Networks}}", worker + ).stdout ) + assert set(networks) == {f"{stack.name}_workers"} assert ( stack.compose( "exec", @@ -196,9 +197,12 @@ def test_given_packaged_services_when_redeployed_and_restarted_then_pinned_runs_ # Even a valid location bearer cannot publish without physical image evidence. stack.compose("rm", "--force", "location") + # Docker CLI 29 sends a zero process limit that Podman applies; -1 is unlimited on both. unavailable: subprocess.CompletedProcess[str] = stack.docker( "run", "--rm", + "--pids-limit", + "-1", "--network", f"{stack.name}_workers", "--env", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index acd4cde4b9..69792e82d5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -1,5 +1,6 @@ """A published schedule executes in a real pinned container, not a mocked launcher.""" +import json import signal import sqlite3 import sys @@ -64,6 +65,37 @@ ) }, ), + WorkerExecutionTestCase( + "launcher defaults and job resources bound the worker, capped by max_memory", + "SELECT 41 AS amount", + "SELECT amount FROM orders", + "success", + [(41,)], + launcher="docker", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py" + ).read_text() + }, + launcher_settings={ + "run": {"memory": "768m", "cpus": 2, "pids": 256, "max_memory": "1g"} + }, + expected_host_config={"Memory": 1 << 30, "NanoCpus": 500_000_000, "PidsLimit": 256}, + ), + WorkerExecutionTestCase( + "without max_memory a job cannot raise memory above its location container's", + "SELECT 43 AS amount", + "SELECT amount FROM orders", + "success", + [(43,)], + launcher="docker", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py" + ).read_text() + }, + expected_host_config={"Memory": 768 << 20, "NanoCpus": 500_000_000}, + ), WorkerExecutionTestCase( "container warehouse error retains failed node evidence", "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS BIGINT) AS amount FROM source", @@ -91,6 +123,29 @@ def test_given_a_scheduled_docker_job_when_published_then_a_pinned_worker_execut time.sleep(0.02) state = worker_container(worker.docker, worker.api.location_id, run["run_id"]) assert state["ExitCode"] == {"success": 0, "failed": 1}[test_case.expected_status] + name: str = f"sqlbuild-{worker.api.location_id}-{run['run_id']}" + host: dict[str, Any] = json.loads( + docker_command( + worker.docker.socket, "inspect", "--format", "{{json .HostConfig}}", name + ) + ) + location: dict[str, Any] = json.loads( + docker_command( + worker.docker.socket, + "inspect", + "--format", + "{{json .HostConfig}}", + worker.docker.container, + ) + ) + assert {key: host[key] for key in test_case.expected_host_config} == ( + test_case.expected_host_config + ) + assert location["CapDrop"] and location["SecurityOpt"] + assert (host["CapDrop"], host["SecurityOpt"]) == ( + location["CapDrop"], + location["SecurityOpt"], + ) @REQUIRES_DOCKER @@ -239,8 +294,8 @@ def test_given_an_executing_container_when_scheduler_restarts_then_the_same_work "scheduler", "--database-env", "SQLBUILD_SCHEDULER_TEST_STORE", - "--launchers", - str(tmp_path / "launchers.json"), + "--config", + str(tmp_path / "server.toml"), ], log, tmp_path / "replacement-scheduler.stdout", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py index 7aeb27703f..73c8397482 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -71,8 +71,8 @@ def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_an "scheduler", "--database-env", "SQLBUILD_SCHEDULER_TEST_STORE", - "--launchers", - str(tmp_path / "launchers.json"), + "--config", + str(tmp_path / "server.toml"), ], log, tmp_path / "replacement-scheduler.stdout", @@ -157,8 +157,8 @@ def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_w "scheduler", "--database-env", "SQLBUILD_SCHEDULER_TEST_STORE", - "--launchers", - str(tmp_path / "launchers.json"), + "--config", + str(tmp_path / "server.toml"), ] handle: int = os.pidfd_open(pid) try: @@ -288,8 +288,8 @@ def test_given_a_worker_lost_with_its_scheduler_when_restarted_then_the_run_fail "scheduler", "--database-env", "SQLBUILD_SCHEDULER_TEST_STORE", - "--launchers", - str(tmp_path / "launchers.json"), + "--config", + str(tmp_path / "server.toml"), ], log, tmp_path / "replacement-scheduler.stdout", diff --git a/website/src/content/docs/docs/cli/dev.mdx b/website/src/content/docs/docs/cli/dev.mdx new file mode 100644 index 0000000000..776b7571df --- /dev/null +++ b/website/src/content/docs/docs/cli/dev.mdx @@ -0,0 +1,50 @@ +--- +title: "sqb dev" +description: "Run the server, scheduler and a reloading code location for one project on your machine." +sidebar: + label: "dev" +--- + +Run the control API, the scheduler and a code location for the project in the current directory, +and open the web UI at the address it prints: + +```bash +sqb dev +``` + +```text +Development API ready: 127.0.0.1:8000 +Development location: 01a1d7ff-0b3a-7631-ba35-e44d44d072f2 +Authentication disabled on loopback; triggers start paused +``` + +The location compiles the project and publishes it to the server. When you save a model, +orchestration file or other project input, it compiles and publishes again, so the UI and the +next run use the new version. + +## Options + +| Option | Default | Effect | +| --- | --- | --- | +| `--port` | `8000` | Port to listen on, on `127.0.0.1`. `0` picks a free port | +| `--target` | The configured target | Target to compile and run | +| `--control-store-url` | `.sqlbuild/control.sqlite3` in the project | Control store URL or path | +| `--no-scheduler` | Off | Serve and reload without running scheduled work | +| `--triggers` | Off | Keep each trigger's default status and your changes to it, instead of pausing every trigger at start-up | + +## Run scheduled jobs locally + +Triggers start paused, so a schedule never runs on your machine by accident. Resume a trigger in +the UI to let it run, or start with `--triggers` to keep the status each trigger was declared with: + +```bash +sqb dev --triggers +``` + +## Details + +- `sqb dev` listens only on loopback and needs no sign-in. Use [`sqb server`](/docs/cli/server/) + for a server other people or machines connect to. +- The control store keeps runs, trigger changes and history between sessions. Delete + `.sqlbuild/control.sqlite3` to start again from nothing. +- Stop it with Ctrl+C; it prints `Development services stopped successfully`. diff --git a/website/src/content/docs/docs/cli/location.mdx b/website/src/content/docs/docs/cli/location.mdx new file mode 100644 index 0000000000..c40cac7fba --- /dev/null +++ b/website/src/content/docs/docs/cli/location.mdx @@ -0,0 +1,52 @@ +--- +title: "sqb location" +description: "Run a code location that publishes a project to the server, and check its health." +sidebar: + label: "location" +--- + +Run a code location: it compiles the project in its own Python environment, publishes it to the +server, and publishes again when the project changes: + +```bash +export SQLBUILD_LOCATION_TOKEN=... +sqb location serve --server https://control.example.com --location-id "$LOCATION_ID" --target production +``` + +Create the location and its token with `sqb admin create-location orders` and +`sqb admin issue-location-token` on the server's host. The location reads the token from +`SQLBUILD_LOCATION_TOKEN` and never opens the control database. + +## Check a location's health + +Ask the server how a location and target are doing, for example from a container health check: + +```bash +sqb location health --server https://control.example.com --location-id "$LOCATION_ID" --target production +``` + +```json +{"location_id":"01a1d7ff-0b3a-7631-ba35-e44d44d072f2","target":"production","state":"healthy"} +``` + +| `state` | Meaning | Exit code | +| --- | --- | --- | +| `healthy` | An instance checked in within 90 seconds and its last load published | `0` | +| `degraded` | The warehouse connection check after the last load failed | `0` | +| `falling_back` | The last load failed, so runs use the previous snapshot | `1` | +| `broken` | No instance has checked in for 90 seconds | `1` | + +The command exits `2` when it cannot read the state: the server is unreachable, or the token is +missing, expired or for another location. It needs the location's token, not a person's sign-in. + +A Compose health check for the location service: + +```yaml +healthcheck: + test: [CMD, sqb, location, health, --server, "http://server:8000", --location-id, "${SQLBUILD_LOCATION_ID}", --target, development] + interval: 30s + start_period: 60s +``` + +A `falling_back` location still runs scheduled jobs from its last good snapshot. Restarting it +does not fix the failed load; fix the project and deploy it again. diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2037da023e..a54dca2394 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -5,13 +5,66 @@ sidebar: label: "server" --- -Run the control API and scheduler without loading project Python: +Run the control API and scheduler without loading project Python, with its settings in +`server.toml`: ```bash -sqb server +sqb server --config server.toml ``` -Use `sqb server --help` for service, authentication and launcher options. +## Configure the server + +`server.toml` holds the listener, retention, sign-in and the launchers that start workers. The +server reads it once at start-up, so restart the server after you change it: + +```toml +[server] +host = "0.0.0.0" +port = 8000 +default_role = "viewer" + +[retention] +run_log_days = 7 + +[trusted_proxy] +origin = "https://control.example.com" +trusted_peers = ["10.0.0.5"] + +[trusted_proxy.identities] +orders-person = "0190c3c1-5a4e-7b8e-9f00-2a7d1c9e4b11" + +[[launchers.docker]] +location = "orders" +target = "development" +server_url = "http://server:8000" +max_workers = 2 +socket = "/var/run/docker.sock" +location_container = "orders-location" +executable = "/usr/local/bin/sqb" +``` + +| Section | Keys | +| --- | --- | +| `[server]` | `host` (default `127.0.0.1`), `port` (`8000`), `database_env` (`SQLBUILD_CONTROL_DATABASE_URL`), `password_origin`, `default_role` (`none`), `scheduler` (`true`) | +| `[retention]` | `statistics_days`, `audit_days`, `snapshot_count`, `run_log_days`, `run_log_count`; see [Set retention](#set-retention) | +| `[oidc]` | `origin`, `issuer`, `client_id`, `identities`, and optionally `client_secret_env`, `certificate_authority`, `groups_claim` and `group_roles` | +| `[trusted_proxy]` | `origin`, `trusted_peers` and `identities` | +| `[[launchers.docker]]` | `location`, `target`, `server_url`, `max_workers`, `socket`, `location_container`, `executable`, and an optional `run` table; see [Limit worker containers](#limit-worker-containers) | +| `[[launchers.local]]` | `location`, `target`, `server_url`, `max_workers`, and absolute `project_dir` and `executable` paths | +| `[launchers.concurrency_limits]` | The most runs at once for each job `concurrency_groups` name, such as `orders_pool = 2` | + +- A flag overrides the same setting in the file: `sqb server --config server.toml --port 9000` + listens on port 9000. +- Keep secrets in environment variables. The file names the variable, as `database_env` and + `client_secret_env` do, never the secret itself. +- Each launcher names a location by the name you gave `sqb admin create-location`. Create the + location before you start the server; an unknown name stops it. +- `sqb scheduler --config server.toml` reads the same file for its control database and launchers. +- `sqb admin` doesn't read the file. When `database_env` names another variable, pass the same + name to `sqb admin --database-env` so it changes the server's store. +- An unknown section or key stops the server with the line it is on. + +Use `sqb server --help` for the remaining options. ## Serve over HTTPS @@ -101,21 +154,25 @@ the control server is older than this worker and refused its run report; upgrade ## Set retention -Set how much history the server keeps: +Set how much history the server keeps in the `[retention]` section of `server.toml`: -```bash -sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 \ - --run-log-retention-days 7 --run-log-retention-count 100 +```toml +[retention] +statistics_days = 30 +audit_days = 730 +snapshot_count = 20 +run_log_days = 7 +run_log_count = 100 ``` -| Data | Option | Default | Range | -| --- | --- | --- | --- | -| Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | -| Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | -| Published snapshots per location and target | `--snapshot-retention-count` | `10` | `1`–`65535` | -| Logs of finished runs | `--run-log-retention-days` | `14` | `1`–`65535` | -| Finished runs per location and target that keep their logs | `--run-log-retention-count` | `200` | `1`–`65535` | -| Skipped and covered schedule ticks | None | 30 days | Fixed | +| Data | `[retention]` key | Option | Default | Range | +| --- | --- | --- | --- | --- | +| Hourly warehouse statistics | `statistics_days` | `--statistics-retention-days` | `90` | `1`–`65535` | +| Security audit records | `audit_days` | `--audit-retention-days` | `400` | `1`–`65535` | +| Published snapshots per location and target | `snapshot_count` | `--snapshot-retention-count` | `10` | `1`–`65535` | +| Logs of finished runs | `run_log_days` | `--run-log-retention-days` | `14` | `1`–`65535` | +| Finished runs per location and target that keep their logs | `run_log_count` | `--run-log-retention-count` | `200` | `1`–`65535` | +| Skipped and covered schedule ticks | None | None | 30 days | Fixed | Statistics samples older than the horizon disappear from statistics reads at once and are then deleted in the background. @@ -137,7 +194,7 @@ deletes its node definitions and SQL text when no other snapshot shares them. Re interrupting this cleanup is safe. A finished run loses its log, the events, statements, SQL and output its worker recorded, when it -is older than `--run-log-retention-days` or when more than `--run-log-retention-count` newer runs +is older than `run_log_days` or when more than `run_log_count` newer runs have finished in its location and target, whichever comes first. The run itself, its node results, and the run history the scheduler recorded stay, so run lists, node status and history are unchanged; only the run's log, statements and SQL are no longer shown. A running run keeps its log. @@ -157,20 +214,26 @@ webhook, so give them the same values and webhook. ## Grant roles to OIDC groups (beta) With OIDC sign-in, members of an identity-provider group can get a role without a per-person -assignment. Add `groups_claim` and `group_roles` to the JSON file passed to `--oidc-config`: +assignment. Add `groups_claim` and `group_roles` to the `[oidc]` section of `server.toml`: -```json -{ - "origin": "https://control.example.com", - "issuer": "https://login.example.com", - "client_id": "orders-control", - "identities": {"orders-person": "0190c3c1-5a4e-7b8e-9f00-2a7d1c9e4b11"}, - "groups_claim": "groups", - "group_roles": [ - {"group": "orders-operators", "role": "operator", "location_id": "0190c3c1-6b2f-7c3d-8e44-5f1a2b3c4d5e", "target": null}, - {"group": "orders-readers", "role": "viewer", "location_id": null, "target": null} - ] -} +```toml +[oidc] +origin = "https://control.example.com" +issuer = "https://login.example.com" +client_id = "orders-control" +groups_claim = "groups" + +[oidc.identities] +orders-person = "0190c3c1-5a4e-7b8e-9f00-2a7d1c9e4b11" + +[[oidc.group_roles]] +group = "orders-operators" +role = "operator" +location_id = "0190c3c1-6b2f-7c3d-8e44-5f1a2b3c4d5e" + +[[oidc.group_roles]] +group = "orders-readers" +role = "viewer" ``` | Key | Meaning | @@ -333,6 +396,58 @@ Details: - An invalid value fails publication with the job's name, such as `job nightly_orders overlap must be queue or skip, not coalesce`. +## Limit worker containers + +A Docker worker starts with its location container's memory, CPU and process limits. Set limits +for every run of a launcher in its `run` table, and for one job with `resources`: + +```toml +[[launchers.docker]] +location = "orders" +target = "development" +server_url = "http://server:8000" +max_workers = 2 +socket = "/var/run/docker.sock" +location_container = "orders-location" +executable = "/usr/local/bin/sqb" + +[launchers.docker.run] +memory = "1g" +cpus = 1 +pids = 512 +max_memory = "4g" +``` + +```python +from sqlbuild.orchestration import job, resources + +nightly_orders = job( + "nightly_orders", + description="Orders and their daily totals", + select="+order_totals", + resources=resources(memory="3g", cpus=2), +) +``` + +| Setting | `run` table | `resources` | Value | +| --- | --- | --- | --- | +| `memory` | Yes | Yes | Bytes, or a size such as `"512m"` or `"2g"` | +| `cpus` | Yes | Yes | A positive number, such as `0.5` | +| `pids` | Yes | Yes | The most processes, a positive whole number | +| `cgroup_parent` | Yes | No | The cgroup workers start under | +| `max_memory` | Yes | No | The most memory any worker of the launcher gets | + +- A job's `resources` win over the launcher's `run` table, which wins over the location + container's limits. +- A worker never gets more than `max_memory`: a job that asks for more gets `max_memory`. + Without `max_memory`, a job's `resources` can't go above the location container's memory, CPU + or process limits; set `max_memory` to let jobs ask for more memory than the location has. +- Workers keep the location container's dropped capabilities and security options, such as + `no-new-privileges`. +- A job in a group uses its own `resources`. Local launchers ignore `resources`. +- An invalid value fails publication with the job's name, such as + `job nightly_orders resources: cpus must be a positive number`. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 10ca9b2fcf..cee743e406 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -293,6 +293,8 @@ "label": "Server", "items": [ "docs/cli/server", + "docs/cli/location", + "docs/cli/dev", "docs/cli/login" ] }