Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 64 additions & 7 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ jobs:
wheels: ${{ steps.scope.outputs.wheels }}
recovery: ${{ steps.scope.outputs.recovery }}
ui: ${{ steps.scope.outputs.ui }}
podman: ${{ steps.scope.outputs.podman }}
cold_models: ${{ steps.tiers.outputs.cold_models }}
dense_models: ${{ steps.tiers.outputs.dense_models }}
cache_models: ${{ steps.tiers.outputs.cache_models }}
Expand Down Expand Up @@ -79,11 +80,12 @@ jobs:
echo "clickhouse=$9"
echo "recovery=${10}"
echo "ui=${11}"
echo "podman=${12}"
} >> "${GITHUB_OUTPUT}"
}

if [[ "${EVENT_NAME}" != "pull_request" && "${EVENT_NAME}" != "merge_group" ]]; then
write_scope true true true true true true true true true true true
write_scope true true true true true true true true true true true true
exit 0
fi

Expand All @@ -95,7 +97,7 @@ jobs:
if [[ -z "${BASE_SHA}" || -z "${HEAD_SHA}" ]] || \
! git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}" > "${changed_paths}"; then
echo "Could not diff ${BASE_SHA:-<none>}...${HEAD_SHA:-<none>}; running every suite" >&2
write_scope true true true true true true true true true true true
write_scope true true true true true true true true true true true true
exit 0
fi

Expand Down Expand Up @@ -144,7 +146,7 @@ jobs:
&& "${HEAD_REF}" == release-please--branches--* ]]; } && release_metadata_only; then
echo "The change is only release metadata; skipping the suites" \
| tee -a "${GITHUB_STEP_SUMMARY}"
write_scope false false false false false false false false false false false
write_scope false false false false false false false false false false false false
exit 0
fi

Expand All @@ -158,6 +160,7 @@ jobs:
compiler_differential=false
recovery=false
ui=false
podman=false
broad=false
changed=false
# A change that only touches the website, root-level Markdown, the README demo GIFs or the
Expand Down Expand Up @@ -204,6 +207,14 @@ jobs:
recovery=true
;;
esac
case "${path}" in
# The reference stack and the Docker launcher, which Podman serves through its
# Docker-compatible API.
# Every path here also selects the wheels the job installs.
deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|crates/sqlbuild-cli/src/workers/*|crates/sqlbuild-cli/src/server_config/*|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|.github/workflows/verify.yml)
podman=true
;;
esac
case "${path}" in
scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|tests/goldens/*|tests/e2e/fixtures/*|website/examples/*|.github/workflows/verify.yml|Makefile)
compiler_differential=true
Expand Down Expand Up @@ -271,7 +282,7 @@ jobs:
done < "${changed_paths}"

if [[ "${changed}" != "true" ]]; then
write_scope true true true true true true true true true true true
write_scope true true true true true true true true true true true true
exit 0
fi

Expand All @@ -298,7 +309,8 @@ jobs:
"${compiler_differential}" \
"${clickhouse}" \
"${recovery}" \
"${ui}"
"${ui}" \
"${podman}"

# Pull requests and merge groups run the 3k-model compile jobs and the 1k dense job; the
# nightly run and manual runs add 5k and 10k. Budgets and ratio gates are the same in both.
Expand Down Expand Up @@ -1231,6 +1243,46 @@ jobs:
- name: Verify packaged Compose deployment and operational recovery
run: make test-compose-packaged

podman:
name: Packaged Compose lifecycle on Podman
needs: [changes, lockfile, wheels]
if: needs.changes.outputs.podman == 'true' && needs.changes.outputs.wheels == 'true' && needs.changes.outputs.full == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 25
env:
# The Docker CLI and its Compose plugin drive rootful Podman through its API socket.
DOCKER_HOST: unix:///run/podman/podman.sock
DOCKER_SOCKET: /run/podman/podman.sock
# Podman has no BuildKit API: buildx would build in its own container and never load the image.
DOCKER_BUILDKIT: "0"
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup-uv
with:
version: "0.12.3"
enable-cache: true
- name: Install Python test dependencies without rebuilding the packages
run: |
uv python install 3.12
uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli
- uses: actions/download-artifact@v4
with:
name: dist-wheel-linux-x86_64
path: dist
- name: Install the same wheels used by the images
run: uv pip install --no-deps dist/*.whl
- name: Start the Podman API socket with group access for the runner
run: |
sudo systemctl start podman.socket
sudo chmod 0711 /run/podman
sudo chgrp "$(id -g)" /run/podman/podman.sock
sudo chmod 0660 /run/podman/podman.sock
podman --version
docker version --format 'Podman API {{.Server.Version}}'
docker compose version
- name: Verify packaged Compose deployment and operational recovery
run: make test-compose-packaged

ui:
name: UI checks and unit tests
needs: changes
Expand Down Expand Up @@ -1296,6 +1348,7 @@ jobs:
clickhouse,
wheels,
compose,
podman,
ui,
]
runs-on: ubuntu-24.04
Expand Down Expand Up @@ -1323,6 +1376,7 @@ jobs:
CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }}
WHEELS_RESULT: ${{ needs.wheels.result }}
COMPOSE_RESULT: ${{ needs.compose.result }}
PODMAN_RESULT: ${{ needs.podman.result }}
UI_SELECTED: ${{ needs.changes.outputs.ui }}
UI_RESULT: ${{ needs.ui.result }}
LABEL_ONLY: ${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify' }}
Expand All @@ -1335,6 +1389,7 @@ jobs:
POSTGRES_SELECTED: ${{ needs.changes.outputs.postgres }}
CLICKHOUSE_SELECTED: ${{ needs.changes.outputs.clickhouse }}
WHEELS_SELECTED: ${{ needs.changes.outputs.wheels }}
PODMAN_SELECTED: ${{ needs.changes.outputs.podman }}
run: |
if [[ "${LABEL_ONLY}" == "true" ]]; then
echo "A label other than full-verify changed; this run verifies nothing"
Expand Down Expand Up @@ -1371,7 +1426,8 @@ jobs:
"${POSTGRES_RESULT}" \
"${CLICKHOUSE_RESULT}" \
"${WHEELS_RESULT}" \
"${COMPOSE_RESULT}"; do
"${COMPOSE_RESULT}" \
"${PODMAN_RESULT}"; do
if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then
echo "A selected verification suite did not pass: ${result}" >&2
exit 1
Expand All @@ -1398,7 +1454,8 @@ jobs:
"${POSTGRES_SELECTED}:${POSTGRES_RESULT}" \
"${CLICKHOUSE_SELECTED}:${CLICKHOUSE_RESULT}" \
"${WHEELS_SELECTED}:${WHEELS_RESULT}" \
"${WHEELS_SELECTED}:${COMPOSE_RESULT}"; do
"${WHEELS_SELECTED}:${COMPOSE_RESULT}" \
"${PODMAN_SELECTED}:${PODMAN_RESULT}"; do
if [[ "${pair%%:*}" == "true" && "${pair#*:}" != "success" ]]; then
echo "A suite this change needs did not run to success: ${pair#*:}" >&2
exit 1
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
"""A job asks for more memory than its worker may have and fewer CPUs than the launcher default."""

from sqlbuild.orchestration import job, resources, schedule

annual_orders = job(
"annual_orders",
select="orders",
description="Build orders.",
resources=resources(memory="2g", cpus=0.5),
schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"),
)
49 changes: 24 additions & 25 deletions clients/typescript/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,21 +42,21 @@ separate authentication contract, not an inferred privilege of this client.
## Authenticate through a trusted proxy

Create a person with `sqb admin create-principal orders-reader`, then put the
returned `principal_id` in a server-owned JSON file. Subjects are exact ASCII
upstream identities, not inferred local usernames or email addresses:

```json
{
"origin": "https://control.example.com",
"trusted_peers": ["127.0.0.1"],
"identities": {
"external-orders-reader": "00000000-0000-0000-0000-000000000001"
}
}
returned `principal_id` in the `[trusted_proxy]` section of the server's
`server.toml`. Subjects are exact ASCII upstream identities, not inferred local
usernames or email addresses:

```toml
[trusted_proxy]
origin = "https://control.example.com"
trusted_peers = ["127.0.0.1"]

[trusted_proxy.identities]
external-orders-reader = "00000000-0000-0000-0000-000000000001"
```

Replace the example UUID with the created person's ID, then start
`sqb server --trusted-proxy-config proxy.json`. The proxy must terminate HTTPS,
`sqb server --config server.toml`. The proxy must terminate HTTPS,
remove any client-supplied `X-SQB-User`, and replace it with the authenticated
upstream subject. Restrict access to the HTTP backend to the configured proxy
peers. Trusting loopback also trusts other local processes on that host;
Expand All @@ -74,23 +74,22 @@ is inferred from the subject.
Register an authorization-code client with your HTTPS identity provider, using
`https://control.example.com/api/auth/oidc/callback` as its redirect URI. Create
each person through the local control-store-owner administration path and put
their UUID and exact provider subject in a server-owned JSON configuration:

```json
{
"origin": "https://control.example.com",
"issuer": "https://identity.example.com",
"client_id": "orders-control",
"identities": {
"external-orders-reader": "00000000-0000-0000-0000-000000000001"
}
}
their UUID and exact provider subject in the `[oidc]` section of `server.toml`:

```toml
[oidc]
origin = "https://control.example.com"
issuer = "https://identity.example.com"
client_id = "orders-control"

[oidc.identities]
external-orders-reader = "00000000-0000-0000-0000-000000000001"
```

Start `sqb server --oidc-config oidc.json` behind an HTTPS-terminating proxy that
Start `sqb server --config server.toml` behind an HTTPS-terminating proxy that
preserves the configured Host and restricts access to the HTTP backend. For a
confidential client, add `client_secret_env` naming the server environment
variable holding its secret; never put the secret in the JSON file. To trust a
variable holding its secret; never put the secret in `server.toml`. To trust a
private provider CA, add `certificate_authority` pointing to a PEM bundle.
Relative certificate paths resolve from the server process's working directory.
TLS verification remains mandatory and upstream HTTP redirects are not followed.
Expand Down
5 changes: 4 additions & 1 deletion clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6284,7 +6284,10 @@ export interface operations {
};
read_location_health: {
parameters: {
query?: never;
query?: {
/** @description Only this target's observations */
target?: string;
};
header?: never;
path: {
location_id: string;
Expand Down
25 changes: 20 additions & 5 deletions compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: ${COMPOSE_PROJECT_NAME:-sqlbuild}

services:
postgres:
image: postgres:17.6-alpine3.22
image: docker.io/library/postgres:17.6-alpine3.22
environment:
POSTGRES_USER: sqlbuild
POSTGRES_DB: sqlbuild
Expand All @@ -24,15 +24,17 @@ services:
target: server
args:
PYTHON_VERSION: ${PYTHON_VERSION:-3.12}
SQLBUILD_SOURCE: ${SQLBUILD_SOURCE:-wheels}
SQLBUILD_VERSION: ${SQLBUILD_VERSION:-}
environment:
- SQLBUILD_CONTROL_DATABASE_URL=${SQLBUILD_CONTROL_DATABASE_URL:?Set SQLBUILD_CONTROL_DATABASE_URL}
- SQLBUILD_PLATFORM_WEBHOOK_URL
group_add: ["${DOCKER_GID:?Set DOCKER_GID to the Docker socket group}"]
volumes:
- ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock
configs:
- source: launchers
target: /etc/sqlbuild/launchers.json
- source: server
target: /etc/sqlbuild/server.toml
ports:
- "${SQLBUILD_BIND_ADDRESS:-127.0.0.1}:${SQLBUILD_PORT:-8000}:8000"
healthcheck:
Expand All @@ -54,6 +56,8 @@ services:
target: location
args:
PYTHON_VERSION: ${PYTHON_VERSION:-3.12}
SQLBUILD_SOURCE: ${SQLBUILD_SOURCE:-wheels}
SQLBUILD_VERSION: ${SQLBUILD_VERSION:-}
SQLBUILD_GIT_REVISION: ${SQLBUILD_GIT_REVISION:?Set SQLBUILD_GIT_REVISION}
command: [location, serve, --server, "http://server:8000", --location-id, "${SQLBUILD_LOCATION_ID:?Set SQLBUILD_LOCATION_ID}", --target, development]
environment:
Expand All @@ -68,9 +72,20 @@ services:
networks: [workers]

configs:
launchers:
server:
content: |
{"docker":[{"location_id":"${SQLBUILD_LOCATION_ID}","target":"development","server_url":"http://server:8000","max_workers":1,"socket":"/var/run/docker.sock","location_container":"${COMPOSE_PROJECT_NAME:-sqlbuild}-location","executable":"/usr/local/bin/sqb"}]}
[server]
host = "0.0.0.0"
port = 8000

[[launchers.docker]]
location = "orders"
target = "development"
server_url = "http://server:8000"
max_workers = 1
socket = "/var/run/docker.sock"
location_container = "${COMPOSE_PROJECT_NAME:-sqlbuild}-location"
executable = "/usr/local/bin/sqb"

volumes:
control:
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ sqlbuild-process-ownership.workspace = true
sqlbuild-recall.workspace = true
sqlbuild-rules.workspace = true
tokio.workspace = true
toml.workspace = true
uuid.workspace = true

[target.'cfg(unix)'.dependencies]
Expand Down
Loading
Loading