From b541bd50e94795e0c3753e249a5359661173bd72 Mon Sep 17 00:00:00 2001 From: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:51:10 +0000 Subject: [PATCH 1/5] feat: add downstream app worker contract Signed-off-by: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> --- .docker/downstream-app.yml | 13 +++ .github/workflows/proxy-tests.yml | 40 +++++++ README.md | 1 + dev-worker | 62 +++++++++- docs/apps-development.md | 41 ++++--- docs/downstream-consumers.md | 108 ++++++++++++++++++ tests/worker/contract.bats | 34 ++++++ .../fixtures/other-app/appinfo/info.xml | 18 +++ tests/worker/fixtures/other-app/marker.txt | 1 + .../fixtures/sample-app/appinfo/info.xml | 18 +++ tests/worker/fixtures/sample-app/marker.txt | 1 + 11 files changed, 315 insertions(+), 22 deletions(-) create mode 100644 .docker/downstream-app.yml create mode 100644 docs/downstream-consumers.md create mode 100644 tests/worker/fixtures/other-app/appinfo/info.xml create mode 100644 tests/worker/fixtures/other-app/marker.txt create mode 100644 tests/worker/fixtures/sample-app/appinfo/info.xml create mode 100644 tests/worker/fixtures/sample-app/marker.txt diff --git a/.docker/downstream-app.yml b/.docker/downstream-app.yml new file mode 100644 index 00000000..c343423e --- /dev/null +++ b/.docker/downstream-app.yml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +services: + nextcloud: + volumes: + - ${APP_SOURCE_DIR:?APP_SOURCE_DIR is required}:${APP_TARGET_DIR:?APP_TARGET_DIR is required} + nginx: + volumes: + - ${APP_SOURCE_DIR:?APP_SOURCE_DIR is required}:${APP_TARGET_DIR:?APP_TARGET_DIR is required}:ro + playwright: + volumes: + - ${APP_SOURCE_DIR:?APP_SOURCE_DIR is required}:${APP_TARGET_DIR:?APP_TARGET_DIR is required} diff --git a/.github/workflows/proxy-tests.yml b/.github/workflows/proxy-tests.yml index 62a893a2..7b76eb79 100644 --- a/.github/workflows/proxy-tests.yml +++ b/.github/workflows/proxy-tests.yml @@ -155,3 +155,43 @@ jobs: if [ "$MARIADB_VERSION" = 10.11 ]; then DB_TYPE=mariadb sh ./dev-worker maria-peer destroy || true fi + + downstream-consumer: + name: Downstream consumer integration + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Build PHP 8.3 development image + run: docker build -f .docker/Dockerfile.php83 -t ghcr.io/librecodecoop/nextcloud-dev-php83:latest .docker + - name: Prepare independent downstream checkouts + run: | + cp -a tests/worker/fixtures/sample-app /tmp/consumer-a + cp -a tests/worker/fixtures/sample-app /tmp/consumer-b + printf 'alpha\n' > /tmp/consumer-a/marker.txt + printf 'beta\n' > /tmp/consumer-b/marker.txt + - name: Start downstream workers and run setup hooks + run: | + APP_ID=sample_app APP_SOURCE_DIR=/tmp/consumer-a APP_SETUP_COMMAND='printf setup-a > /var/www/html/data/downstream-setup' DB_TYPE=sqlite sh ./dev-worker consumer-a up + APP_ID=sample_app APP_SOURCE_DIR=/tmp/consumer-b APP_SETUP_COMMAND='printf setup-b > /var/www/html/data/downstream-setup' DB_TYPE=sqlite sh ./dev-worker consumer-b up + - name: Verify checkout, hook, URLs, and isolation + run: | + test "$(DB_TYPE=sqlite sh ./dev-worker consumer-a exec cat /var/www/html/apps-extra/sample_app/marker.txt)" = alpha + test "$(DB_TYPE=sqlite sh ./dev-worker consumer-b exec cat /var/www/html/apps-extra/sample_app/marker.txt)" = beta + test "$(DB_TYPE=sqlite sh ./dev-worker consumer-a exec cat /var/www/html/data/downstream-setup)" = setup-a + test "$(DB_TYPE=sqlite sh ./dev-worker consumer-b exec cat /var/www/html/data/downstream-setup)" = setup-b + DB_TYPE=sqlite sh ./dev-worker consumer-a urls | grep -q 'https://ncdev-consumer-a.localhost' + DB_TYPE=sqlite sh ./dev-worker consumer-b urls | grep -q 'https://ncdev-consumer-b-mailpit.localhost' + - name: Verify teardown affects only the selected worker + run: | + DB_TYPE=sqlite sh ./dev-worker consumer-a destroy + DB_TYPE=sqlite sh ./dev-worker consumer-b status | grep -q 'installed: true' + test "$(DB_TYPE=sqlite sh ./dev-worker consumer-b exec cat /var/www/html/apps-extra/sample_app/marker.txt)" = beta + - name: Collect worker logs + if: failure() + run: | + DB_TYPE=sqlite sh ./dev-worker consumer-a logs || true + DB_TYPE=sqlite sh ./dev-worker consumer-b logs || true + - name: Destroy remaining worker + if: always() + run: DB_TYPE=sqlite sh ./dev-worker consumer-b destroy || true diff --git a/README.md b/README.md index 21d20d30..cde99b79 100644 --- a/README.md +++ b/README.md @@ -21,4 +21,5 @@ and other advanced configuration, see the - [Advanced setup](docs/advanced-setup.md) - [App development](docs/apps-development.md) +- [Downstream app and devcontainer contract](docs/downstream-consumers.md) - [FAQ](docs/faq.md) \ No newline at end of file diff --git a/dev-worker b/dev-worker index 28131854..e407b76e 100644 --- a/dev-worker +++ b/dev-worker @@ -14,6 +14,7 @@ Commands: logs Print worker logs config Render the resolved Compose configuration status Run occ status in the worker + urls Print the worker's Nextcloud and Mailpit URLs db-exec Run a command in the selected database container destroy Stop the worker and remove only its mutable state @@ -24,6 +25,9 @@ Environment: DB_SQL_MODE= Optional global SQL mode for MySQL-compatible backends PHP_VERSION= Existing PHP image selector VERSION_NEXTCLOUD= Existing Nextcloud ref selector + APP_ID= Optional downstream Nextcloud app identifier + APP_SOURCE_DIR= Optional downstream app checkout to mount + APP_SETUP_COMMAND= Optional idempotent command run after readiness WORKER_READY_TIMEOUT=180 Seconds to wait for a clean installation EOF } @@ -65,6 +69,49 @@ esac worker_root="$repo_root/.workers/$worker_id" worker_volumes_dir="$worker_root/volumes" +worker_app_id_file="$worker_root/app-id" +worker_app_source_file="$worker_root/app-source" + +stored_app_id="" +stored_app_source="" +[ ! -f "$worker_app_id_file" ] || stored_app_id="$(cat "$worker_app_id_file")" +[ ! -f "$worker_app_source_file" ] || stored_app_source="$(cat "$worker_app_source_file")" + +app_id="${APP_ID:-$stored_app_id}" +app_source="${APP_SOURCE_DIR:-$stored_app_source}" + +if [ -n "$app_id" ] || [ -n "$app_source" ]; then + [ -n "$app_id" ] && [ -n "$app_source" ] || { + echo "APP_ID and APP_SOURCE_DIR must be provided together" >&2 + exit 2 + } + case "$app_id" in + [a-z][a-z0-9_]*) ;; + *) + echo "Invalid APP_ID: $app_id" >&2 + exit 2 + ;; + esac + [ -d "$app_source" ] || { + echo "APP_SOURCE_DIR is not a directory: $app_source" >&2 + exit 2 + } + app_source="$(CDPATH= cd -- "$app_source" && pwd -P)" + if [ -n "$stored_app_id" ] && [ "$stored_app_id" != "$app_id" ]; then + echo "Worker $worker_id is already bound to APP_ID=$stored_app_id" >&2 + exit 2 + fi + if [ -n "$stored_app_source" ] && [ "$stored_app_source" != "$app_source" ]; then + echo "Worker $worker_id is already bound to APP_SOURCE_DIR=$stored_app_source" >&2 + exit 2 + fi + mkdir -p "$worker_root" + printf '%s\n' "$app_id" > "$worker_app_id_file" + printf '%s\n' "$app_source" > "$worker_app_source_file" + export APP_ID="$app_id" + export APP_SOURCE_DIR="$app_source" + export APP_TARGET_DIR="/var/www/html/apps-extra/$app_id" +fi export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-ncdev-$worker_id}" export WORKER_VOLUMES_DIR="$worker_volumes_dir" @@ -89,7 +136,13 @@ else fi compose() { - docker compose --project-directory "$repo_root" --file "$repo_root/docker-compose.yml" "$@" + if [ -n "${APP_SOURCE_DIR:-}" ]; then + docker compose --project-directory "$repo_root" \ + --file "$repo_root/docker-compose.yml" \ + --file "$repo_root/.docker/downstream-app.yml" "$@" + else + docker compose --project-directory "$repo_root" --file "$repo_root/docker-compose.yml" "$@" + fi } wait_for_mariadb() { @@ -151,6 +204,9 @@ case "$command" in compose up -d fi wait_until_ready + if [ -n "${APP_SETUP_COMMAND:-}" ]; then + compose exec -T nextcloud sh -lc "$APP_SETUP_COMMAND" + fi ;; exec) [ "$#" -gt 0 ] || { @@ -169,6 +225,10 @@ case "$command" in status) compose exec -T nextcloud occ status ;; + urls) + printf 'Nextcloud: https://%s.localhost\n' "$COMPOSE_PROJECT_NAME" + printf 'Mailpit: https://%s-mailpit.localhost\n' "$COMPOSE_PROJECT_NAME" + ;; db-exec) [ "$db_type" != "sqlite" ] || { echo "SQLite workers do not have a database container" >&2 diff --git a/docs/apps-development.md b/docs/apps-development.md index e7ab966f..48029398 100644 --- a/docs/apps-development.md +++ b/docs/apps-development.md @@ -1,29 +1,28 @@ # Start development of apps -You will need create (or clone) the folder of the app that you will work inside the folder `volumes/nextcloud/apps-extra`. +For new app-development workflows, keep the app checkout outside NCDD and use +the [downstream app and devcontainer contract](downstream-consumers.md). This +avoids cloning application source into NCDD's mutable Nextcloud data directory +and allows multiple isolated worktrees to reuse the same canonical runtime. -It's not required install all dependencis like php or nodejs to develop apps, with this project is only use the bash in container to compile app. +The legacy `volumes/nextcloud/apps-extra` workflow remains possible for +existing local setups, but downstream repositories should prefer the worker +contract for automation, concurrent worktrees and devcontainer integration. -## Sample +## Example -Using the [LibreSign](https://github.com/LibreSign/libresign): +```bash +APP_ID=my_app \ +APP_SOURCE_DIR=/path/to/my_app \ +DB_TYPE=sqlite \ +sh ./dev-worker my-app up -To install LibreSign in the structure of develop is required [up servicer](#up-services). After nextcloud config and install. - - open folder `volumes/nextcloud/app-extra` - - clone project with `git clone https://github.com/LibreSign/libresign.git` - - open bash in nextcloud container with `docker compose exec -u www-data nextcloud bash` - - go to folder `apps-extra/libresign` - ```bash - cd apps-extra/libresign - ``` - - Now you can run all the necessaries commands to build the project, i.e: - ```bash - # download composer dependencies - composer install - # download JS dependencies - npm ci - # build and watch JS changes - npm run watch - ``` +DB_TYPE=sqlite sh ./dev-worker my-app urls +DB_TYPE=sqlite sh ./dev-worker my-app exec sh -lc \ + 'cd /var/www/html/apps-extra/my_app && composer install' +``` + +The app checkout stays owned by the downstream repository. NCDD owns only the +isolated runtime state under `.workers/`. ⬅️ [Back to index](../README.md) diff --git a/docs/downstream-consumers.md b/docs/downstream-consumers.md new file mode 100644 index 00000000..a68804b3 --- /dev/null +++ b/docs/downstream-consumers.md @@ -0,0 +1,108 @@ +# Downstream app and devcontainer contract + +This repository can be the canonical Nextcloud runtime for downstream app +repositories. A consumer keeps its application checkout outside this repository +and mounts it into an isolated worker instead of copying the Compose topology. + +## Worker contract + +Provide an app identifier and checkout path when a worker is created: + +```bash +APP_ID=my_app \ +APP_SOURCE_DIR=/path/to/my_app \ +DB_TYPE=sqlite \ +sh ./dev-worker my-app up +``` + +The worker persists the app binding under its own `.workers/` +metadata. Later commands only need the worker id and the runtime dimensions: + +```bash +DB_TYPE=sqlite sh ./dev-worker my-app status +DB_TYPE=sqlite sh ./dev-worker my-app exec pwd +DB_TYPE=sqlite sh ./dev-worker my-app urls +DB_TYPE=sqlite sh ./dev-worker my-app destroy +``` + +The checkout is mounted at: + +```text +/var/www/html/apps-extra/ +``` + +The same checkout is visible to the Nextcloud, nginx and optional Playwright +services through `.docker/downstream-app.yml`. + +A worker cannot be rebound to a different app id or checkout path. Destroy the +worker first when you intentionally want a new binding. + +## Runtime dimensions + +The downstream contract reuses the same worker options as native NCDD workers: + +- `PHP_VERSION` +- `VERSION_NEXTCLOUD` +- `DB_TYPE` +- `MARIADB_VERSION` where applicable +- `DB_SQL_MODE` where applicable + +No fixed host application or mail port is required. Run: + +```bash +sh ./dev-worker my-app urls +``` + +to get deterministic hostnames based on the worker's Compose project name. + +## Post-readiness setup + +A downstream project can run an idempotent command inside the Nextcloud +container after the worker becomes ready: + +```bash +APP_ID=my_app \ +APP_SOURCE_DIR=/path/to/my_app \ +APP_SETUP_COMMAND='cd /var/www/html/apps-extra/my_app && composer install && occ app:enable my_app' \ +sh ./dev-worker my-app up +``` + +`APP_SETUP_COMMAND` is intentionally executed as a shell command inside the +development container. Treat it as trusted developer input; do not populate it +from untrusted issue, PR or network content. + +## Devcontainer adapters + +The runtime service intended for a downstream devcontainer is `nextcloud`. +The reusable Compose pieces are: + +```text +/path/to/nextcloud-docker-development/docker-compose.yml +/path/to/nextcloud-docker-development/.docker/downstream-app.yml +``` + +A downstream `.devcontainer` should stay thin: it may select the `nextcloud` +service and its workspace folder, while NCDD continues to own the Nextcloud, +database, proxy, mail and supporting service definitions. + +The consumer must supply the same contract variables used by `dev-worker`: +a unique `COMPOSE_PROJECT_NAME`, isolated `WORKER_VOLUMES_DIR`, +`APP_ID`, absolute `APP_SOURCE_DIR`, and +`APP_TARGET_DIR=/var/www/html/apps-extra/`. + +The LibreSign migration is tracked separately; this contract intentionally +contains no LibreSign-specific setup. + +## Isolation guarantees + +Each worker owns: + +- a Compose project name; +- a mutable volume directory; +- persisted downstream app binding metadata; +- database state when a network database is selected; +- deterministic `*.localhost` service hostnames. + +Destroying one worker removes only that worker's Compose resources and mutable +state. The downstream source checkout is a bind mount and is never deleted by +`destroy`. diff --git a/tests/worker/contract.bats b/tests/worker/contract.bats index 668074f7..e39a95c8 100644 --- a/tests/worker/contract.bats +++ b/tests/worker/contract.bats @@ -74,3 +74,37 @@ setup() { grep -q 'pdo_sqlite' "$dockerfile" done } + + +@test "downstream app checkout is mounted through the shared worker contract" { + fixture="$REPO_ROOT/tests/worker/fixtures/sample-app" + run env APP_ID=sample_app APP_SOURCE_DIR="$fixture" DB_TYPE=sqlite sh "$WORKER" consumer-a config + [ "$status" -eq 0 ] + [[ "$output" == *"$fixture:/var/www/html/apps-extra/sample_app"* ]] + [[ "$output" == *"name: ncdev-consumer-a"* ]] +} + +@test "worker remembers downstream app binding by worker id" { + fixture="$REPO_ROOT/tests/worker/fixtures/sample-app" + run env APP_ID=sample_app APP_SOURCE_DIR="$fixture" DB_TYPE=sqlite sh "$WORKER" consumer-memory config + [ "$status" -eq 0 ] + + run env DB_TYPE=sqlite sh "$WORKER" consumer-memory config + [ "$status" -eq 0 ] + [[ "$output" == *"$fixture:/var/www/html/apps-extra/sample_app"* ]] + + rm -rf "$REPO_ROOT/.workers/consumer-memory" +} + +@test "worker rejects rebinding an existing consumer to another checkout" { + fixture="$REPO_ROOT/tests/worker/fixtures/sample-app" + other="$REPO_ROOT/tests/worker/fixtures/other-app" + run env APP_ID=sample_app APP_SOURCE_DIR="$fixture" DB_TYPE=sqlite sh "$WORKER" consumer-bound config + [ "$status" -eq 0 ] + + run env APP_ID=sample_app APP_SOURCE_DIR="$other" DB_TYPE=sqlite sh "$WORKER" consumer-bound config + [ "$status" -eq 2 ] + [[ "$output" == *"already bound to APP_SOURCE_DIR"* ]] + + rm -rf "$REPO_ROOT/.workers/consumer-bound" +} diff --git a/tests/worker/fixtures/other-app/appinfo/info.xml b/tests/worker/fixtures/other-app/appinfo/info.xml new file mode 100644 index 00000000..2195178d --- /dev/null +++ b/tests/worker/fixtures/other-app/appinfo/info.xml @@ -0,0 +1,18 @@ + + + + other_app + Worker fixture app + Fixture for the downstream worker contract + Fixture for the downstream worker contract. + 1.0.0 + agpl + LibreCode coop and contributors + OtherApp + + + + diff --git a/tests/worker/fixtures/other-app/marker.txt b/tests/worker/fixtures/other-app/marker.txt new file mode 100644 index 00000000..e45c9c26 --- /dev/null +++ b/tests/worker/fixtures/other-app/marker.txt @@ -0,0 +1 @@ +other diff --git a/tests/worker/fixtures/sample-app/appinfo/info.xml b/tests/worker/fixtures/sample-app/appinfo/info.xml new file mode 100644 index 00000000..abb1a440 --- /dev/null +++ b/tests/worker/fixtures/sample-app/appinfo/info.xml @@ -0,0 +1,18 @@ + + + + sample_app + Worker fixture app + Fixture for the downstream worker contract + Fixture for the downstream worker contract. + 1.0.0 + agpl + LibreCode coop and contributors + SampleApp + + + + diff --git a/tests/worker/fixtures/sample-app/marker.txt b/tests/worker/fixtures/sample-app/marker.txt new file mode 100644 index 00000000..4a580070 --- /dev/null +++ b/tests/worker/fixtures/sample-app/marker.txt @@ -0,0 +1 @@ +alpha From eeae59260cd4b08aed3c6d5ecf0a4d060c647e33 Mon Sep 17 00:00:00 2001 From: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:53:00 +0000 Subject: [PATCH 2/5] fix: keep downstream setup ownership safe Signed-off-by: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> --- dev-worker | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dev-worker b/dev-worker index e407b76e..71aca4c9 100644 --- a/dev-worker +++ b/dev-worker @@ -86,12 +86,18 @@ if [ -n "$app_id" ] || [ -n "$app_source" ]; then exit 2 } case "$app_id" in - [a-z][a-z0-9_]*) ;; + [a-z]*) ;; *) echo "Invalid APP_ID: $app_id" >&2 exit 2 ;; esac + case "$app_id" in + *[!a-z0-9_]*) + echo "Invalid APP_ID: $app_id" >&2 + exit 2 + ;; + esac [ -d "$app_source" ] || { echo "APP_SOURCE_DIR is not a directory: $app_source" >&2 exit 2 @@ -205,7 +211,7 @@ case "$command" in fi wait_until_ready if [ -n "${APP_SETUP_COMMAND:-}" ]; then - compose exec -T nextcloud sh -lc "$APP_SETUP_COMMAND" + compose exec -T -u www-data nextcloud sh -lc "$APP_SETUP_COMMAND" fi ;; exec) From 005c6acd95a0840634eff00ff63c05f966eaae67 Mon Sep 17 00:00:00 2001 From: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:54:08 +0000 Subject: [PATCH 3/5] test: assert normalized downstream bind mounts Signed-off-by: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> --- tests/worker/contract.bats | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/worker/contract.bats b/tests/worker/contract.bats index e39a95c8..1f838190 100644 --- a/tests/worker/contract.bats +++ b/tests/worker/contract.bats @@ -80,7 +80,8 @@ setup() { fixture="$REPO_ROOT/tests/worker/fixtures/sample-app" run env APP_ID=sample_app APP_SOURCE_DIR="$fixture" DB_TYPE=sqlite sh "$WORKER" consumer-a config [ "$status" -eq 0 ] - [[ "$output" == *"$fixture:/var/www/html/apps-extra/sample_app"* ]] + [[ "$output" == *"source: $fixture"* ]] + [[ "$output" == *"target: /var/www/html/apps-extra/sample_app"* ]] [[ "$output" == *"name: ncdev-consumer-a"* ]] } @@ -91,7 +92,8 @@ setup() { run env DB_TYPE=sqlite sh "$WORKER" consumer-memory config [ "$status" -eq 0 ] - [[ "$output" == *"$fixture:/var/www/html/apps-extra/sample_app"* ]] + [[ "$output" == *"source: $fixture"* ]] + [[ "$output" == *"target: /var/www/html/apps-extra/sample_app"* ]] rm -rf "$REPO_ROOT/.workers/consumer-memory" } From aad1c067526b7b35ec4abb8aaaf832a23be9cf41 Mon Sep 17 00:00:00 2001 From: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:55:03 +0000 Subject: [PATCH 4/5] fix: keep downstream teardown independent of checkout Signed-off-by: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> --- dev-worker | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dev-worker b/dev-worker index 71aca4c9..7c6c8f76 100644 --- a/dev-worker +++ b/dev-worker @@ -98,11 +98,12 @@ if [ -n "$app_id" ] || [ -n "$app_source" ]; then exit 2 ;; esac - [ -d "$app_source" ] || { + if [ -d "$app_source" ]; then + app_source="$(CDPATH= cd -- "$app_source" && pwd -P)" + elif [ "$command" != "destroy" ]; then echo "APP_SOURCE_DIR is not a directory: $app_source" >&2 exit 2 - } - app_source="$(CDPATH= cd -- "$app_source" && pwd -P)" + fi if [ -n "$stored_app_id" ] && [ "$stored_app_id" != "$app_id" ]; then echo "Worker $worker_id is already bound to APP_ID=$stored_app_id" >&2 exit 2 @@ -252,7 +253,9 @@ case "$command" in # worker root through the existing Nextcloud image as root instead of # relying on host ownership. if [ -d "$worker_root" ]; then - compose run --rm --no-deps -u 0 --entrypoint sh -v "$worker_root:/worker" nextcloud -c 'find /worker -mindepth 1 -exec rm -rf -- {} +' >/dev/null 2>&1 || true + docker compose --project-directory "$repo_root" --file "$repo_root/docker-compose.yml" \ + run --rm --no-deps -u 0 --entrypoint sh -v "$worker_root:/worker" nextcloud \ + -c 'find /worker -mindepth 1 -exec rm -rf -- {} +' >/dev/null 2>&1 || true fi case "$worker_root" in "$repo_root"/.workers/"$worker_id"|"$repo_root"/.workers/"$worker_id"/) ;; From 546853e819da58e1b9043171d161cbb21b17a7c4 Mon Sep 17 00:00:00 2001 From: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:57:39 +0000 Subject: [PATCH 5/5] refactor: make worker app-agnostic Signed-off-by: Vitor Mattos <1079143+vitormattos@users.noreply.github.com> --- .docker/bin/build-nextcloud-image | 38 +++++ .docker/downstream-app.yml | 13 -- .github/workflows/proxy-tests.yml | 153 +---------------- .github/workflows/worker-tests.yml | 175 ++++++++++++++++++++ README.md | 4 +- dev-worker | 81 +++------ docker-compose.yml | 3 - docs/apps-development.md | 65 ++++++-- docs/compose-extensions.md | 103 ++++++++++++ docs/downstream-consumers.md | 108 ------------ tests/worker/contract.bats | 54 +++--- tests/worker/fixtures/compose-extension.yml | 30 ++++ 12 files changed, 447 insertions(+), 380 deletions(-) create mode 100644 .docker/bin/build-nextcloud-image delete mode 100644 .docker/downstream-app.yml create mode 100644 .github/workflows/worker-tests.yml create mode 100644 docs/compose-extensions.md delete mode 100644 docs/downstream-consumers.md create mode 100644 tests/worker/fixtures/compose-extension.yml diff --git a/.docker/bin/build-nextcloud-image b/.docker/bin/build-nextcloud-image new file mode 100644 index 00000000..351e652b --- /dev/null +++ b/.docker/bin/build-nextcloud-image @@ -0,0 +1,38 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -eu + +repo_root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +compose_file="$repo_root/docker-compose.yml" + +image="$( + docker compose --project-directory "$repo_root" --file "$compose_file" config --images | + grep '^ghcr.io/librecodecoop/nextcloud-dev-php' | + head -n 1 +)" + +[ -n "$image" ] || { + echo "Could not resolve the Nextcloud development image from docker-compose.yml" >&2 + exit 2 +} + +case "$image" in + ghcr.io/librecodecoop/nextcloud-dev-php*:latest) ;; + *) + echo "Unexpected Nextcloud development image: $image" >&2 + exit 2 + ;; +esac + +php_version="${image#*nextcloud-dev-php}" +php_version="${php_version%%:*}" +dockerfile="$repo_root/.docker/Dockerfile.php$php_version" + +[ -f "$dockerfile" ] || { + echo "No Dockerfile for PHP $php_version: $dockerfile" >&2 + exit 2 +} + +docker build -f "$dockerfile" -t "$image" "$repo_root/.docker" diff --git a/.docker/downstream-app.yml b/.docker/downstream-app.yml deleted file mode 100644 index c343423e..00000000 --- a/.docker/downstream-app.yml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors -# SPDX-License-Identifier: AGPL-3.0-or-later - -services: - nextcloud: - volumes: - - ${APP_SOURCE_DIR:?APP_SOURCE_DIR is required}:${APP_TARGET_DIR:?APP_TARGET_DIR is required} - nginx: - volumes: - - ${APP_SOURCE_DIR:?APP_SOURCE_DIR is required}:${APP_TARGET_DIR:?APP_TARGET_DIR is required}:ro - playwright: - volumes: - - ${APP_SOURCE_DIR:?APP_SOURCE_DIR is required}:${APP_TARGET_DIR:?APP_TARGET_DIR is required} diff --git a/.github/workflows/proxy-tests.yml b/.github/workflows/proxy-tests.yml index 7b76eb79..4392e6e9 100644 --- a/.github/workflows/proxy-tests.yml +++ b/.github/workflows/proxy-tests.yml @@ -23,7 +23,7 @@ jobs: detik-install: false file-install: false - name: Run unit tests - run: bats tests/proxy/lease.bats tests/proxy/infrastructure.bats tests/proxy/services.bats tests/proxy/diagnostics.bats tests/proxy/compose-policy.bats tests/worker/contract.bats + run: bats tests/proxy/lease.bats tests/proxy/infrastructure.bats tests/proxy/services.bats tests/proxy/diagnostics.bats tests/proxy/compose-policy.bats integration: name: Proxy Docker integration tests @@ -44,154 +44,3 @@ jobs: REPO_ROOT="$GITHUB_WORKSPACE" COMPOSE_PROJECT_NAME=proxytesta docker compose --file tests/proxy/fixtures/compose.yml config --quiet - name: Run Docker integration tests run: bats tests/proxy/integration.bats - - sqlite-worker: - name: SQLite worker integration - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Build PHP 8.3 development image - run: docker build -f .docker/Dockerfile.php83 -t ghcr.io/librecodecoop/nextcloud-dev-php83:latest .docker - - name: Start isolated SQLite workers - run: | - DB_TYPE=sqlite sh ./dev-worker sqlite-a up - DB_TYPE=sqlite sh ./dev-worker sqlite-b up - - name: Verify SQLite runtime - run: | - DB_TYPE=sqlite sh ./dev-worker sqlite-a exec php -r 'exit(extension_loaded("pdo_sqlite") ? 0 : 1);' - DB_TYPE=sqlite sh ./dev-worker sqlite-a exec occ status | grep -q 'installed: true' - DB_TYPE=sqlite sh ./dev-worker sqlite-b exec occ status | grep -q 'installed: true' - - name: Verify SQLite worker state isolation - run: | - DB_TYPE=sqlite sh ./dev-worker sqlite-a exec sh -c 'printf alpha > /var/www/html/data/worker-probe' - DB_TYPE=sqlite sh ./dev-worker sqlite-b exec sh -c 'printf beta > /var/www/html/data/worker-probe' - alpha="$(DB_TYPE=sqlite sh ./dev-worker sqlite-a exec cat /var/www/html/data/worker-probe)" - beta="$(DB_TYPE=sqlite sh ./dev-worker sqlite-b exec cat /var/www/html/data/worker-probe)" - printf 'sqlite-a worker_probe=%s\nsqlite-b worker_probe=%s\n' "$alpha" "$beta" - test "$alpha" = alpha - test "$beta" = beta - - name: Verify SQLite starts no database service - run: | - database_containers="$(docker ps --format '{{.Names}}' | grep -E '^ncdev-sqlite-(a|b)-database-1$' || true)" - if [ -n "$database_containers" ]; then - printf 'Unexpected database containers:\n%s\n' "$database_containers" >&2 - exit 1 - fi - - name: Collect worker logs - if: failure() - run: | - DB_TYPE=sqlite sh ./dev-worker sqlite-a logs || true - DB_TYPE=sqlite sh ./dev-worker sqlite-b logs || true - - name: Destroy workers - if: always() - run: | - DB_TYPE=sqlite sh ./dev-worker sqlite-a destroy || true - DB_TYPE=sqlite sh ./dev-worker sqlite-b destroy || true - - mariadb-worker: - name: MariaDB ${{ matrix.mariadb }} worker integration - runs-on: ubuntu-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - mariadb: ['10.6', '10.11'] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Build PHP 8.3 development image - run: docker build -f .docker/Dockerfile.php83 -t ghcr.io/librecodecoop/nextcloud-dev-php83:latest .docker - - name: Start MariaDB worker - env: - MARIADB_VERSION: ${{ matrix.mariadb }} - DB_SQL_MODE: ONLY_FULL_GROUP_BY - run: | - worker="maria-${MARIADB_VERSION//./-}" - DB_TYPE=mariadb sh ./dev-worker "$worker" up - - name: Verify MariaDB version and SQL mode - env: - MARIADB_VERSION: ${{ matrix.mariadb }} - DB_SQL_MODE: ONLY_FULL_GROUP_BY - run: | - worker="maria-${MARIADB_VERSION//./-}" - version="$(DB_TYPE=mariadb sh ./dev-worker "$worker" db-exec mariadb -uroot -proot -Nse 'SELECT VERSION()')" - case "$version" in - "$MARIADB_VERSION".*) ;; - *) echo "Unexpected MariaDB version: $version" >&2; exit 1 ;; - esac - DB_TYPE=mariadb sh ./dev-worker "$worker" db-exec mariadb -uroot -proot -Nse 'SELECT @@GLOBAL.sql_mode' | grep -q 'ONLY_FULL_GROUP_BY' - DB_TYPE=mariadb sh ./dev-worker "$worker" status | grep -q 'installed: true' - - name: Verify two-worker isolation - if: matrix.mariadb == '10.11' - env: - MARIADB_VERSION: ${{ matrix.mariadb }} - DB_SQL_MODE: ONLY_FULL_GROUP_BY - run: | - DB_TYPE=mariadb sh ./dev-worker maria-peer up - DB_TYPE=mariadb sh ./dev-worker maria-10-11 exec sh -c 'printf alpha > /var/www/html/data/worker-probe' - DB_TYPE=mariadb sh ./dev-worker maria-peer exec sh -c 'printf beta > /var/www/html/data/worker-probe' - alpha="$(DB_TYPE=mariadb sh ./dev-worker maria-10-11 exec cat /var/www/html/data/worker-probe)" - beta="$(DB_TYPE=mariadb sh ./dev-worker maria-peer exec cat /var/www/html/data/worker-probe)" - printf 'maria-10-11 worker_probe=%s\nmaria-peer worker_probe=%s\n' "$alpha" "$beta" - test "$alpha" = alpha - test "$beta" = beta - - name: Collect worker logs - if: failure() - env: - MARIADB_VERSION: ${{ matrix.mariadb }} - run: | - worker="maria-${MARIADB_VERSION//./-}" - DB_TYPE=mariadb sh ./dev-worker "$worker" logs || true - if [ "$MARIADB_VERSION" = 10.11 ]; then - DB_TYPE=mariadb sh ./dev-worker maria-peer logs || true - fi - - name: Destroy workers - if: always() - env: - MARIADB_VERSION: ${{ matrix.mariadb }} - run: | - worker="maria-${MARIADB_VERSION//./-}" - DB_TYPE=mariadb sh ./dev-worker "$worker" destroy || true - if [ "$MARIADB_VERSION" = 10.11 ]; then - DB_TYPE=mariadb sh ./dev-worker maria-peer destroy || true - fi - - downstream-consumer: - name: Downstream consumer integration - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Build PHP 8.3 development image - run: docker build -f .docker/Dockerfile.php83 -t ghcr.io/librecodecoop/nextcloud-dev-php83:latest .docker - - name: Prepare independent downstream checkouts - run: | - cp -a tests/worker/fixtures/sample-app /tmp/consumer-a - cp -a tests/worker/fixtures/sample-app /tmp/consumer-b - printf 'alpha\n' > /tmp/consumer-a/marker.txt - printf 'beta\n' > /tmp/consumer-b/marker.txt - - name: Start downstream workers and run setup hooks - run: | - APP_ID=sample_app APP_SOURCE_DIR=/tmp/consumer-a APP_SETUP_COMMAND='printf setup-a > /var/www/html/data/downstream-setup' DB_TYPE=sqlite sh ./dev-worker consumer-a up - APP_ID=sample_app APP_SOURCE_DIR=/tmp/consumer-b APP_SETUP_COMMAND='printf setup-b > /var/www/html/data/downstream-setup' DB_TYPE=sqlite sh ./dev-worker consumer-b up - - name: Verify checkout, hook, URLs, and isolation - run: | - test "$(DB_TYPE=sqlite sh ./dev-worker consumer-a exec cat /var/www/html/apps-extra/sample_app/marker.txt)" = alpha - test "$(DB_TYPE=sqlite sh ./dev-worker consumer-b exec cat /var/www/html/apps-extra/sample_app/marker.txt)" = beta - test "$(DB_TYPE=sqlite sh ./dev-worker consumer-a exec cat /var/www/html/data/downstream-setup)" = setup-a - test "$(DB_TYPE=sqlite sh ./dev-worker consumer-b exec cat /var/www/html/data/downstream-setup)" = setup-b - DB_TYPE=sqlite sh ./dev-worker consumer-a urls | grep -q 'https://ncdev-consumer-a.localhost' - DB_TYPE=sqlite sh ./dev-worker consumer-b urls | grep -q 'https://ncdev-consumer-b-mailpit.localhost' - - name: Verify teardown affects only the selected worker - run: | - DB_TYPE=sqlite sh ./dev-worker consumer-a destroy - DB_TYPE=sqlite sh ./dev-worker consumer-b status | grep -q 'installed: true' - test "$(DB_TYPE=sqlite sh ./dev-worker consumer-b exec cat /var/www/html/apps-extra/sample_app/marker.txt)" = beta - - name: Collect worker logs - if: failure() - run: | - DB_TYPE=sqlite sh ./dev-worker consumer-a logs || true - DB_TYPE=sqlite sh ./dev-worker consumer-b logs || true - - name: Destroy remaining worker - if: always() - run: DB_TYPE=sqlite sh ./dev-worker consumer-b destroy || true diff --git a/.github/workflows/worker-tests.yml b/.github/workflows/worker-tests.yml new file mode 100644 index 00000000..ac6b3cd8 --- /dev/null +++ b/.github/workflows/worker-tests.yml @@ -0,0 +1,175 @@ +name: Worker tests + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + unit: + name: Worker contract tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Setup Bats-core + uses: bats-core/bats-action@77d6fb60505b4d0d1d73e48bd035b55074bbfb43 # 4.0.0 + with: + support-install: false + assert-install: false + detik-install: false + file-install: false + - name: Run worker contract tests + run: bats tests/worker/contract.bats + + sqlite: + name: SQLite worker integration + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Build default PHP development image + run: sh .docker/bin/build-nextcloud-image + - name: Start isolated SQLite workers + run: | + DB_TYPE=sqlite sh ./dev-worker sqlite-a up + DB_TYPE=sqlite sh ./dev-worker sqlite-b up + - name: Verify SQLite runtime + run: | + DB_TYPE=sqlite sh ./dev-worker sqlite-a exec php -r 'exit(extension_loaded("pdo_sqlite") ? 0 : 1);' + DB_TYPE=sqlite sh ./dev-worker sqlite-a exec occ status | grep -q 'installed: true' + DB_TYPE=sqlite sh ./dev-worker sqlite-b exec occ status | grep -q 'installed: true' + - name: Verify SQLite worker state isolation + run: | + DB_TYPE=sqlite sh ./dev-worker sqlite-a exec sh -c 'printf alpha > /var/www/html/data/worker-probe' + DB_TYPE=sqlite sh ./dev-worker sqlite-b exec sh -c 'printf beta > /var/www/html/data/worker-probe' + alpha="$(DB_TYPE=sqlite sh ./dev-worker sqlite-a exec cat /var/www/html/data/worker-probe)" + beta="$(DB_TYPE=sqlite sh ./dev-worker sqlite-b exec cat /var/www/html/data/worker-probe)" + printf 'sqlite-a worker_probe=%s\nsqlite-b worker_probe=%s\n' "$alpha" "$beta" + test "$alpha" = alpha + test "$beta" = beta + - name: Verify SQLite starts no database service + run: | + database_containers="$(docker ps --format '{{.Names}}' | grep -E '^ncdev-sqlite-(a|b)-database-1$' || true)" + if [ -n "$database_containers" ]; then + printf 'Unexpected database containers:\n%s\n' "$database_containers" >&2 + exit 1 + fi + - name: Collect worker logs + if: failure() + run: | + DB_TYPE=sqlite sh ./dev-worker sqlite-a logs || true + DB_TYPE=sqlite sh ./dev-worker sqlite-b logs || true + - name: Destroy workers + if: always() + run: | + DB_TYPE=sqlite sh ./dev-worker sqlite-a destroy || true + DB_TYPE=sqlite sh ./dev-worker sqlite-b destroy || true + + mariadb: + name: MariaDB ${{ matrix.mariadb }} worker integration + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mariadb: ['10.6', '10.11'] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Build default PHP development image + run: sh .docker/bin/build-nextcloud-image + - name: Start MariaDB worker + env: + MARIADB_VERSION: ${{ matrix.mariadb }} + DB_SQL_MODE: ONLY_FULL_GROUP_BY + run: | + worker="maria-${MARIADB_VERSION//./-}" + DB_TYPE=mariadb sh ./dev-worker "$worker" up + - name: Verify MariaDB version and SQL mode + env: + MARIADB_VERSION: ${{ matrix.mariadb }} + DB_SQL_MODE: ONLY_FULL_GROUP_BY + run: | + worker="maria-${MARIADB_VERSION//./-}" + version="$(DB_TYPE=mariadb sh ./dev-worker "$worker" db-exec mariadb -uroot -proot -Nse 'SELECT VERSION()')" + case "$version" in + "$MARIADB_VERSION".*) ;; + *) echo "Unexpected MariaDB version: $version" >&2; exit 1 ;; + esac + DB_TYPE=mariadb sh ./dev-worker "$worker" db-exec mariadb -uroot -proot -Nse 'SELECT @@GLOBAL.sql_mode' | grep -q 'ONLY_FULL_GROUP_BY' + DB_TYPE=mariadb sh ./dev-worker "$worker" status | grep -q 'installed: true' + - name: Verify two-worker isolation + if: matrix.mariadb == '10.11' + env: + MARIADB_VERSION: ${{ matrix.mariadb }} + DB_SQL_MODE: ONLY_FULL_GROUP_BY + run: | + DB_TYPE=mariadb sh ./dev-worker maria-peer up + DB_TYPE=mariadb sh ./dev-worker maria-10-11 exec sh -c 'printf alpha > /var/www/html/data/worker-probe' + DB_TYPE=mariadb sh ./dev-worker maria-peer exec sh -c 'printf beta > /var/www/html/data/worker-probe' + alpha="$(DB_TYPE=mariadb sh ./dev-worker maria-10-11 exec cat /var/www/html/data/worker-probe)" + beta="$(DB_TYPE=mariadb sh ./dev-worker maria-peer exec cat /var/www/html/data/worker-probe)" + printf 'maria-10-11 worker_probe=%s\nmaria-peer worker_probe=%s\n' "$alpha" "$beta" + test "$alpha" = alpha + test "$beta" = beta + - name: Collect worker logs + if: failure() + env: + MARIADB_VERSION: ${{ matrix.mariadb }} + run: | + worker="maria-${MARIADB_VERSION//./-}" + DB_TYPE=mariadb sh ./dev-worker "$worker" logs || true + if [ "$MARIADB_VERSION" = 10.11 ]; then + DB_TYPE=mariadb sh ./dev-worker maria-peer logs || true + fi + - name: Destroy workers + if: always() + env: + MARIADB_VERSION: ${{ matrix.mariadb }} + run: | + worker="maria-${MARIADB_VERSION//./-}" + DB_TYPE=mariadb sh ./dev-worker "$worker" destroy || true + if [ "$MARIADB_VERSION" = 10.11 ]; then + DB_TYPE=mariadb sh ./dev-worker maria-peer destroy || true + fi + + compose-extension: + name: Compose extension integration + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Build default PHP development image + run: sh .docker/bin/build-nextcloud-image + - name: Start a worker with two externally mounted apps + env: + NCDD_COMPOSE_OVERRIDE: ${{ github.workspace }}/tests/worker/fixtures/compose-extension.yml + TEST_APP_A: ${{ github.workspace }}/tests/worker/fixtures/sample-app + TEST_APP_B: ${{ github.workspace }}/tests/worker/fixtures/other-app + run: DB_TYPE=sqlite sh ./dev-worker extension up + - name: Verify both apps use the same worker runtime + env: + NCDD_COMPOSE_OVERRIDE: ${{ github.workspace }}/tests/worker/fixtures/compose-extension.yml + TEST_APP_A: ${{ github.workspace }}/tests/worker/fixtures/sample-app + TEST_APP_B: ${{ github.workspace }}/tests/worker/fixtures/other-app + run: | + test "$(DB_TYPE=sqlite sh ./dev-worker extension exec cat /var/www/html/apps-extra/sample_app/marker.txt)" = alpha + test "$(DB_TYPE=sqlite sh ./dev-worker extension exec cat /var/www/html/apps-extra/other_app/marker.txt)" = other + DB_TYPE=sqlite sh ./dev-worker extension status | grep -q 'installed: true' + - name: Collect worker logs + if: failure() + env: + NCDD_COMPOSE_OVERRIDE: ${{ github.workspace }}/tests/worker/fixtures/compose-extension.yml + TEST_APP_A: ${{ github.workspace }}/tests/worker/fixtures/sample-app + TEST_APP_B: ${{ github.workspace }}/tests/worker/fixtures/other-app + run: DB_TYPE=sqlite sh ./dev-worker extension logs || true + - name: Destroy worker + if: always() + env: + NCDD_COMPOSE_OVERRIDE: ${{ github.workspace }}/tests/worker/fixtures/compose-extension.yml + TEST_APP_A: ${{ github.workspace }}/tests/worker/fixtures/sample-app + TEST_APP_B: ${{ github.workspace }}/tests/worker/fixtures/other-app + run: DB_TYPE=sqlite sh ./dev-worker extension destroy || true diff --git a/README.md b/README.md index cde99b79..b9e1e66b 100644 --- a/README.md +++ b/README.md @@ -21,5 +21,5 @@ and other advanced configuration, see the - [Advanced setup](docs/advanced-setup.md) - [App development](docs/apps-development.md) -- [Downstream app and devcontainer contract](docs/downstream-consumers.md) -- [FAQ](docs/faq.md) \ No newline at end of file +- [Compose extensions and devcontainers](docs/compose-extensions.md) +- [FAQ](docs/faq.md) diff --git a/dev-worker b/dev-worker index 7c6c8f76..5a3e041c 100644 --- a/dev-worker +++ b/dev-worker @@ -25,9 +25,7 @@ Environment: DB_SQL_MODE= Optional global SQL mode for MySQL-compatible backends PHP_VERSION= Existing PHP image selector VERSION_NEXTCLOUD= Existing Nextcloud ref selector - APP_ID= Optional downstream Nextcloud app identifier - APP_SOURCE_DIR= Optional downstream app checkout to mount - APP_SETUP_COMMAND= Optional idempotent command run after readiness + NCDD_COMPOSE_OVERRIDE= Optional Compose override appended to the base topology WORKER_READY_TIMEOUT=180 Seconds to wait for a clean installation EOF } @@ -58,6 +56,26 @@ esac repo_root="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" db_type="${DB_TYPE:-mysql}" +compose_override="${NCDD_COMPOSE_OVERRIDE:-}" + +if [ -n "$compose_override" ]; then + case "$compose_override" in + /*) ;; + *) + override_dir="$(dirname -- "$compose_override")" + override_name="$(basename -- "$compose_override")" + [ -d "$override_dir" ] || { + echo "NCDD_COMPOSE_OVERRIDE directory does not exist: $override_dir" >&2 + exit 2 + } + compose_override="$(CDPATH= cd -- "$override_dir" && pwd -P)/$override_name" + ;; + esac + [ -f "$compose_override" ] || { + echo "NCDD_COMPOSE_OVERRIDE is not a file: $compose_override" >&2 + exit 2 + } +fi case "$db_type" in sqlite|mysql|mariadb|pgsql) ;; @@ -69,56 +87,6 @@ esac worker_root="$repo_root/.workers/$worker_id" worker_volumes_dir="$worker_root/volumes" -worker_app_id_file="$worker_root/app-id" -worker_app_source_file="$worker_root/app-source" - -stored_app_id="" -stored_app_source="" -[ ! -f "$worker_app_id_file" ] || stored_app_id="$(cat "$worker_app_id_file")" -[ ! -f "$worker_app_source_file" ] || stored_app_source="$(cat "$worker_app_source_file")" - -app_id="${APP_ID:-$stored_app_id}" -app_source="${APP_SOURCE_DIR:-$stored_app_source}" - -if [ -n "$app_id" ] || [ -n "$app_source" ]; then - [ -n "$app_id" ] && [ -n "$app_source" ] || { - echo "APP_ID and APP_SOURCE_DIR must be provided together" >&2 - exit 2 - } - case "$app_id" in - [a-z]*) ;; - *) - echo "Invalid APP_ID: $app_id" >&2 - exit 2 - ;; - esac - case "$app_id" in - *[!a-z0-9_]*) - echo "Invalid APP_ID: $app_id" >&2 - exit 2 - ;; - esac - if [ -d "$app_source" ]; then - app_source="$(CDPATH= cd -- "$app_source" && pwd -P)" - elif [ "$command" != "destroy" ]; then - echo "APP_SOURCE_DIR is not a directory: $app_source" >&2 - exit 2 - fi - if [ -n "$stored_app_id" ] && [ "$stored_app_id" != "$app_id" ]; then - echo "Worker $worker_id is already bound to APP_ID=$stored_app_id" >&2 - exit 2 - fi - if [ -n "$stored_app_source" ] && [ "$stored_app_source" != "$app_source" ]; then - echo "Worker $worker_id is already bound to APP_SOURCE_DIR=$stored_app_source" >&2 - exit 2 - fi - mkdir -p "$worker_root" - printf '%s\n' "$app_id" > "$worker_app_id_file" - printf '%s\n' "$app_source" > "$worker_app_source_file" - export APP_ID="$app_id" - export APP_SOURCE_DIR="$app_source" - export APP_TARGET_DIR="/var/www/html/apps-extra/$app_id" -fi export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-ncdev-$worker_id}" export WORKER_VOLUMES_DIR="$worker_volumes_dir" @@ -143,10 +111,10 @@ else fi compose() { - if [ -n "${APP_SOURCE_DIR:-}" ]; then + if [ -n "$compose_override" ]; then docker compose --project-directory "$repo_root" \ --file "$repo_root/docker-compose.yml" \ - --file "$repo_root/.docker/downstream-app.yml" "$@" + --file "$compose_override" "$@" else docker compose --project-directory "$repo_root" --file "$repo_root/docker-compose.yml" "$@" fi @@ -211,9 +179,6 @@ case "$command" in compose up -d fi wait_until_ready - if [ -n "${APP_SETUP_COMMAND:-}" ]; then - compose exec -T -u www-data nextcloud sh -lc "$APP_SETUP_COMMAND" - fi ;; exec) [ "$#" -gt 0 ] || { diff --git a/docker-compose.yml b/docker-compose.yml index 977ef331..7a880da9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,6 @@ services: - ${DB_HOST:-${DB_TYPE:-mysql}} nextcloud: image: ghcr.io/librecodecoop/nextcloud-dev-php${PHP_VERSION:-83}:latest - # build: - # context: .docker/ - # dockerfile: Dockerfile.php${PHP_VERSION:-83} volumes: - ./.docker/scripts/wait-for-db.php:/var/www/scripts/wait-for-db.php - ${WORKER_VOLUMES_DIR:-./volumes}/php:/var/www/php-config diff --git a/docs/apps-development.md b/docs/apps-development.md index 48029398..7827125c 100644 --- a/docs/apps-development.md +++ b/docs/apps-development.md @@ -1,28 +1,61 @@ # Start development of apps -For new app-development workflows, keep the app checkout outside NCDD and use -the [downstream app and devcontainer contract](downstream-consumers.md). This -avoids cloning application source into NCDD's mutable Nextcloud data directory -and allows multiple isolated worktrees to reuse the same canonical runtime. +Nextcloud applications can continue to be developed directly under +`volumes/nextcloud/apps-extra`. That directory is an app space and may contain +as many applications as a development setup needs. -The legacy `volumes/nextcloud/apps-extra` workflow remains possible for -existing local setups, but downstream repositories should prefer the worker -contract for automation, concurrent worktrees and devcontainer integration. +For repositories that keep application checkouts outside this repository, use a +[Compose extension](compose-extensions.md) rather than creating another +Nextcloud topology. The extension may mount one app, several apps, or add +supporting services while NCDD remains responsible for the runtime. -## Example +## Local apps-extra workflow + +Clone or create applications under: + +```text +volumes/nextcloud/apps-extra/ +``` + +Then use the `nextcloud` container to install dependencies or run development +commands: ```bash -APP_ID=my_app \ -APP_SOURCE_DIR=/path/to/my_app \ +docker compose exec -u www-data nextcloud bash +cd apps-extra/my_app +composer install +npm ci +``` + +## External application checkouts + +A project-specific Compose override can mount any number of source trees into +the same worker. For example: + +```yaml +services: + nextcloud: + volumes: + - /work/my_app:/var/www/html/apps-extra/my_app + - /work/my_dependency:/var/www/html/apps-extra/my_dependency +``` + +Start the runtime with: + +```bash +NCDD_COMPOSE_OVERRIDE=/work/project/ncdd.override.yml \ DB_TYPE=sqlite \ -sh ./dev-worker my-app up +sh ./dev-worker my-project up +``` -DB_TYPE=sqlite sh ./dev-worker my-app urls -DB_TYPE=sqlite sh ./dev-worker my-app exec sh -lc \ +Application-specific setup remains owned by the application repository. It can +run commands explicitly through the worker, for example: + +```bash +NCDD_COMPOSE_OVERRIDE=/work/project/ncdd.override.yml \ +DB_TYPE=sqlite \ +sh ./dev-worker my-project exec sh -lc \ 'cd /var/www/html/apps-extra/my_app && composer install' ``` -The app checkout stays owned by the downstream repository. NCDD owns only the -isolated runtime state under `.workers/`. - ⬅️ [Back to index](../README.md) diff --git a/docs/compose-extensions.md b/docs/compose-extensions.md new file mode 100644 index 00000000..00e4115b --- /dev/null +++ b/docs/compose-extensions.md @@ -0,0 +1,103 @@ +# Compose extensions and devcontainers + +The base Docker Compose topology in this repository is the canonical Nextcloud +development runtime. Projects may extend that topology without teaching NCDD +which application is being developed. + +This keeps one lifecycle for Nextcloud, PHP, databases, proxy, mail and optional +services while allowing a consumer to mount zero, one or many applications and +add project-specific services. + +## Worker extension contract + +Create a normal Compose override in the consuming project. For example: + +```yaml +services: + nextcloud: + volumes: + - /work/app-a:/var/www/html/apps-extra/app_a + - /work/app-b:/var/www/html/apps-extra/app_b + nginx: + volumes: + - /work/app-a:/var/www/html/apps-extra/app_a:ro + - /work/app-b:/var/www/html/apps-extra/app_b:ro +``` + +Pass that file to the existing worker lifecycle: + +```bash +NCDD_COMPOSE_OVERRIDE=/work/project/ncdd.override.yml \ +DB_TYPE=sqlite \ +sh ./dev-worker project-a up +``` + +The same override must be supplied to worker commands that need the complete +Compose model: + +```bash +NCDD_COMPOSE_OVERRIDE=/work/project/ncdd.override.yml \ +DB_TYPE=sqlite \ +sh ./dev-worker project-a status + +NCDD_COMPOSE_OVERRIDE=/work/project/ncdd.override.yml \ +DB_TYPE=sqlite \ +sh ./dev-worker project-a destroy +``` + +The override is deliberately generic. NCDD does not assign a primary app, +persist an app id, decide where application source lives, or execute +application-specific setup hooks. + +## Runtime dimensions + +Compose extensions reuse the same worker dimensions as the base environment: + +- `PHP_VERSION` +- `VERSION_NEXTCLOUD` +- `DB_TYPE` +- `MARIADB_VERSION` where applicable +- `DB_SQL_MODE` where applicable + +Run: + +```bash +sh ./dev-worker project-a urls +``` + +to print the deterministic Nextcloud and Mailpit hostnames for a worker. + +## Application setup + +Dependency installation, app enablement and other initialization belong to the +consumer repository. Keep those steps in its scripts, Makefile, task runner or +devcontainer lifecycle and invoke the NCDD worker when container access is +needed. + +For example: + +```bash +NCDD_COMPOSE_OVERRIDE=/work/project/ncdd.override.yml \ +sh ./dev-worker project-a exec sh -lc \ + 'cd /var/www/html/apps-extra/app_a && composer install && occ app:enable app_a' +``` + +This avoids growing an NCDD API for application-specific setup. + +## Devcontainers + +A downstream devcontainer should remain a thin adapter over the same Compose +topology. It can use the `nextcloud` service as its runtime service and add a +project-owned override with its source mounts or extra services. + +Do not copy the Nextcloud, database, proxy, mail or Redis definitions into the +application repository. Those remain owned by NCDD. + +## Isolation + +Each worker owns its Compose project name and mutable state under +`.workers/`. A Compose extension changes that worker's service +model but does not create a second lifecycle. + +The worker id is an isolation namespace for development convenience, not a +security boundary between untrusted workloads sharing the same Docker daemon. diff --git a/docs/downstream-consumers.md b/docs/downstream-consumers.md deleted file mode 100644 index a68804b3..00000000 --- a/docs/downstream-consumers.md +++ /dev/null @@ -1,108 +0,0 @@ -# Downstream app and devcontainer contract - -This repository can be the canonical Nextcloud runtime for downstream app -repositories. A consumer keeps its application checkout outside this repository -and mounts it into an isolated worker instead of copying the Compose topology. - -## Worker contract - -Provide an app identifier and checkout path when a worker is created: - -```bash -APP_ID=my_app \ -APP_SOURCE_DIR=/path/to/my_app \ -DB_TYPE=sqlite \ -sh ./dev-worker my-app up -``` - -The worker persists the app binding under its own `.workers/` -metadata. Later commands only need the worker id and the runtime dimensions: - -```bash -DB_TYPE=sqlite sh ./dev-worker my-app status -DB_TYPE=sqlite sh ./dev-worker my-app exec pwd -DB_TYPE=sqlite sh ./dev-worker my-app urls -DB_TYPE=sqlite sh ./dev-worker my-app destroy -``` - -The checkout is mounted at: - -```text -/var/www/html/apps-extra/ -``` - -The same checkout is visible to the Nextcloud, nginx and optional Playwright -services through `.docker/downstream-app.yml`. - -A worker cannot be rebound to a different app id or checkout path. Destroy the -worker first when you intentionally want a new binding. - -## Runtime dimensions - -The downstream contract reuses the same worker options as native NCDD workers: - -- `PHP_VERSION` -- `VERSION_NEXTCLOUD` -- `DB_TYPE` -- `MARIADB_VERSION` where applicable -- `DB_SQL_MODE` where applicable - -No fixed host application or mail port is required. Run: - -```bash -sh ./dev-worker my-app urls -``` - -to get deterministic hostnames based on the worker's Compose project name. - -## Post-readiness setup - -A downstream project can run an idempotent command inside the Nextcloud -container after the worker becomes ready: - -```bash -APP_ID=my_app \ -APP_SOURCE_DIR=/path/to/my_app \ -APP_SETUP_COMMAND='cd /var/www/html/apps-extra/my_app && composer install && occ app:enable my_app' \ -sh ./dev-worker my-app up -``` - -`APP_SETUP_COMMAND` is intentionally executed as a shell command inside the -development container. Treat it as trusted developer input; do not populate it -from untrusted issue, PR or network content. - -## Devcontainer adapters - -The runtime service intended for a downstream devcontainer is `nextcloud`. -The reusable Compose pieces are: - -```text -/path/to/nextcloud-docker-development/docker-compose.yml -/path/to/nextcloud-docker-development/.docker/downstream-app.yml -``` - -A downstream `.devcontainer` should stay thin: it may select the `nextcloud` -service and its workspace folder, while NCDD continues to own the Nextcloud, -database, proxy, mail and supporting service definitions. - -The consumer must supply the same contract variables used by `dev-worker`: -a unique `COMPOSE_PROJECT_NAME`, isolated `WORKER_VOLUMES_DIR`, -`APP_ID`, absolute `APP_SOURCE_DIR`, and -`APP_TARGET_DIR=/var/www/html/apps-extra/`. - -The LibreSign migration is tracked separately; this contract intentionally -contains no LibreSign-specific setup. - -## Isolation guarantees - -Each worker owns: - -- a Compose project name; -- a mutable volume directory; -- persisted downstream app binding metadata; -- database state when a network database is selected; -- deterministic `*.localhost` service hostnames. - -Destroying one worker removes only that worker's Compose resources and mutable -state. The downstream source checkout is a bind mount and is never deleted by -`destroy`. diff --git a/tests/worker/contract.bats b/tests/worker/contract.bats index 1f838190..6a5ee5ce 100644 --- a/tests/worker/contract.bats +++ b/tests/worker/contract.bats @@ -69,44 +69,42 @@ setup() { [[ "$output" == *"Unsupported MARIADB_VERSION"* ]] } -@test "supported PHP images install PDO SQLite" { - for dockerfile in "$REPO_ROOT"/.docker/Dockerfile.php81 "$REPO_ROOT"/.docker/Dockerfile.php82 "$REPO_ROOT"/.docker/Dockerfile.php83; do +@test "all PHP development images install PDO SQLite" { + found=0 + for dockerfile in "$REPO_ROOT"/.docker/Dockerfile.php*; do + [ -f "$dockerfile" ] || continue + found=1 grep -q 'pdo_sqlite' "$dockerfile" done + [ "$found" -eq 1 ] } +@test "worker accepts a generic Compose override with multiple app mounts" { + override="$REPO_ROOT/tests/worker/fixtures/compose-extension.yml" + app_a="$REPO_ROOT/tests/worker/fixtures/sample-app" + app_b="$REPO_ROOT/tests/worker/fixtures/other-app" -@test "downstream app checkout is mounted through the shared worker contract" { - fixture="$REPO_ROOT/tests/worker/fixtures/sample-app" - run env APP_ID=sample_app APP_SOURCE_DIR="$fixture" DB_TYPE=sqlite sh "$WORKER" consumer-a config - [ "$status" -eq 0 ] - [[ "$output" == *"source: $fixture"* ]] - [[ "$output" == *"target: /var/www/html/apps-extra/sample_app"* ]] - [[ "$output" == *"name: ncdev-consumer-a"* ]] -} - -@test "worker remembers downstream app binding by worker id" { - fixture="$REPO_ROOT/tests/worker/fixtures/sample-app" - run env APP_ID=sample_app APP_SOURCE_DIR="$fixture" DB_TYPE=sqlite sh "$WORKER" consumer-memory config - [ "$status" -eq 0 ] + run env \ + NCDD_COMPOSE_OVERRIDE="$override" \ + TEST_APP_A="$app_a" \ + TEST_APP_B="$app_b" \ + DB_TYPE=sqlite \ + sh "$WORKER" compose-extension config - run env DB_TYPE=sqlite sh "$WORKER" consumer-memory config [ "$status" -eq 0 ] - [[ "$output" == *"source: $fixture"* ]] + [[ "$output" == *"source: $app_a"* ]] [[ "$output" == *"target: /var/www/html/apps-extra/sample_app"* ]] - - rm -rf "$REPO_ROOT/.workers/consumer-memory" + [[ "$output" == *"source: $app_b"* ]] + [[ "$output" == *"target: /var/www/html/apps-extra/other_app"* ]] } -@test "worker rejects rebinding an existing consumer to another checkout" { - fixture="$REPO_ROOT/tests/worker/fixtures/sample-app" - other="$REPO_ROOT/tests/worker/fixtures/other-app" - run env APP_ID=sample_app APP_SOURCE_DIR="$fixture" DB_TYPE=sqlite sh "$WORKER" consumer-bound config - [ "$status" -eq 0 ] - - run env APP_ID=sample_app APP_SOURCE_DIR="$other" DB_TYPE=sqlite sh "$WORKER" consumer-bound config +@test "worker rejects a missing Compose override before startup" { + run env NCDD_COMPOSE_OVERRIDE="$REPO_ROOT/does-not-exist.yml" sh "$WORKER" missing-override config [ "$status" -eq 2 ] - [[ "$output" == *"already bound to APP_SOURCE_DIR"* ]] + [[ "$output" == *"NCDD_COMPOSE_OVERRIDE is not a file"* ]] +} - rm -rf "$REPO_ROOT/.workers/consumer-bound" +@test "worker CI does not pin a concrete PHP series" { + workflow="$REPO_ROOT/.github/workflows/worker-tests.yml" + ! grep -Eq 'Dockerfile\.php[0-9]+|nextcloud-dev-php[0-9]+' "$workflow" } diff --git a/tests/worker/fixtures/compose-extension.yml b/tests/worker/fixtures/compose-extension.yml new file mode 100644 index 00000000..d6a818d0 --- /dev/null +++ b/tests/worker/fixtures/compose-extension.yml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +services: + nextcloud: + volumes: + - type: bind + source: ${TEST_APP_A:?TEST_APP_A is required} + target: /var/www/html/apps-extra/sample_app + - type: bind + source: ${TEST_APP_B:?TEST_APP_B is required} + target: /var/www/html/apps-extra/other_app + nginx: + volumes: + - type: bind + source: ${TEST_APP_A:?TEST_APP_A is required} + target: /var/www/html/apps-extra/sample_app + read_only: true + - type: bind + source: ${TEST_APP_B:?TEST_APP_B is required} + target: /var/www/html/apps-extra/other_app + read_only: true + playwright: + volumes: + - type: bind + source: ${TEST_APP_A:?TEST_APP_A is required} + target: /var/www/html/apps-extra/sample_app + - type: bind + source: ${TEST_APP_B:?TEST_APP_B is required} + target: /var/www/html/apps-extra/other_app