From 4907dbecf23a3f3e68579a8138530d81d5a9f62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Fri, 2 Oct 2026 10:54:49 +0200 Subject: [PATCH] feat(cli): rework build / test environment handling Adds an explicit environment (env) subcommand which allows users to list and interact with existing build / test environments. Additionally contains extensive internal refactoring to make the environment tracking and cleanup more robust. --- .github/workflows/ci.yaml | 22 +- Cargo.lock | 529 ++++- Cargo.toml | 6 +- README.md | 4 +- debian/control | 7 + debian/rules.py | 24 +- docs/index.md | 3 +- docs/usage/build.md | 31 +- docs/usage/config.md | 11 +- docs/usage/env.md | 68 + docs/usage/test.md | 30 +- packages/debmagic/CHANGELOG.md | 1 + packages/debmagic/Cargo.toml | 1 + packages/debmagic/README.md | 10 +- packages/debmagic/src/build/attach.rs | 93 - packages/debmagic/src/build/mod.rs | 494 ++--- packages/debmagic/src/build/source.rs | 11 +- packages/debmagic/src/build_intent.rs | 71 +- packages/debmagic/src/cli.rs | 102 +- packages/debmagic/src/config.rs | 54 +- packages/debmagic/src/data_dir.rs | 86 + packages/debmagic/src/driver/config.rs | 44 +- packages/debmagic/src/driver/driver_bare.rs | 32 +- packages/debmagic/src/driver/driver_docker.rs | 68 +- packages/debmagic/src/driver/driver_lxd.rs | 97 +- packages/debmagic/src/driver/mod.rs | 196 +- packages/debmagic/src/env.rs | 177 ++ packages/debmagic/src/environment/mod.rs | 1713 +++++++++++++++++ packages/debmagic/src/environment/process.rs | 81 + packages/debmagic/src/environment/registry.rs | 1114 +++++++++++ packages/debmagic/src/main.rs | 56 +- .../debmagic/src/migrations/0001_registry.sql | 55 + packages/debmagic/src/package.rs | 4 +- packages/debmagic/src/test/intent.rs | 42 +- packages/debmagic/src/test/run.rs | 555 +++--- packages/debmagic/tests/assets/config1.toml | 2 +- 36 files changed, 4865 insertions(+), 1029 deletions(-) create mode 100644 docs/usage/env.md delete mode 100644 packages/debmagic/src/build/attach.rs create mode 100644 packages/debmagic/src/data_dir.rs create mode 100644 packages/debmagic/src/env.rs create mode 100644 packages/debmagic/src/environment/mod.rs create mode 100644 packages/debmagic/src/environment/process.rs create mode 100644 packages/debmagic/src/environment/registry.rs create mode 100644 packages/debmagic/src/migrations/0001_registry.sql diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a5189d0..2c196eb8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -141,6 +141,10 @@ jobs: contents: read # Push of the pre-built CI image is additionally gated to main below. packages: write + env: + # Registry and Environment Host roots together, so the cached build + # tree stays registered under the same Environment id. + DEBMAGIC_DATA_DIR: /tmp/debmagic-data steps: - uses: actions/checkout@v5 - uses: Swatinem/rust-cache@v2 @@ -149,16 +153,18 @@ jobs: - name: Cache the debmagic build tree uses: actions/cache@v4 with: - path: /tmp/debmagic - key: debmagic-build-${{ hashFiles('Cargo.lock', 'debian/changelog', 'debian/control', 'debian/rules') }} - restore-keys: debmagic-build- - - name: Prune stale build trees from the restored cache + path: /tmp/debmagic-data + key: debmagic-data-${{ hashFiles('Cargo.lock', 'debian/changelog', 'debian/control', 'debian/rules') }} + restore-keys: debmagic-data- + - name: Prune Environments of other package versions from the restored cache run: | - mkdir -p /tmp/debmagic current="$(dpkg-parsechangelog -SSource)-$(dpkg-parsechangelog -SVersion)" - find /tmp/debmagic -mindepth 1 -maxdepth 1 -type d \ - ! -name "$current" ! -name "${current}-test" \ - -exec rm -rf {} + + cargo run --locked -q -p debmagic -- env list \ + | awk -v current="$current" 'NR > 1 && $4 != current { print $1 }' \ + | while read -r id; do + cargo run --locked -q -p debmagic -- env clean "$id" \ + || echo "::warning::could not prune Environment $id" + done - name: Resolve CI image tag id: image run: | diff --git a/Cargo.lock b/Cargo.lock index 07ccdad5..b16b7ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -93,6 +99,15 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -183,6 +198,12 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -286,6 +307,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.4" @@ -397,6 +424,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -416,6 +458,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -447,6 +498,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + [[package]] name = "deb822-derive" version = "0.2.0" @@ -482,7 +542,7 @@ dependencies = [ "log", "rowan", "textwrap", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -523,6 +583,7 @@ dependencies = [ "serde_json", "sha1", "sha2 0.11.0", + "sqlx", "test-case", "tokio", "toml", @@ -577,6 +638,7 @@ dependencies = [ "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -620,12 +682,27 @@ dependencies = [ "const-random", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" +dependencies = [ + "serde", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -652,18 +729,55 @@ dependencies = [ "typeid", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + [[package]] name = "find-msvc-tools" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -686,6 +800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -694,6 +809,28 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.34" @@ -702,13 +839,13 @@ checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -725,9 +862,9 @@ checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", @@ -736,7 +873,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -820,7 +956,18 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -838,12 +985,45 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -1239,7 +1419,18 @@ dependencies = [ "bitflags", "libc", "plain", - "redox_syscall", + "redox_syscall 0.9.4", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", ] [[package]] @@ -1248,6 +1439,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -1349,6 +1549,35 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + [[package]] name = "pathdiff" version = "0.2.3" @@ -1410,12 +1639,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.34" @@ -1550,6 +1773,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.9.4" @@ -1807,6 +2039,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -1972,6 +2210,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "smawk" @@ -1989,12 +2230,201 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64 0.22.1", + "bytes", + "cfg-if", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.1", + "hashlink 0.11.1", + "indexmap", + "log", + "memchr", + "percent-encoding", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.111", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" +dependencies = [ + "cfg-if", + "dotenvy", + "either", + "heck", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-sqlite", + "syn 2.0.111", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags", + "byteorder", + "bytes", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "serde", + "sha1", + "sha2 0.11.0", + "sqlx-core", + "thiserror", + "tracing", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "rand", + "serde", + "serde_json", + "sha2 0.11.0", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami 2.1.3", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2101,22 +2531,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -2180,6 +2610,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -2296,10 +2737,23 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -2333,6 +2787,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -2345,6 +2805,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -2399,6 +2874,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2552,6 +3033,12 @@ dependencies = [ "wasite", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" + [[package]] name = "winapi-util" version = "0.1.11" @@ -2731,7 +3218,7 @@ checksum = "2462ea039c445496d8793d052e13787f2b90e750b833afee748e601c17621ed9" dependencies = [ "arraydeque", "encoding_rs", - "hashlink", + "hashlink 0.10.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fcec6a2d..80522b4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["packages/debmagic", "packages/debmagic-common"] [workspace.package] edition = "2024" -rust-version = "1.89" +rust-version = "1.94" [workspace.dependencies] # internal @@ -27,7 +27,7 @@ chrono = { version = ">=0.4.42" } regex = { version = ">=1.12.2" } reqwest = { version = ">=0.13", default-features = false, features = ["rustls", "stream"] } futures-util = "0.3" -tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread"] } +tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread", "sync", "time"] } debian-changelog = { version = ">=0.2.14" } debian-control = { version = ">=0.1.39" } deb822-lossless = { version = ">=0.2" } @@ -35,6 +35,8 @@ ignore = { version = ">=0.4.25" } md-5 = { version = ">=0.11" } sha1 = { version = ">=0.11" } sha2 = { version = ">=0.11" } +# `sqlite` bundles libsqlite3; Debian's sqlx maps it to the system library +sqlx = { version = ">=0.9", default-features = false, features = ["runtime-tokio", "sqlite", "macros", "migrate"] } # dev dependencies test-case = { version = ">=3.3.1" } diff --git a/README.md b/README.md index 2ecba0ae..1c889883 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Modern, robust & easy [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) Debmagic unifies the the packaging experience as a single, streamlined tool. Additionally, `debmagic-pkg` allows you to define package build recipes in Python. - **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) -- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; a failed build prints `debmagic env shell ` - **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host - **Python packaging API**: replace complicated `debian/rules` Makefiles with typed Python `debian/rules.py`, with optional `dh` compatibility @@ -30,7 +30,7 @@ uvx debmagic build binary --driver docker | `debmagic build binary` | Build a binary package (`.deb`) in an isolated environment | | `debmagic build source` | Create a source package (`.dsc`) for upload (incl signing) | | `debmagic test` | Run the package's autopkgtest tests (`debian/tests/`) against a prior build | -| `debmagic shell` | Attach an interactive shell to the build environment | +| `debmagic env` | List, clean, or open a shell in a build or test environment | | `debmagic sign` | GPG-sign a `.changes` file (and its `.dsc`/`.buildinfo`) on the host | | `debmagic config` | Inspect and edit the effective `debmagic.toml` configuration | | `debmagic check` | Lint the package *(in progress)* | diff --git a/debian/control b/debian/control index 4216a09a..ec31dd9e 100644 --- a/debian/control +++ b/debian/control @@ -38,6 +38,13 @@ Build-Depends: librust-md-5-dev (>= 0.11), librust-sha1-dev (>= 0.11), librust-sha2-dev (>= 0.11), + librust-sqlx-dev (>= 0.9), + librust-sqlx+runtime-tokio-dev (>= 0.9), + librust-sqlx+sqlite-dev (>= 0.9), + librust-sqlx+macros-dev (>= 0.9), + librust-sqlx+migrate-dev (>= 0.9), + libsqlite3-dev, + pkg-config, Rules-Requires-Root: no X-Style: black Standards-Version: 4.7.2 diff --git a/debian/rules.py b/debian/rules.py index 9c3bb05d..68f00037 100755 --- a/debian/rules.py +++ b/debian/rules.py @@ -17,13 +17,15 @@ ) # TODO: add this section to our rust debmagic module to make setting this up less annoying -os.environ.update({ - "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", - "CARGO": "/usr/share/cargo/bin/cargo", - "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", - "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", - "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}" -}) +os.environ.update( + { + "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", + "CARGO": "/usr/share/cargo/bin/cargo", + "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", + "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", + "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}", + } +) packages = { "debmagic": ("packages/debmagic", "debmagic"), @@ -33,10 +35,14 @@ cargo_lock = Path("Cargo.lock") cargo_lock_saved = Path("Cargo.lock.saved") + def dh_auto(build: Build, stage: str, use_destdir: bool = False): for pkg_name, (path, python_pkg_name) in packages.items(): destdir = f" --destdir debian/{pkg_name} " if use_destdir else "" - build.cmd(f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}") + build.cmd( + f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}" + ) + @dhp.override def dh_auto_configure(build: Build): @@ -68,4 +74,4 @@ def dh_auto_clean(build: Build): dh_auto(build, "dh_auto_clean") -pkg.pack() \ No newline at end of file +pkg.pack() diff --git a/docs/index.md b/docs/index.md index bff6dd8e..15128be9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ Working with Debian packages without `debmagic` means juggling with several inde Debmagic unifies this into one CLI with useful defaults: - **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) -- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; analyze failures with `debmagic env shell` - **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host - **Package uploading**: `debmagic upload` (to `debian`, `ubuntu`, `ppa:user/repo`) pushes signed builds to archives and PPAs via scp/sftp, with configurable pre-upload checks - **Upstream tracking**: `debmagic upstream list`/`switch` replace uscan/uupdate, reading the package's existing `debian/watch` and `debian/copyright` @@ -37,6 +37,7 @@ Then dive into the pages below — start with [Getting started](usage/getting-st usage/getting-started.md usage/build.md usage/test.md +usage/env.md usage/source.md usage/upload.md usage/upstream.md diff --git a/docs/usage/build.md b/docs/usage/build.md index 412b9ca5..9601d2b1 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -22,8 +22,8 @@ debmagic build binary --driver lxd \ |---|---| | `--driver <...>` | [Build environment driver to use](#picking-a-driver) | | `--source-sync ` | [Which source files to include](#source-file-staging) | -| `--persistent` | Retain the build environment. This does not do incremental builds. | -| `--incremental` | Do incremental builds by syncing changed sources only; implies `persistent` | +| `--persistent ` | How long the build environment outlives this build: `on-failure` (default) keeps it when the build command fails, `always` keeps it after every build, `no` tears it down. A bare `--persistent` means `always` | +| `--incremental` | Do incremental builds by syncing changed sources only; forces `--persistent=always` | | `--distro ` | [Select the target distro/release](#selecting-a-distrorelease) (e.g. `trixie`, `resolute`) | | `--proposed` | Use [build dependencies from `proposed`](#proposed-dependencies) pocket | | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | @@ -34,9 +34,8 @@ debmagic build binary --driver lxd \ | `--apt-update-age ` | [When a persistent environment re-runs `apt-get update`](#apt-update-age) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--output-dir ` | Directory to put the resulting build artifacts | -| `--shell-on-failure` | On build failure, drop into an interactive shell in the build environment when stdout is a TTY | -[`debmagic shell`](#inspecting-a-failed-build) — attach an interactive shell to a build environment +[`debmagic env shell`](#inspecting-a-failed-build) — attach an interactive shell to an Environment ## Picking a driver @@ -52,18 +51,13 @@ There's no auto-detection; pick one and pass it explicitly every time (or config ## Inspecting a failed build -On failure the build environment is torn down by default. Pass `--shell-on-failure` to drop into an interactive shell inside the build environment when stdout is a TTY (destroyed on shell exit unless `--persistent` was used). - -To inspect after the run finishes, pass `--persistent` up front, then: +When the build command itself fails and this run's Persistence is `on-failure` (the default) or `always`, the Environment stays and debmagic prints `debmagic env shell `. Setup failures, and failures after the build command succeeded (export, signing), tear it down. `no` always tears it down and prints no hint. ```shell -# if you're in the package still -debmagic shell -# from the outside: -debmagic shell --source-dir /path/to/parent/of/debian/dir +debmagic env shell ``` -This attaches an interactive shell inside the still-running (or restartable) build environment, at the package's build directory. +Each `debmagic env shell` is its own shell, so you can open more than one. A bare `debmagic env shell` attaches only when this checkout has exactly one Environment. ## Mirror selection @@ -80,7 +74,7 @@ You can persistently set this flag in `.config/debmagic/config.toml`. ## Apt update age Fresh environments always run `apt-get update` once on creation. -When a persistent environment is reused, `--apt-update-age` decides whether the apt index is refreshed again: +When an Environment is reused, `--apt-update-age` decides whether the apt index is refreshed again: | Value | Behavior | |---|---| @@ -113,23 +107,24 @@ You can iterate on the same build for faster compile times. ### Persistent container -Every `debmagic build binary` invocation creates a new container by default and tears it down afterwards. -For repeated attempts against the same package and distro, add `--persistent` to retain and reuse the running environment while restaging the source tree for each build: +Every `debmagic build binary` keeps the Environment when the build command fails (`--persistent=on-failure`, the default) and tears it down when the build succeeds. Pass `--persistent` (that is, `--persistent=always`) to keep it after success too, and reuse it on the next build while restaging the source tree: ```shell debmagic build binary --driver lxd --persistent \ --source-dir . --output-dir /tmp/out ``` +`--persistent=no` discards a kept Environment and starts fresh. A later `on-failure` success also tears down an Environment a previous `always` left behind. + ### Incremental builds Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. -This flag implies `--persistent`, and cannot be combined with `--clean yes`. +This flag forces `--persistent=always`, including over an explicit `no` or `on-failure`, and cannot be combined with `--clean yes`. The preserved build tree is kept even when the environment itself is *not* reused (e.g. a fresh CI runner where the tree was restored from a cache). Quilt patches that a previous build or shell session left applied in the build tree are unapplied with `dpkg-source --after-build` before sources are synced, so the worktree remains the source of truth and stale `.pc` state cannot break later builds. -The work dir (`temp_build_dir`, default `/tmp/debmagic`) keeps the artifacts of builds, including their `.changes` files. +The Environment's Host root lives under `environments_dir` (default `$XDG_DATA_HOME/debmagic/environments`). Build artifacts, including `.changes` files, are exported to `output_dir`. ## Selecting a distro/release @@ -234,4 +229,4 @@ Instead of repeating CLI flags on every invocation, drop a [config file](config. ## Internals -- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic shell` instead of `lxc`/`docker` commands directly. +- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic env shell` instead of `lxc`/`docker` commands directly. diff --git a/docs/usage/config.md b/docs/usage/config.md index 781393e6..68a347b9 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -14,6 +14,8 @@ Config files are merged in order of increasing precedence, so later files overri Only files that exist are read; missing ones are skipped. Command-line flags override whatever the merged config resolves to. +`DEBMAGIC_DATA_DIR` relocates the Environment registry (`db.sqlite`) and the default `environments_dir` together. A relative value is resolved to an absolute path against the current working directory. It is not a config file key. + ## Options All keys are optional. @@ -21,16 +23,16 @@ All keys are optional. | Key | Type | Default | CLI flag | Description | |---|---|---|---|---| | `driver.default` | enum | — | `--driver` | Build driver (`docker`, `bare`, `lxd`, `incus`) | -| `driver.persistent` | bool | `false` | `--persistent` | Keep and reuse the build environment across runs instead of tearing it down. | +| `driver.persistent` | enum | `"on-failure"` | `--persistent` | How long the Environment outlives the claim: `"no"`, `"on-failure"`, or `"always"`. A bare `--persistent` means `always`. | | `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror used for build-dependency resolution. Not used by the `bare` driver. | | `driver.proposed` | bool | `false` | `--proposed` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | | `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update`: `"now"` (every build), `"never"` (only on creation), or a maximum index age like `"1d"`, `"12h"`, `"30m"`. Fresh environments always update once. Not used by the `bare` driver. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | -| `temp_build_dir` | path | `/tmp/debmagic` | — | Where build trees are staged. | +| `environments_dir` | path | `$XDG_DATA_HOME/debmagic/environments` | — | Parent directory of Environment Host roots. Must be absolute or start with `~/`. The Environment registry is always `$XDG_DATA_HOME/debmagic/db.sqlite` (override both with `DEBMAGIC_DATA_DIR`). | | `output_dir` | path | `build` | `-o`/`--output-dir` | Where build artifacts are exported; relative paths resolve against the package root. | -| `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Implies `persistent`; incompatible with `clean`. | +| `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Forces Persistence `always`; incompatible with `clean`. | | `source_sync_mode` | enum | `tracked` | `--source-sync` | Which source files are staged (see below). | | `build_debug_symbols` | bool | `false` | `--debug-symbols` | Build the automatic `-dbgsym` debug symbol package. | | `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` so tests are skipped. | @@ -40,7 +42,6 @@ All keys are optional. | `sign.sign_command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | | `sign.notify` | bool | `false` | `--sign-notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | -| `shell_on_failure` | bool | `false` | `--shell-on-failure` | On build or test failure, drop into an interactive shell in the environment when stdout is a TTY. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | | `upstream.verify_signatures` | bool | `true` | `--no-signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream switch` (see [Upstream](upstream.md#signature-verification)). | | `upload.targets` | map | — | `debmagic upload ` | Named upload targets for [uploading](upload.md), merged field-by-field over the builtins (`ppa`, `ubuntu`, `debian`). | @@ -106,7 +107,7 @@ notify = true [driver] default = "lxd" -persistent = true +persistent = "always" apt_mirror = "http:///ubuntu" [driver.docker] diff --git a/docs/usage/env.md b/docs/usage/env.md new file mode 100644 index 00000000..aeb88c61 --- /dev/null +++ b/docs/usage/env.md @@ -0,0 +1,68 @@ +# Environments + +Every build and test runs in an Environment: a container, VM or chroot that a Driver creates for one package version and distro. +debmagic remembers all Environments on this machine, so you can inspect, reuse and clean them up later. + +```shell +debmagic env list # what exists, and in what state +debmagic env shell [] # open a shell in an Environment +debmagic env clean # destroy everything Stale +debmagic env rm ... # destroy specific Environments +``` + +`env` is short for `debmagic environment`. + +## list + +Prints every Environment, after asking each Driver whether it still exists: + +| Column | Meaning | +|---|---| +| ID | Environment id, used by `shell` and `rm` | +| PURPOSE | `build` or `test` | +| DRIVER | Isolation backend | +| PACKAGE | Package name and version | +| DISTRO | Codename | +| PERSIST | Persistence of the run that created it: `no`, `on-failure`, or `always` | +| STATUS | `live`, `stale`, or `unreachable` | +| ATTACH | Number of open `debmagic env shell` sessions | +| SOURCE | Source tree path | + +An Environment is **Stale** when its Driver resource or Host root is gone, or when it has Persistence `no` and neither its creating command nor a shell is still running. +`on-failure` and `always` Environments stay `live` after their command exits. +**Unreachable** means the Driver could not be queried, so debmagic cannot tell whether the Environment still exists. + +## shell + +```shell +debmagic env shell # inside a Source tree: its only Environment +debmagic env shell # anywhere +``` + +Without an id, debmagic picks the Environment of the current Source tree. +Environments of the version in `debian/changelog` take precedence over those of earlier versions. +If there are none or several (for example a build and a test Environment), it lists them and asks for an id. + +Each call opens a new shell, so you can open several. +While a shell is open, the Environment is kept, even with Persistence `no`, and a build or test that would reuse it refuses to start instead of resetting it under you. + +## clean and rm + +Each package version gets its own Environment, so kept ones accumulate. + +`debmagic env clean` destroys all **Stale** Environments on the machine and leaves healthy ones alone. +Unreachable Environments are skipped and reported. +An Environment that a concurrent run has just recreated is skipped too. + +`debmagic env rm ...` (alias `remove`) destroys the given Environments, even healthy ones. +It refuses while a shell or the creating command is still running, and refuses Unreachable Environments. +The remaining ids are still processed when one fails. + +If the Driver fails to destroy its resource (for example `docker rm` fails), the Environment stays listed so the resource is not lost; retry once the Driver works. +`--force` removes the Environment anyway, also when it is Unreachable, and only warns if the Driver resource may have leaked. + +## Data directory + +The registry and the Host roots live under `$XDG_DATA_HOME/debmagic` (see [`environments_dir`](config.md)). +Keep this on a local filesystem. +With a network-mounted home directory, set `DEBMAGIC_DATA_DIR` to a local path. diff --git a/docs/usage/test.md b/docs/usage/test.md index 96393f17..268fd5dd 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -5,7 +5,7 @@ Quick reference for running a package's declared Debian autopkgtest tests with ` ## TL;DR - Entry point: `debmagic test` — runs tests from `debian/tests/control`; needs a prior `debmagic build` -- Requires a completed build in the same build root (or pass `--changes` to point at exported artifacts) +- Requires a completed binary build (looked up from the Environment registry's Invocation history, or pass `--changes`) ```shell cd your-package @@ -17,7 +17,7 @@ debmagic test --driver docker `debmagic test` installs the binary packages from a prior build and runs the package's declared autopkgtest tests (`debian/tests/control`) inside a **fresh, separate** driver-managed environment. -The test environment is never the build environment — even when `--persistent` reuses a container across runs, +The test environment is never the build environment — even when Persistence `on-failure` or `always` reuses a container across runs, the test tree is reset and the `.debs` are reinstalled each time. The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside the container (or on the host for the bare driver). No `autopkgtest-virt-*` backends are used. @@ -26,23 +26,22 @@ The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside t | Option | Description | |---|---| -| `--driver <...>` | Test environment driver (defaults to the driver recorded in the prior build's `environment.json`) | -| `--persistent` | Retain the test environment after the run for reattach/debug | +| `--driver <...>` | Test environment driver (defaults to the driver recorded on the prior binary-build Invocation) | +| `--persistent ` | How long the test environment outlives this run: `on-failure` (default) keeps it when the tests fail, `always` keeps it after every run, `no` tears it down. A bare `--persistent` means `always` | | `--strict` | Treat skipped tests and "no tests declared" as failures (exit code 2) | -| `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use) | -| `--distro ` | Override the target distro for the test environment (defaults to the prior build's distro from `environment.json`, not the changelog) | +| `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use). Overrides the artifact path only; driver and distro still default to the prior binary-build Invocation | +| `--distro ` | Override the target distro for the test environment (defaults to the prior binary-build Invocation's distro, not the changelog) | | `--proposed` | Enable the `-proposed` pocket in the test environment | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | | `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--allow-host-test` | Allow the bare driver, which runs autopkgtest as root on the host | -| `--shell-on-failure` | On test failure, drop into an interactive shell in the test environment when stdout is a TTY | Driver-specific flags (`--driver-docker-base-image`, `--driver-lxd-*`) mirror `debmagic build`. ## Picking a driver -Use the same drivers as for builds. Pass `--driver` explicitly (or rely on the driver recorded in the prior build's `environment.json`): +Use the same drivers as for builds. Pass `--driver` explicitly (or rely on the driver recorded on the prior binary-build Invocation): | Driver | Isolation the Environment provides | |---|---| @@ -66,16 +65,21 @@ If no `debian/tests/control` exists (or it declares no tests), the run exits 0 w ## Inspecting a failed test run -On failure the test environment is torn down by default. Pass `--shell-on-failure` to drop into an interactive shell inside the test environment when stdout is a TTY (destroyed on shell exit unless `--persistent` was used). +When the tests themselves fail and this run's Persistence is `on-failure` (the default) or `always`, the Environment stays and debmagic prints `debmagic env shell `. A strict skip tears the Environment down. `no` always tears it down and prints no hint. -Test output and logs are exported to a `test/` subdirectory of the build root; the path is printed at the end of the run. +Test output and logs are exported to a `.test/` directory next to the `.changes` file (for `pkg_1.0-1_amd64.changes`, that is `pkg_1.0-1_amd64.test/`); the path is printed at the end of the run. A later run replaces that directory only if debmagic wrote it; otherwise the run fails rather than deleting it. + +```shell +debmagic env shell +``` ## Prior build required -By default `debmagic test` resolves the prior build from the build root (same layout as `debmagic shell`). If no build artifacts are found: -run `debmagic build` first +By default `debmagic test` looks up the latest successful binary-build Invocation for this Source tree whose exported `.changes` still exists. If more than one DistroVersion matches, it errors and lists them — pass `--distro`. If none are found: + +run `debmagic build binary` first -Use `--changes` to supply a `.changes` file from an exported output directory instead. +Use `--changes` to supply a `.changes` file from an exported output directory. That overrides the artifact path only. Driver and distro still come from the prior binary-build Invocation when one exists; pass `--driver` and `--distro` when it does not. Several DistroVersions still require `--distro`. ## Bare driver diff --git a/packages/debmagic/CHANGELOG.md b/packages/debmagic/CHANGELOG.md index 059f7682..038e312f 100644 --- a/packages/debmagic/CHANGELOG.md +++ b/packages/debmagic/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `debmagic switch ` upstream source fetching and orig tarballs handling with `debmagic upstream` (and caching them) - dynamic interval for running `apt update` configured by `--apt-update-age` / `driver.apt_update_age` +- `debmagic env list`, `debmagic env clean`, and `debmagic env shell` for a machine-wide view of environments ## [0.0.1-alpha.8] - 2026-09-20 diff --git a/packages/debmagic/Cargo.toml b/packages/debmagic/Cargo.toml index f7569c23..439f4e97 100644 --- a/packages/debmagic/Cargo.toml +++ b/packages/debmagic/Cargo.toml @@ -35,6 +35,7 @@ ignore = { workspace = true } md-5 = { workspace = true } sha1 = { workspace = true } sha2 = { workspace = true } +sqlx = { workspace = true } [dev-dependencies] test-case = { workspace = true } diff --git a/packages/debmagic/README.md b/packages/debmagic/README.md index 0ef9552c..7a76bc72 100644 --- a/packages/debmagic/README.md +++ b/packages/debmagic/README.md @@ -5,7 +5,7 @@ Modern, robust & easy tooling for building and packaging [Debian](https://debian - **Build any package** in an isolated container environment with `debmagic build` - **Run Debian autopkgtest tests** against built packages with `debmagic test` - **Lint** with `debmagic check` -- **Debug** build environments interactively with `debmagic shell` +- **Debug** environments interactively with `debmagic env shell` ## Installation @@ -59,8 +59,8 @@ Use `--strict` to fail on skipped or undeclared tests (exit code 2). The bare dr ### Useful options - `--distro ` — select the target distro/release (e.g. `trixie`, `noble`) if the changelog is ambiguous -- `--persistent` — retain the build environment for repeated attempts -- `--incremental` — sync only changed sources for faster rebuilds; implies `--persistent` +- `--persistent` — keep the build environment after every build (`always`); the default `on-failure` keeps it only when the build command fails +- `--incremental` — sync only changed sources for faster rebuilds; forces `--persistent=always` - `--sign` — GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo` - `--apt-mirror ` — use a faster mirror for build-dependency resolution @@ -68,10 +68,10 @@ Any of these can be persisted in a `debmagic.toml` config file instead of repeat ### Inspecting a failed build -Failed builds tear down their environment by default. Pass `--shell-on-failure` to drop into a shell when stdout is a TTY, or build with `--persistent` and attach afterwards: +A failed build or test keeps the Environment when Persistence is `on-failure` (the default) or `always`, and prints how to attach: ```shell -debmagic shell +debmagic env shell ``` ## Documentation diff --git a/packages/debmagic/src/build/attach.rs b/packages/debmagic/src/build/attach.rs deleted file mode 100644 index 15110d4b..00000000 --- a/packages/debmagic/src/build/attach.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Unix-socket attach/detach tracking for build environments. -//! -//! While a build keeps its environment alive after a failure, a small -//! socket server in the build root lets concurrent `debmagic shell` -//! sessions register themselves, so the environment is only torn down -//! once the last attached shell detaches. - -use core::time; -use std::net::Shutdown; -use std::os::unix::net::{UnixListener, UnixStream}; -use std::sync::{Arc, Mutex}; -use std::{ - fs, io, - io::{Read, Write}, - path::{Path, PathBuf}, - thread, -}; - -use anyhow::Context; - -fn socket_path_for_build(build_root: &Path) -> PathBuf { - build_root.join("build.sock") -} - -pub fn start_socket_server( - build_root: &Path, - should_exit: Arc>, -) -> anyhow::Result> { - let sock = socket_path_for_build(build_root); - if sock.exists() { - // try to remove stale socket file - let _ = fs::remove_file(&sock); - } - - let listener = UnixListener::bind(&sock) - .with_context(|| format!("failed to bind unix socket {}", sock.display()))?; - - // Set non-blocking mode so we can check the exit flag - listener - .set_nonblocking(true) - .context("failed to set socket non-blocking")?; - - let handle = thread::spawn(move || { - let mut num_attached = 0u64; - loop { - // Check if we should exit - let exit_requested = *should_exit.lock().unwrap(); - if exit_requested && num_attached == 0 { - break; - } - - match listener.accept() { - Ok((mut s, _)) => { - let mut buf = String::new(); - if s.read_to_string(&mut buf).is_err() { - let _ = s.shutdown(Shutdown::Both); - continue; - } - let cmd = buf.trim(); - match cmd { - "attach" => { - num_attached += 1; - } - "detach" => { - num_attached = num_attached.saturating_sub(1); - } - _ => {} - } - let _ = s.shutdown(Shutdown::Both); - } - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { - // No connection available, sleep briefly to avoid busy-waiting - thread::sleep(time::Duration::from_millis(10)); - } - Err(_) => break, - } - } - let _ = fs::remove_file(&sock); - }); - - Ok(handle) -} - -pub fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { - let sock = socket_path_for_build(build_root); - let mut stream = UnixStream::connect(&sock) - .with_context(|| format!("failed to connect to socket {}", sock.display()))?; - stream - .write_all(cmd.as_bytes()) - .context("failed to send socket command")?; - stream.shutdown(Shutdown::Write).ok(); - Ok(()) -} diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index f791c4d0..3b3008ad 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -1,200 +1,64 @@ -use std::sync::{Arc, Mutex}; use std::{ fs, io, - io::{BufReader, IsTerminal, stdout}, - path::{Path, PathBuf}, + path::Path, process::{Command, Stdio}, }; -use crate::build::attach::{send_socket_command, start_socket_server}; use crate::build::source::{source_manifest_path, stage_dir, stage_source_tree}; use crate::build_intent::{BuildIntent, BuildIntentInput, BuildKind, resolve_build_intent}; -use crate::driver::{ - Driver, DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, EnvironmentPurpose, - config::DriverConfig, create_driver, create_driver_from_metadata, remove_environment_root, +use crate::driver::{Driver, DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; +use crate::environment::{ + ClaimedEnvironment, CommandCompletion, HostRootPolicy, InvocationKind, Registry, claim_command, }; -use crate::{config::Config, package::PackageTarget}; +use crate::package::PackageTarget; use anyhow::{Context, anyhow}; use debmagic_common::debian::source::SourceFormat; use debmagic_common::package::SourcePackage; pub mod artifacts; -pub mod attach; pub mod source; pub use source::SourceSyncMode; -struct Build { +struct PlannedBuild { environment: Environment, - driver: Driver, - attached: bool, - output_dir: PathBuf, - clean: bool, - build_debug_symbols: bool, - run_test: bool, - host_arch_variant: Option, + root_policy: HostRootPolicy, + incremental: bool, } -impl Build { - pub fn create(environment: Environment, intent: &BuildIntent) -> anyhow::Result { - let driver = create_driver( - &environment, - &intent.config.driver, - &intent.driver_overrides, - ) - .context(format!("failed to create {:?} driver", environment.driver))?; - Ok(Self { - environment, - driver, - attached: false, - output_dir: intent.output_dir.clone(), - clean: intent.config.clean, - build_debug_symbols: intent.config.build_debug_symbols, - run_test: intent.config.run_test, - host_arch_variant: intent.config.host_arch_variant.clone(), - }) - } - - pub fn from_build_root( - build_root: &Path, - driver_config: &DriverConfig, - ) -> anyhow::Result { - let metadata_path = build_root.join("environment.json"); - if !metadata_path.is_file() { - return Err(anyhow!("No environment.json found")); - } - let file = fs::OpenOptions::new().read(true).open(&metadata_path)?; - let metadata = || -> anyhow::Result { - let reader = BufReader::new(&file); - let metadata: EnvironmentMetadata = - serde_json::from_reader(reader).with_context(|| { - format!( - "Failed to read environment metadata from {} - invalid json", - metadata_path.display() - ) - })?; - Ok(metadata) - }(); - - let metadata = metadata?; - - let driver = create_driver_from_metadata(driver_config, &metadata)?; - - let attached = send_socket_command(build_root, "attach").is_ok(); - - Ok(Self { - environment: metadata.environment.clone(), - driver, - attached, - output_dir: PathBuf::new(), - clean: false, - build_debug_symbols: false, - run_test: true, - host_arch_variant: None, - }) - } - - pub fn detach(&self) -> anyhow::Result<()> { - let build_root = &self.environment.root_dir; - if self.attached { - send_socket_command(build_root, "detach")?; +fn plan_build(intent: &BuildIntent, target: &PackageTarget) -> PlannedBuild { + let package = &target.package; + let environment = Environment::new( + intent.driver, + package.name(), + &format!("{}-{}", package.name(), package.version()), + &intent.source_dir, + target.distro.clone(), + intent.config.driver.persistent, + EnvironmentPurpose::Build, + &intent.config.environments_dir, + ); + + // An incremental sync needs a manifest from a previous build of the same + // kind; without one the tree is reset so no stale files leak into the + // build. A source build shares the Environment with the binary tree, so + // only its own stage dir is restaged. + let manifest = source_manifest_path(&environment, intent.kind); + let incremental = intent.config.incremental && manifest.is_file(); + let root_policy = if incremental { + HostRootPolicy::Keep + } else if intent.kind.is_source() { + HostRootPolicy::ResetStage { + stage_dir: stage_dir(&environment, intent.kind), } - Ok(()) - } - - pub fn write_metadata(&self) -> anyhow::Result<()> { - let metadata = EnvironmentMetadata { - environment: self.environment.clone(), - driver_metadata: self.driver.driver_metadata(), - }; - let path = self.environment.root_dir.join("environment.json"); - let json = serde_json::to_string_pretty(&metadata) - .context("Failed to serialize environment metadata")?; - fs::write(path, json)?; - Ok(()) - } -} - -fn get_build_root_and_identifier( - temp_build_dir: &Path, - package: &SourcePackage, -) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", package.name(), package.version()); - let build_root = temp_build_dir.join(&package_identifier); - (package_identifier, build_root) -} - -/// Create the output dir and stage the source tree into the environment. -fn stage_sources( - environment: &Environment, - intent: &BuildIntent, - target: &PackageTarget, -) -> anyhow::Result<()> { - fs::create_dir_all(&intent.output_dir).context("failed to create output directory")?; - environment - .create_dirs() - .context("failed to create build directories")?; - stage_source_tree( - environment, - intent.kind, - &target.package, - intent.config.source_sync_mode, - intent.config.incremental, - ) -} - -fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Result { - let (package_identifier, build_root) = - get_build_root_and_identifier(&intent.config.temp_build_dir, &target.package); - - let environment = Environment { - driver: intent.driver, - package_name: target.package.name().to_string(), - package_identifier, - root_dir: build_root.clone(), - distro: target.distro.clone(), - persistent: intent.config.driver.persistent, - purpose: EnvironmentPurpose::Build, + } else { + HostRootPolicy::Reset }; - - if intent.config.driver.persistent && build_root.exists() { - // For persistent containers, starting first lets root inside delete - // container-owned files the host user can't remove. - let build = Build::create(environment.clone(), intent) - .context(format!("failed to create {:?} driver", intent.driver))?; - // An incremental sync needs a manifest from a previous build of the - // same kind; without one the tree is reset so no stale files leak - // into the build. - let sync_incrementally = - intent.config.incremental && source_manifest_path(&environment, intent.kind).is_file(); - if sync_incrementally { - if !build.driver.reused_environment() { - // e.g. a new CI runner with a restored build tree; cargo's own - // fingerprinting discards whatever the new toolchain/archive - // state invalidates - println!("Keeping incremental build tree in a fresh build environment"); - } - unapply_quilt_patches(&build, intent.kind, &target.package)?; - } else if intent.kind.is_source() { - // A source build shares the environment with the binary tree: - // only its own stage dir is restaged, never the whole root. - reset_stage_dir(&build, &environment, intent.kind)?; - } else { - build - .driver - .reset_root() - .context("failed to reset persistent build directory")?; - } - stage_sources(&environment, intent, target)?; - return Ok(build); + PlannedBuild { + environment, + root_policy, + incremental, } - - remove_environment_root(&build_root, &intent.config.driver)?; - - crate::output::step("Staging source tree"); - stage_sources(&environment, intent, target)?; - - Build::create(environment, intent) } /// Whether an incremental sync must first unapply quilt patches in the @@ -212,16 +76,16 @@ fn needs_quilt_unapply(source_format: SourceFormat, staged_source_dir: &Path) -> /// `dpkg-source --after-build --unapply-patches` restores the pre-patch /// state from the `.pc` backups and removes the quilt db entirely. fn unapply_quilt_patches( - build: &Build, + driver: &Driver, + environment: &Environment, kind: BuildKind, package: &SourcePackage, ) -> anyhow::Result<()> { - let staged_source_dir = stage_dir(&build.environment, kind); + let staged_source_dir = stage_dir(environment, kind); if !needs_quilt_unapply(package.source_format(), &staged_source_dir) { return Ok(()); } - build - .driver + driver .run_command_checked( &["dpkg-source", "--after-build", "--unapply-patches", "."], &staged_source_dir, @@ -231,46 +95,6 @@ fn unapply_quilt_patches( .context("failed to unapply quilt patches left in the build tree") } -/// Remove every previous leftover from a stage dir (binary residue like -/// `debian//` install trees makes `dpkg-source -b` abort on "unwanted -/// binary file"). Only this dir is wiped — the other kind's tree sharing -/// the environment keeps its incremental outputs. -fn reset_stage_dir( - build: &Build, - environment: &Environment, - kind: BuildKind, -) -> anyhow::Result<()> { - let dir = stage_dir(environment, kind); - if !dir.exists() { - return Ok(()); - } - build - .driver - .run_command_checked(&["find", ".", "-mindepth", "1", "-delete"], &dir, true, &[]) - .context("failed to reset the source stage directory") -} - -pub fn get_shell_in_build(config: &Config, package: &SourcePackage) -> anyhow::Result<()> { - let (_package_identifier, build_root) = - get_build_root_and_identifier(&config.temp_build_dir, package); - let build = Build::from_build_root(&build_root, &config.driver)?; - // the binary tree is the iteration workflow; fall back to the source - // tree when only source builds ever ran - let kind = if stage_dir(&build.environment, BuildKind::Binary).exists() { - BuildKind::Binary - } else { - BuildKind::Source - }; - let result = build - .driver - .interactive_shell(&stage_dir(&build.environment, kind)); - - build.detach()?; - - result?; - Ok(()) -} - fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool, run_test: bool) -> String { let mut options = existing .unwrap_or_default() @@ -292,16 +116,15 @@ struct BuildRequest<'a> { target: &'a PackageTarget, } -/// Shared build orchestration: prepare the environment, run `build_commands` -/// in it, export the artifacts to the output dir, sign them if requested, and -/// clean up (dropping into a shell first when `--shell-on-failure` is set). -/// While the run is in progress, a socket server lets concurrent -/// `debmagic shell` sessions attach to the environment. -fn run_build( - request: &BuildRequest, - build_commands: impl FnOnce(&Build) -> anyhow::Result<()>, -) -> anyhow::Result { +/// Shared build orchestration: claim the Environment, run `build_commands` +/// in it, export the artifacts to the output dir, and sign them if requested. +/// Recording, the failure shell, and finish belong to [`claim_command`]. +async fn run_build( + request: &BuildRequest<'_>, + build_commands: impl FnOnce(&ClaimedEnvironment) -> anyhow::Result<()>, +) -> anyhow::Result { let sign = &request.intent.config.sign; + let registry = Registry::open_default_or_ephemeral().await?; let package = &request.target.package; crate::output::stage(&format!( @@ -309,96 +132,105 @@ fn run_build( package.name(), package.version() )); - let build = prepare_build_env(request.intent, request.target) - .context("failed to prepare build environment")?; - build - .write_metadata() - .context("failed to write environment metadata")?; - - let should_exit = Arc::new(Mutex::new(false)); - let socket_server_handle = - start_socket_server(&build.environment.root_dir, should_exit.clone())?; - - let stop_socket_server = || { - *should_exit.lock().unwrap() = true; - if !socket_server_handle.is_finished() { - println!("Waiting for all attached shells to exit..."); - } - socket_server_handle.join().ok(); - }; - - let result = build_commands(&build).and_then(|()| { - crate::output::stage("Exporting artifacts"); - let changes_file = artifacts::export_build_artifacts( - &build.environment.work_dir(), - &build.output_dir, - request.intent.kind, - )?; - if sign.source.is_enabled() { - crate::output::stage(&format!("Signing {}", build.environment.package_identifier)); - crate::sign::sign_file( - &changes_file, - sign, - sign.source, - sign.notify, - &build.environment.package_identifier, - )?; - } - Ok(changes_file) - }); - - if let Err(error) = &result { - if request.intent.shell_on_failure && stdout().is_terminal() { - eprintln!("Build failed: {error}. Dropping into shell..."); - if let Err(shell_error) = build - .driver - .interactive_shell(&stage_dir(&build.environment, request.intent.kind)) + let planned = plan_build(request.intent, request.target); + let build_kind = request.intent.kind; + let version = package.version().to_string(); + + claim_command( + ®istry, + planned.environment, + &request.intent.config.driver, + &request.intent.driver_overrides, + planned.root_policy, + |environment, driver| { + fs::create_dir_all(&request.intent.output_dir) + .context("failed to create output directory")?; + if planned.incremental + && let Some(driver) = driver { - eprintln!("Dropping into shell failed: {shell_error}"); + unapply_quilt_patches(driver, environment, build_kind, package)?; } - } else if request.intent.shell_on_failure { - eprintln!("Build failed: {error}"); - eprintln!( - "--shell-on-failure is set but stdout is not a TTY; skipping interactive shell" - ); - } else { - eprintln!("Build failed: {error}"); - eprintln!("Re-run with --shell-on-failure to inspect the build environment"); - } - if let Err(cleanup_error) = build.driver.cleanup() { - eprintln!("Failed to clean up build environment: {cleanup_error}"); - } - stop_socket_server(); - return result; - } - - stop_socket_server(); - build - .driver - .cleanup() - .context("failed to clean up build environment")?; - result + crate::output::step("Staging source tree"); + stage_source_tree( + environment, + build_kind, + package, + request.intent.config.source_sync_mode, + request.intent.config.incremental, + ) + }, + InvocationKind::Build(build_kind), + &version, + |claimed| { + let built = build_commands(claimed); + let shell_worthy = built.is_err(); + let result = built.and_then(|()| { + let environment = claimed.environment(); + crate::output::stage(&format!( + "Exporting artifacts to {}", + request.intent.output_dir.display() + )); + let changes_file = artifacts::export_build_artifacts( + &environment.work_dir(), + &request.intent.output_dir, + build_kind, + )?; + if sign.source.is_enabled() { + crate::output::stage(&format!("Signing {}", environment.package_identifier)); + crate::sign::sign_file( + &changes_file, + sign, + sign.source, + sign.notify, + &environment.package_identifier, + )?; + } + Ok(changes_file) + }); + let success = result.is_ok(); + let changes_path = match (&result, build_kind) { + (Ok(path), BuildKind::Binary) => Some(path.clone()), + _ => None, + }; + let (value, failure_lines) = match result { + Ok(path) => (Ok(path), Vec::new()), + Err(error) => { + let failure_lines = vec![format!("Build failed: {error}")]; + (Err(error), failure_lines) + } + }; + Ok(CommandCompletion { + value, + success, + shell_worthy, + finish_error_fails_command: success, + changes_path, + failure_lines, + }) + }, + ) + .await } -pub fn build_package( +pub async fn build_package( intent: &BuildIntent, target: &PackageTarget, changes_options: &[String], -) -> anyhow::Result { +) -> anyhow::Result { let request = BuildRequest { intent, target }; - run_build(&request, |build| { + run_build(&request, |claimed| { crate::output::stage("Building binary packages"); - let staged_source_dir = stage_dir(&build.environment, BuildKind::Binary); + let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); // build-essential is an implicit dependency that `apt-get build-dep` // won't resolve, so install it explicitly. No-op when the environment // already has it (idempotent, and the bare driver runs on the host). - build.driver.run_command_checked( + claimed.driver().run_command_checked( &["apt-get", "-y", "install", "build-essential"], &staged_source_dir, true, &[], )?; - build.driver.run_command_checked( + claimed.driver().run_command_checked( &["apt-get", "-y", "build-dep", "."], &staged_source_dir, true, @@ -407,11 +239,11 @@ pub fn build_package( let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); let options = deb_build_options( inherited_options.as_deref(), - build.build_debug_symbols, - build.run_test, + intent.config.build_debug_symbols, + intent.config.run_test, ); let mut env_add = vec![("DEB_BUILD_OPTIONS", options.as_str())]; - if let Some(variant) = build.host_arch_variant.as_deref() { + if let Some(variant) = intent.config.host_arch_variant.as_deref() { env_add.push(("DEB_HOST_ARCH_VARIANT", variant)); } let mut dpkg_buildpackage_args: Vec = vec![ @@ -420,7 +252,7 @@ pub fn build_package( "-uc".into(), "-ui".into(), ]; - if !build.clean { + if !intent.config.clean { // Non-incremental builds already stage a clean source tree, while // incremental builds preserve their outputs intentionally. dpkg_buildpackage_args.push("-nc".into()); @@ -431,7 +263,7 @@ pub fn build_package( dpkg_buildpackage_args.push("-b".into()); let dpkg_buildpackage_args: Vec<&str> = dpkg_buildpackage_args.iter().map(String::as_str).collect(); - build.driver.run_command_checked( + claimed.driver().run_command_checked( &dpkg_buildpackage_args, &staged_source_dir, false, @@ -439,6 +271,7 @@ pub fn build_package( )?; Ok(()) }) + .await } /// Confirm `cmd` is on `PATH`, failing with an actionable message (rather @@ -500,16 +333,16 @@ pub async fn build_source_package( target: &PackageTarget, changes_options: &[String], include_orig: bool, -) -> anyhow::Result { +) -> anyhow::Result { if intent.driver == DriverType::Bare { check_dpkg_buildpackage_available()?; } // dpkg-source looks for the orig tarball in the parent of the source - // dir it builds, which inside the environment is the work dir. - let orig_fetch_dir = intent - .config - .temp_build_dir + // dir it builds, which inside the environment is the work dir. Fetch + // into the data directory first; `temp_build_dir` no longer exists. + let orig_fetch_dir = crate::data_dir::data_dir() + .context("locating the debmagic data directory")? .join("orig") .join(target.package.name()); let orig_tarball = crate::upstream::orig::fetch_orig_tarball( @@ -527,20 +360,21 @@ pub async fn build_source_package( })?; let request = BuildRequest { intent, target }; - run_build(&request, |build| { + run_build(&request, |claimed| { crate::output::stage("Building source package"); - let staged_source_dir = stage_dir(&build.environment, BuildKind::Source); + let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Source); if let Some(tarball) = &orig_tarball { // the work dir is the staged source dir's parent, which is where // dpkg-source looks for the tarball - let destination = build - .environment - .work_dir() - .join(tarball.file_name().expect("orig tarball has a file name")); + let destination = claimed.environment().work_dir().join( + tarball + .file_name() + .expect("orig tarball has a file name"), + ); stage_file(tarball, &destination)?; } - if build.clean { - build.driver.run_command_checked( + if intent.config.clean { + claimed.driver().run_command_checked( &["apt-get", "-y", "build-dep", "."], &staged_source_dir, true, @@ -555,7 +389,7 @@ pub async fn build_source_package( "-uc".into(), "-ui".into(), ]; - if !build.clean { + if !intent.config.clean { args.push("-nc".into()); } // -sa/-sd decide whether the .changes references the orig tarball @@ -568,9 +402,10 @@ pub async fn build_source_package( args.push(format!("--changes-option={option}")); } let args: Vec<&str> = args.iter().map(String::as_str).collect(); - let result = build - .driver - .run_command_checked(&args, &staged_source_dir, false, &[]); + let result = + claimed + .driver() + .run_command_checked(&args, &staged_source_dir, false, &[]); if result.is_err() && needs_quilt_unapply(target.package.source_format(), &staged_source_dir) { @@ -581,6 +416,7 @@ pub async fn build_source_package( result?; Ok(()) }) + .await .context("failed to build source package") } @@ -638,6 +474,7 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { .context("host's /etc/os-release does not match the build target distro")?; } build_package(&intent, &target, &command.changes_options) + .await .context("Building the package failed")? }; @@ -662,19 +499,6 @@ pub async fn build(command: BuildCommand) -> anyhow::Result<()> { Ok(()) } -/// Run the `debmagic shell` command: attach to the currently active -/// build environment and open an interactive shell in it. -pub fn shell( - fallback_dir: &Path, - source_dir: Option<&Path>, - config_file: Option<&Path>, -) -> anyhow::Result<()> { - let source_dir = crate::package::resolve_source_dir(fallback_dir, source_dir)?; - let config = Config::load(Some(&source_dir), config_file)?; - let identity = crate::package::load_package(&source_dir)?; - get_shell_in_build(&config, &identity) -} - #[cfg(test)] mod tests { use super::*; diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index 08540426..1c898c04 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -524,7 +524,7 @@ pub fn stage_source_tree( mod tests { use std::os::unix::fs::MetadataExt; - use crate::driver::{DriverType, Environment, EnvironmentPurpose}; + use crate::driver::{DriverType, Environment, EnvironmentPurpose, Persistence}; use super::*; @@ -547,9 +547,10 @@ mod tests { driver: DriverType::Bare, package_name: "example".to_string(), package_identifier: "example-1.0".to_string(), + source_dir: source_dir.clone(), root_dir: build_root_dir.clone(), distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), - persistent: true, + persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; environment.create_dirs()?; @@ -615,9 +616,10 @@ mod tests { driver: DriverType::Bare, package_name: "example".to_string(), package_identifier: "example-1.0".to_string(), + source_dir: PathBuf::from("/src"), root_dir: PathBuf::from("/tmp/debmagic/example-1.0"), distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), - persistent: true, + persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; @@ -665,9 +667,10 @@ mod tests { driver: DriverType::Bare, package_name: "example".to_string(), package_identifier: "example-1.0".to_string(), + source_dir: source_dir.clone(), root_dir: test_root.join("build"), distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), - persistent: true, + persistence: Persistence::Always, purpose: EnvironmentPurpose::Build, }; environment.create_dirs()?; diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 891fc9fd..52cf1f9c 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -5,8 +5,7 @@ use anyhow::Context; use crate::{ build::source::SourceSyncMode, config::Config, - driver::{DriverType, config::DriverOverrides}, - package::resolve_source_dir, + driver::{DriverType, Persistence, config::DriverOverrides}, sign::{SignMode, SignTool}, }; @@ -35,7 +34,7 @@ pub struct BuildIntentInput { pub config_file: Option, pub driver: Option, pub kind: BuildKind, - pub persistent: Option, + pub persistent: Option, pub incremental: Option, pub debug_symbols: Option, pub test: Option, @@ -47,7 +46,6 @@ pub struct BuildIntentInput { pub clean: Option, pub source_sync: Option, pub host_arch_variant: Option, - pub shell_on_failure: Option, pub driver_overrides: DriverOverrides, } @@ -60,13 +58,15 @@ pub struct BuildIntent { pub output_dir: PathBuf, pub driver: DriverType, pub kind: BuildKind, - pub shell_on_failure: bool, pub config: Config, pub driver_overrides: DriverOverrides, } pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result { - let source_dir = resolve_source_dir(&input.fallback_dir, input.source_dir.as_deref())?; + // Canonicalized: Environment ids and registry lookups key on the + // source_dir string, so all entry points must agree on one spelling. + let source_dir = std::fs::canonicalize(input.source_dir.unwrap_or(input.fallback_dir)) + .context("resolving source dir failed")?; let mut config = Config::load(Some(&source_dir), input.config_file.as_deref())?; @@ -121,17 +121,14 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result anyhow::Result<()> { let cfg = Config::load(None, Some(&asset_config()))?; - assert!(cfg.driver.persistent); + assert_eq!(cfg.driver.persistent, Persistence::Always); assert_eq!( cfg.driver.docker.base_images.get("debian:trixie"), Some(&"some-debian-trixie-image:latest".to_string()) @@ -268,12 +264,12 @@ mod tests { fn resolve_applies_incremental_implies_persistent() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - input.persistent = Some(false); + input.persistent = Some(Persistence::No); input.incremental = Some(true); let intent = resolve_build_intent(input)?; assert!(intent.config.incremental); - assert!(intent.config.driver.persistent); + assert_eq!(intent.config.driver.persistent, Persistence::Always); Ok(()) } @@ -281,34 +277,20 @@ mod tests { fn resolve_honours_persistent_without_incremental() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - // config1.toml has persistent = true; CLI can turn it off - input.persistent = Some(false); + // config1.toml has persistent = "always"; CLI can turn it off + input.persistent = Some(Persistence::No); input.incremental = Some(false); let intent = resolve_build_intent(input)?; assert!(!intent.config.incremental); - assert!(!intent.config.driver.persistent); + assert_eq!(intent.config.driver.persistent, Persistence::No); Ok(()) } #[test] - fn resolve_passes_through_shell_on_failure() -> anyhow::Result<()> { + fn leftover_shell_on_failure_key_is_ignored() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - input.shell_on_failure = Some(true); - - let intent = resolve_build_intent(input)?; - assert!(intent.shell_on_failure); - Ok(()) - } - - #[test] - fn resolve_shell_on_failure_from_config() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.config_file = None; - input.shell_on_failure = None; - let config_path = std::env::temp_dir().join(format!( "debmagic-shell-on-failure-{}.toml", std::process::id() @@ -317,34 +299,23 @@ mod tests { input.config_file = Some(config_path.clone()); let intent = resolve_build_intent(input)?; - assert!(intent.shell_on_failure); + assert_eq!(intent.config.driver.persistent, Persistence::OnFailure); std::fs::remove_file(config_path)?; Ok(()) } - #[test] - fn resolve_keeps_docker_base_image_override() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.driver_overrides.docker.base_image = Some("custom:image".to_string()); - - let intent = resolve_build_intent(input)?; - assert_eq!( - intent.driver_overrides.docker.base_image.as_deref(), - Some("custom:image") - ); - Ok(()) - } - #[test] fn resolve_absolutizes_paths() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let intent = resolve_build_intent(base_input(dir.clone()))?; assert!(intent.source_dir.is_absolute()); assert!(intent.output_dir.is_absolute()); - assert_eq!(intent.source_dir, std::path::absolute(&dir)?); + assert_eq!(intent.source_dir, std::fs::canonicalize(&dir)?); // default output dir is build/, relative to the package root - assert_eq!(intent.output_dir, std::path::absolute(dir.join("build"))?); + assert_eq!( + intent.output_dir, + std::path::absolute(std::fs::canonicalize(&dir)?.join("build"))? + ); Ok(()) } diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 977f4231..37a70202 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -48,8 +48,12 @@ pub struct Cli { pub enum Commands { #[command(about = "Build a debian package: 'binary' (.deb) or 'source' (.dsc) packages")] Build(Box), - #[command(about = "Open an interactive shell to the currently active build environment")] - Shell(ShellSubcommandArgs), + #[command( + name = "environment", + alias = "env", + about = "List, clean, or open a shell in a Driver Environment" + )] + Environment(EnvSubcommandArgs), #[command(about = "Run the package's declared Debian autopkgtest tests against a prior build")] Test(TestSubcommandArgs), #[command(about = "Check the project")] @@ -229,7 +233,7 @@ pub struct CommonBuildArgs { num_args = 0..=1, default_missing_value = "true", action = clap::ArgAction::Set, - help = "Synchronize changed source inputs to preserve build outputs. Implies --persistent" + help = "Synchronize changed source inputs to preserve build outputs. Forces --persistent=always" )] pub incremental: Option, @@ -243,11 +247,11 @@ pub struct CommonBuildArgs { #[arg( long, num_args = 0..=1, - default_missing_value = "true", + default_missing_value = "always", action = clap::ArgAction::Set, - help = "Keep the build environment for reuse after the build finishes" + help = "How long the build environment outlives this build: 'on-failure' (the default) keeps it when the build command fails, 'always' keeps it after every build, 'no' tears it down. A bare --persistent means 'always'. A failed build that keeps the environment prints `debmagic env shell `." )] - pub persistent: Option, + pub persistent: Option, #[arg( long = "source-sync", @@ -346,15 +350,6 @@ pub struct CommonBuildArgs { )] pub clean: Option, - #[arg( - long = "shell-on-failure", - num_args = 0..=1, - default_missing_value = "true", - action = clap::ArgAction::Set, - help = "On build failure, drop into an interactive shell in the build environment when stdout is a TTY. Defaults to the 'shell_on_failure' setting in the config file (false if unset)." - )] - pub shell_on_failure: Option, - #[command(flatten)] pub common: CommonCli, @@ -429,9 +424,39 @@ pub struct SourceTargetArgs { } #[derive(Args, Debug)] -pub struct ShellSubcommandArgs { - #[command(flatten)] - pub common: CommonCli, +pub struct EnvSubcommandArgs { + #[command(subcommand)] + pub command: EnvCommands, +} + +#[derive(Subcommand, Debug)] +pub enum EnvCommands { + #[command(about = "List Environments in the machine-local registry")] + List, + #[command(about = "Destroy all Stale Environments")] + Clean, + #[command( + about = "Destroy Environments, even healthy ones", + visible_alias = "remove" + )] + Rm { + #[arg(required = true, help = "Environment ids (see `debmagic env list`)")] + ids: Vec, + #[arg( + long, + help = "Destroy even when the Environment is Unreachable, without Driver cooperation" + )] + force: bool, + }, + #[command(about = "Open an interactive shell in an Environment")] + Shell { + #[arg( + help = "Environment id; when omitted, unique Environment for the current Source tree" + )] + id: Option, + #[command(flatten)] + common: CommonCli, + }, } #[derive(Args, Debug)] @@ -439,18 +464,18 @@ pub struct TestSubcommandArgs { #[arg( short, long, - help = "Driver type for the test environment. Defaults to the driver recorded in the prior build's environment.json." + help = "Driver type for the test environment. Defaults to the driver recorded on the prior binary-build Invocation." )] pub driver: Option, #[arg( long, num_args = 0..=1, - default_missing_value = "true", + default_missing_value = "always", action = clap::ArgAction::Set, - help = "Keep the test environment for reuse after the test run finishes" + help = "How long the test environment outlives this run: 'on-failure' (the default) keeps it when the tests fail, 'always' keeps it after every run, 'no' tears it down. A bare --persistent means 'always'. A failed run that keeps the environment prints `debmagic env shell `." )] - pub persistent: Option, + pub persistent: Option, #[command(flatten)] pub docker: DockerArgs, @@ -481,7 +506,7 @@ pub struct TestSubcommandArgs { #[arg( long, - help = "Override the target distribution for the test environment. Defaults to the distro recorded in the prior build's environment.json, not the changelog." + help = "Override the target distribution for the test environment. Defaults to the distro of the prior binary-build Invocation, not the changelog." )] pub distro: Option, @@ -505,15 +530,6 @@ pub struct TestSubcommandArgs { )] pub allow_host_test: bool, - #[arg( - long = "shell-on-failure", - num_args = 0..=1, - default_missing_value = "true", - action = clap::ArgAction::Set, - help = "On test failure, drop into an interactive shell in the test environment when stdout is a TTY. Defaults to the 'shell_on_failure' setting in the config file (false if unset)." - )] - pub shell_on_failure: Option, - #[command(flatten)] pub common: CommonCli, } @@ -690,6 +706,7 @@ impl TestSubcommandArgs { #[cfg(test)] mod tests { use super::*; + use crate::driver::Persistence; use test_case::test_case; fn parse_test(args: &[&str]) -> TestSubcommandArgs { @@ -700,16 +717,19 @@ mod tests { } } - // an absent flag must stay None so the config value applies #[test_case(&[], None; "absent")] - #[test_case(&["--shell-on-failure"], Some(true); "bare flag")] - #[test_case(&["--shell-on-failure=false"], Some(false); "explicit false")] - fn tri_state_flag(args: &[&str], expected: Option) { + #[test_case(&["--persistent"], Some(Persistence::Always); "bare flag")] + #[test_case(&["--persistent=always"], Some(Persistence::Always); "always")] + #[test_case(&["--persistent=on-failure"], Some(Persistence::OnFailure); "on failure")] + #[test_case(&["--persistent=no"], Some(Persistence::No); "no")] + fn persistent_flag(args: &[&str], expected: Option) { let args = parse_test(args); - assert_eq!(args.shell_on_failure, expected); - if expected.is_none() { - assert_eq!(args.persistent, None); - assert_eq!(args.proposed, None); - } + assert_eq!(args.persistent, expected); + } + + #[test] + fn persistent_rejects_a_boolean() { + let error = Cli::try_parse_from(["debmagic", "test", "--persistent=true"]).unwrap_err(); + assert!(error.to_string().contains("always"), "{error}"); } } diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 8f4991ad..8b34247e 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -6,7 +6,7 @@ use crate::driver::config::DriverConfig; use crate::sign::{SignMode, SignTool}; use crate::upload::UploadConfig; use crate::upstream::orig::OrigTarballConfig; -use anyhow::{Context, anyhow}; +use anyhow::{Context, anyhow, bail}; use config::{Config as ConfigBuilder, File}; use serde::{Deserialize, Serialize}; @@ -224,7 +224,8 @@ pub fn run_config(command: ConfigCommand) -> anyhow::Result<()> { #[serde(default)] pub struct Config { pub driver: DriverConfig, - pub temp_build_dir: PathBuf, + /// Parent directory of Environment Host roots. + pub environments_dir: PathBuf, /// Where build artifacts are exported; relative paths resolve against /// the package root. pub output_dir: PathBuf, @@ -249,9 +250,6 @@ pub struct Config { /// builds already stage a clean source tree and incremental builds preserve /// outputs intentionally. pub clean: bool, - /// On build or test failure, drop into an interactive shell in the - /// environment when stdout is a TTY. - pub shell_on_failure: bool, /// Build for a dpkg architecture variant (e.g. `amd64v3` on Ubuntu), /// exported as `DEB_HOST_ARCH_VARIANT` for the build. pub host_arch_variant: Option, @@ -305,7 +303,7 @@ impl Default for Config { fn default() -> Self { Self { driver: DriverConfig::default(), - temp_build_dir: PathBuf::from("/tmp/debmagic"), + environments_dir: crate::data_dir::default_environments_dir(), output_dir: PathBuf::from("build"), incremental: false, source_sync_mode: SourceSyncMode::default(), @@ -316,7 +314,6 @@ impl Default for Config { orig_tarball: OrigTarballConfig::default(), upstream: UpstreamConfig::default(), clean: false, - shell_on_failure: false, host_arch_variant: None, } } @@ -381,11 +378,11 @@ impl Config { let build = builder .build() .context("Failed to initialize config reader")?; - let config: anyhow::Result = build + let mut config: Self = build .try_deserialize() - .map_err(|e| anyhow!("Failed to read config: {e}")); - - config + .map_err(|e| anyhow!("Failed to read config: {e}"))?; + config.environments_dir = resolve_environments_dir(&config.environments_dir)?; + Ok(config) } /// Look up a dotted key (e.g. `sign.key`) in the serialized config, @@ -413,11 +410,44 @@ impl Config { } } +/// Host roots are stored in the Environment registry and used from any +/// working directory later, so `environments_dir` must not depend on the cwd. +/// Config layers are merged before this point, so there is no single file to +/// resolve a relative value against: only `~/` is expanded. +fn resolve_environments_dir(dir: &Path) -> anyhow::Result { + if let Ok(rest) = dir.strip_prefix("~") { + let home = dirs::home_dir().context("cannot expand ~ in environments_dir")?; + return Ok(home.join(rest)); + } + if !dir.is_absolute() { + bail!( + "environments_dir must be an absolute path (or start with ~/), got {}", + dir.display() + ); + } + Ok(dir.to_path_buf()) +} + #[cfg(test)] mod tests { use super::*; use crate::driver::DriverType; + #[test] + fn environments_dir_must_not_depend_on_the_cwd() -> anyhow::Result<()> { + assert_eq!( + resolve_environments_dir(Path::new("/var/debmagic"))?, + PathBuf::from("/var/debmagic") + ); + let home = dirs::home_dir().unwrap(); + assert_eq!( + resolve_environments_dir(Path::new("~/envs"))?, + home.join("envs") + ); + assert!(resolve_environments_dir(Path::new("envs")).is_err()); + Ok(()) + } + #[test] fn it_loads_a_simple_config() -> Result<(), anyhow::Error> { let test_asset_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -429,7 +459,7 @@ mod tests { ConfigPathStatus::Used, )])?; assert_eq!(cfg.driver.default, Some(DriverType::Docker)); - assert!(cfg.driver.persistent); + assert_eq!(cfg.driver.persistent, crate::driver::Persistence::Always); assert!( cfg.driver.docker.base_images.get("debian:trixie") diff --git a/packages/debmagic/src/data_dir.rs b/packages/debmagic/src/data_dir.rs new file mode 100644 index 00000000..3424cd1f --- /dev/null +++ b/packages/debmagic/src/data_dir.rs @@ -0,0 +1,86 @@ +use std::{ffi::OsStr, path::PathBuf}; + +use anyhow::{Context, bail}; + +/// Machine-local data directory: `$DEBMAGIC_DATA_DIR`, or +/// `$XDG_DATA_HOME/debmagic` (`~/.local/share/debmagic`). +/// +/// A relative `DEBMAGIC_DATA_DIR` is resolved to an absolute path against the +/// current directory, so later use does not depend on a different working +/// directory. +pub fn data_dir() -> anyhow::Result { + let override_dir = std::env::var_os("DEBMAGIC_DATA_DIR"); + resolve_data_dir(override_dir.as_deref(), dirs::data_dir().as_deref()) +} + +pub fn registry_path() -> anyhow::Result { + Ok(data_dir()?.join("db.sqlite")) +} + +pub fn default_environments_dir() -> PathBuf { + environments_dir_for(data_dir()) +} + +fn resolve_data_dir( + override_dir: Option<&OsStr>, + xdg_data_home: Option<&std::path::Path>, +) -> anyhow::Result { + if let Some(path) = override_dir { + if path.is_empty() { + bail!("DEBMAGIC_DATA_DIR is set but empty"); + } + return std::path::absolute(path).context("resolving DEBMAGIC_DATA_DIR failed"); + } + + xdg_data_home + .map(|path| path.join("debmagic")) + .context("cannot determine the user data directory") +} + +fn environments_dir_for(data: anyhow::Result) -> PathBuf { + data.unwrap_or_else(|_| PathBuf::from("/tmp/debmagic")) + .join("environments") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + #[test] + fn debmagic_data_dir_overrides_xdg() { + let dir = resolve_data_dir(Some(OsStr::new("/tmp/debmagic-data-test")), None).unwrap(); + assert_eq!(dir, PathBuf::from("/tmp/debmagic-data-test")); + assert_eq!( + environments_dir_for(Ok(dir)), + PathBuf::from("/tmp/debmagic-data-test/environments") + ); + } + + #[test] + fn relative_debmagic_data_dir_is_absolute() { + let dir = resolve_data_dir(Some(OsStr::new("debmagic-data")), None).unwrap(); + assert!(dir.is_absolute()); + assert!(dir.ends_with("debmagic-data")); + } + + #[test] + fn empty_debmagic_data_dir_is_rejected() { + let error = resolve_data_dir(Some(OsStr::new("")), None) + .unwrap_err() + .to_string(); + assert!(error.contains("empty")); + } + + #[test] + fn xdg_data_home_is_used_without_override() { + let dir = resolve_data_dir(None, Some(Path::new("/xdg"))).unwrap(); + assert_eq!(dir, PathBuf::from("/xdg/debmagic")); + } + + #[test] + fn missing_data_dir_falls_back_for_environments() { + let environments = environments_dir_for(Err(anyhow::anyhow!("no home"))); + assert_eq!(environments, PathBuf::from("/tmp/debmagic/environments")); + } +} diff --git a/packages/debmagic/src/driver/config.rs b/packages/debmagic/src/driver/config.rs index a8261a30..93bf145f 100644 --- a/packages/debmagic/src/driver/config.rs +++ b/packages/debmagic/src/driver/config.rs @@ -1,3 +1,4 @@ +use clap::ValueEnum; use serde::{Deserialize, Serialize}; use crate::driver::DriverType; @@ -6,6 +7,46 @@ use crate::driver::driver_docker::{DriverDockerConfig, DriverDockerConfigOverrid use crate::driver::driver_lxd::{DriverLxdConfig, DriverLxdConfigOverrides}; use crate::time::RefreshPolicy; +/// How long the current claim lets an Environment outlive that claim. +/// +/// The claim replaces whatever the registry recorded. `on-failure` and +/// `always` reuse a kept Environment; `no` discards it. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, ValueEnum)] +#[serde(rename_all = "kebab-case")] +pub enum Persistence { + /// Torn down when the claim finishes, discarding one already kept. + No, + /// Kept only when the build or TestRun command itself fails. + #[default] + OnFailure, + /// Kept when the claim finishes. + Always, +} + +impl Persistence { + /// Whether this claim reuses an Environment a previous claim kept. + pub fn reuses(self) -> bool { + matches!(self, Self::OnFailure | Self::Always) + } + + pub fn as_str(self) -> &'static str { + match self { + Self::No => "no", + Self::OnFailure => "on-failure", + Self::Always => "always", + } + } + + pub fn from_name(name: &str) -> anyhow::Result { + match name { + "no" => Ok(Self::No), + "on-failure" => Ok(Self::OnFailure), + "always" => Ok(Self::Always), + other => anyhow::bail!("unknown Persistence '{other}'"), + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct DriverConfig { @@ -13,7 +54,8 @@ pub struct DriverConfig { /// require a driver, from the CLI or here; source builds fall back to /// `bare`. pub default: Option, - pub persistent: bool, + /// Lifetime of the Environment after this claim. Unset is `on-failure`. + pub persistent: Persistence, /// Not used by the bare driver, which builds on the host's own sources. pub apt_mirror: Option, /// Also enable the `-proposed` pocket. Not used by the bare diff --git a/packages/debmagic/src/driver/driver_bare.rs b/packages/debmagic/src/driver/driver_bare.rs index 86058c3d..9c3cb073 100644 --- a/packages/debmagic/src/driver/driver_bare.rs +++ b/packages/debmagic/src/driver/driver_bare.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, IsolationCapability, - config::DriverConfig, + ResourceStatus, config::DriverConfig, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -75,15 +75,15 @@ impl EnvironmentDriver for DriverBare { subprocess::command(command).capture(capture).run() } - fn cleanup(&self) -> anyhow::Result<()> { - Ok(()) - } - - fn interactive_shell(&self, _cwd: &Path) -> std::io::Result<()> { - println!( - "source directory of current package build in {}", - _cwd.display() - ); + fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()> { + let shell = + std::env::var_os("SHELL").unwrap_or_else(|| std::ffi::OsString::from("/bin/sh")); + let mut command = Command::new(shell); + command.current_dir(cwd); + let status = command.status()?; + if !status.success() { + return Err(std::io::Error::other(format!("shell exited with {status}"))); + } Ok(()) } @@ -101,4 +101,16 @@ impl EnvironmentDriver for DriverBare { } Ok(()) } + + fn probe_resource(&self) -> ResourceStatus { + if self.environment.root_dir.exists() { + ResourceStatus::Present + } else { + ResourceStatus::Absent + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + Ok(()) + } } diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index 6daaa1dc..41e9017a 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -11,9 +11,9 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, - EnvironmentMetadata, IsolationCapability, config::DriverConfig, container_name_from_metadata, - container_name_metadata, environment_fingerprint, refresh_apt_index, resource_name, - run_checked, translate_path_in_container, + EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, + container_name_from_metadata, container_name_metadata, environment_fingerprint, + refresh_apt_index, resource_name, run_checked, translate_path_in_container, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -240,6 +240,19 @@ impl DriverDocker { Ok(output?.exit_code == 0) } + fn container_name_for(environment: &Environment) -> String { + resource_name( + "debmagic", + &environment.package_name, + &sanitize_docker_reference(&environment.id()), + ) + } + + /// Driver metadata known before the container exists. + pub fn planned_metadata(environment: &Environment) -> HashMap { + container_name_metadata(&Self::container_name_for(environment)) + } + pub fn create( environment: &Environment, driver_config: &DriverConfig, @@ -273,11 +286,7 @@ impl DriverDocker { container_fingerprint_parts.push(purpose); } let desired_fingerprint = environment_fingerprint(&container_fingerprint_parts); - let container_name = resource_name( - "debmagic", - &environment.package_name, - &sanitize_docker_reference(&environment.identifier()), - ); + let container_name = Self::container_name_for(environment); let mut driver = Self { environment: environment.clone(), container_name, @@ -287,7 +296,7 @@ impl DriverDocker { .as_deref() == Some(&desired_fingerprint); - let mut reuse = environment.persistent && environment_matches; + let mut reuse = environment.persistence.reuses() && environment_matches; if reuse { if !driver.container_is_running()? { driver.container_start()?; @@ -366,7 +375,7 @@ impl DriverDocker { ) -> anyhow::Result { Ok(Self { environment: environment.clone(), - container_name: container_name_from_metadata(metadata)?, + container_name: container_name_from_metadata(metadata), reused_environment: true, }) } @@ -416,14 +425,6 @@ impl EnvironmentDriver for DriverDocker { subprocess::command(exec_cmd).capture(capture).run() } - fn cleanup(&self) -> anyhow::Result<()> { - if self.environment.persistent { - Ok(()) - } else { - self.container_remove_force() - } - } - fn reset_root(&self) -> std::io::Result<()> { let find_cmd = [ "find", @@ -480,4 +481,35 @@ impl EnvironmentDriver for DriverDocker { fn isolation_capability(&self) -> IsolationCapability { IsolationCapability::Container } + + fn probe_resource(&self) -> ResourceStatus { + match Command::new("docker") + .args(["inspect", &self.container_name]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + ResourceStatus::Unreachable + } + Err(_) => ResourceStatus::Unreachable, + Ok(output) if output.status.success() => ResourceStatus::Present, + Ok(output) => { + // Docker says "No such object", Podman's docker shim "no such object". + let stderr = String::from_utf8_lossy(&output.stderr).to_lowercase(); + if stderr.contains("no such object") || stderr.contains("no such container") { + ResourceStatus::Absent + } else { + ResourceStatus::Unreachable + } + } + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + match self.probe_resource() { + ResourceStatus::Absent => Ok(()), + ResourceStatus::Present | ResourceStatus::Unreachable => self.container_remove_force(), + } + } } diff --git a/packages/debmagic/src/driver/driver_lxd.rs b/packages/debmagic/src/driver/driver_lxd.rs index ea69dabb..9ec62e27 100644 --- a/packages/debmagic/src/driver/driver_lxd.rs +++ b/packages/debmagic/src/driver/driver_lxd.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, fs, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -9,9 +10,9 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, - EnvironmentMetadata, IsolationCapability, config::DriverConfig, container_name_from_metadata, - container_name_metadata, environment_fingerprint, refresh_apt_index, resource_name, - run_checked, translate_path_in_container, + EnvironmentMetadata, IsolationCapability, ResourceStatus, config::DriverConfig, + container_name_from_metadata, container_name_metadata, environment_fingerprint, + refresh_apt_index, resource_name, run_checked, translate_path_in_container, }; use crate::subprocess::{self, Capture, CommandResult}; @@ -36,6 +37,7 @@ impl LxdVariant { const BUILD_USER_UID: u32 = 1000; const BUILD_USER_GID: u32 = 1000; const ENVIRONMENT_CONFIG_KEY: &str = "user.debmagic.environment"; +const PROJECT_KEY: &str = "project"; const ENVIRONMENT_SETUP_VERSION: &str = "dpkg-dev-norec python3; build-user-v1; raw.idmap-v1"; // ── Config ──────────────────────────────────────────────────────────────────── @@ -181,7 +183,7 @@ impl DriverLxd { resource_name( "debmagic-src", &self.environment.package_name, - &self.environment.identifier(), + &self.environment.id(), ) } @@ -245,6 +247,42 @@ impl DriverLxd { Ok((!fingerprint.is_empty()).then_some(fingerprint)) } + fn container_name_for(environment: &Environment) -> String { + resource_name("debmagic", &environment.package_name, &environment.id()) + } + + /// Project: CLI override > config file value. + fn resolve_project( + driver_config: &DriverConfig, + overrides: &DriverLxdConfigOverrides, + ) -> Option { + overrides + .project + .clone() + .or_else(|| driver_config.lxd.project.clone()) + } + + fn metadata_for(container_name: &str, project: Option<&str>) -> HashMap { + let mut meta = container_name_metadata(container_name); + if let Some(project) = project { + meta.insert(PROJECT_KEY.to_string(), project.to_string()); + } + meta + } + + /// Driver metadata known before the container exists. The project must be + /// recorded up front: reattachment cannot recover it from the config. + pub fn planned_metadata( + environment: &Environment, + driver_config: &DriverConfig, + overrides: &DriverLxdConfigOverrides, + ) -> HashMap { + Self::metadata_for( + &Self::container_name_for(environment), + Self::resolve_project(driver_config, overrides).as_deref(), + ) + } + pub fn create( variant: LxdVariant, environment: &Environment, @@ -253,17 +291,8 @@ impl DriverLxd { apt_mirror: Option<&str>, proposed: bool, ) -> anyhow::Result { - let container_name = resource_name( - "debmagic", - &environment.package_name, - &environment.identifier(), - ); - - // Project: CLI override > config file value - let project = overrides - .project - .clone() - .or_else(|| driver_config.lxd.project.clone()); + let container_name = Self::container_name_for(environment); + let project = Self::resolve_project(driver_config, overrides); let base_image = overrides.base_image.clone().unwrap_or_else(|| { driver_config .lxd @@ -300,7 +329,7 @@ impl DriverLxd { let container_entry = base.container_list_entry()?; let environment_matches = container_entry.is_some() && base.container_environment_fingerprint()?.as_deref() == Some(&desired_fingerprint); - let reusing_container = environment.persistent && environment_matches; + let reusing_container = environment.persistence.reuses() && environment_matches; base.reused_environment = reusing_container; let mut initialized_container = false; @@ -329,7 +358,7 @@ impl DriverLxd { } let mut init = base.lxd_cmd("init"); - if !environment.persistent { + if environment.persistence == crate::driver::Persistence::No { init.arg("--ephemeral"); } init.args([&base_image, &container_name]); @@ -487,12 +516,12 @@ impl DriverLxd { _driver_config: &DriverConfig, metadata: &EnvironmentMetadata, ) -> anyhow::Result { - let project = metadata.driver_metadata.get("project").cloned(); + let project = metadata.driver_metadata.get(PROJECT_KEY).cloned(); Ok(Self { variant, environment: environment.clone(), - container_name: container_name_from_metadata(metadata)?, + container_name: container_name_from_metadata(metadata), project, reused_environment: true, }) @@ -593,11 +622,7 @@ impl DriverLxd { impl EnvironmentDriver for DriverLxd { fn driver_metadata(&self) -> std::collections::HashMap { - let mut meta = container_name_metadata(&self.container_name); - if let Some(ref p) = self.project { - meta.insert("project".to_string(), p.clone()); - } - meta + Self::metadata_for(&self.container_name, self.project.as_deref()) } fn run_command( @@ -615,15 +640,6 @@ impl EnvironmentDriver for DriverLxd { self.exec_in_container(cmd, Some(&container_path), requires_root, env_add, capture) } - fn cleanup(&self) -> anyhow::Result<()> { - if self.environment.persistent { - Ok(()) - } else { - // Ephemeral containers are auto-deleted after stopping. - self.container_stop() - } - } - fn reset_root(&self) -> std::io::Result<()> { self.with_running_container(|driver| { driver.exec_in_container_checked( @@ -673,4 +689,19 @@ impl EnvironmentDriver for DriverLxd { fn isolation_capability(&self) -> IsolationCapability { IsolationCapability::Container } + + fn probe_resource(&self) -> ResourceStatus { + match self.container_list_entry() { + Ok(Some(_)) => ResourceStatus::Present, + Ok(None) => ResourceStatus::Absent, + Err(_) => ResourceStatus::Unreachable, + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + match self.container_list_entry() { + Ok(None) => Ok(()), + Ok(Some(_)) | Err(_) => self.container_delete_force(), + } + } } diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index b22490d0..ff759251 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -25,6 +25,8 @@ pub mod driver_bare; pub mod driver_docker; pub mod driver_lxd; +pub use config::Persistence; + /// Path at which the environment root is bind-mounted inside container-based /// drivers (Docker, LXD, Incus). pub const ENVIRONMENT_DIR_IN_CONTAINER: &str = "/debmagic"; @@ -169,12 +171,22 @@ pub fn container_name_metadata(name: &str) -> HashMap { HashMap::from([(CONTAINER_NAME_KEY.to_string(), name.to_string())]) } -pub fn container_name_from_metadata(metadata: &EnvironmentMetadata) -> anyhow::Result { +/// The container name for reattachment. Falls back to the deterministic name +/// derived from the Environment id when the stored metadata is missing it +/// (e.g. the creating run crashed before `driver_metadata` was recorded) — +/// the id is hex, so the Docker and LXD/Incus name schemes coincide. +pub fn container_name_from_metadata(metadata: &EnvironmentMetadata) -> String { metadata .driver_metadata .get(CONTAINER_NAME_KEY) .cloned() - .context("environment metadata has no container_name") + .unwrap_or_else(|| { + resource_name( + "debmagic", + &metadata.environment.package_name, + &metadata.environment.id(), + ) + }) } #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize)] @@ -188,11 +200,28 @@ pub enum DriverType { impl std::fmt::Display for DriverType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str( - self.to_possible_value() - .expect("no skipped variants") - .get_name(), - ) + f.write_str(self.as_str()) + } +} + +impl DriverType { + pub fn as_str(self) -> &'static str { + match self { + Self::Docker => "docker", + Self::Bare => "bare", + Self::Lxd => "lxd", + Self::Incus => "incus", + } + } + + pub fn from_name(name: &str) -> anyhow::Result { + match name { + "docker" => Ok(Self::Docker), + "bare" => Ok(Self::Bare), + "lxd" => Ok(Self::Lxd), + "incus" => Ok(Self::Incus), + other => anyhow::bail!("unknown Driver '{other}'"), + } } } @@ -218,6 +247,21 @@ pub enum EnvironmentPurpose { } impl EnvironmentPurpose { + pub fn as_str(self) -> &'static str { + match self { + Self::Build => "build", + Self::Test => "test", + } + } + + pub fn from_name(name: &str) -> anyhow::Result { + match name { + "build" => Ok(Self::Build), + "test" => Ok(Self::Test), + other => anyhow::bail!("unknown EnvironmentPurpose '{other}'"), + } + } + /// Extra part for environment fingerprints when purpose is not [`Self::Build`]. pub fn fingerprint_part(self) -> Option<&'static str> { match self { @@ -233,15 +277,57 @@ pub struct Environment { #[serde(default)] pub package_name: String, pub package_identifier: String, + pub source_dir: PathBuf, pub root_dir: PathBuf, pub distro: DistroVersion, #[serde(default)] - pub persistent: bool, + pub persistence: Persistence, #[serde(default)] pub purpose: EnvironmentPurpose, } impl Environment { + #[allow(clippy::too_many_arguments)] + pub fn new( + driver: DriverType, + package_name: &str, + package_identifier: &str, + source_dir: &Path, + distro: DistroVersion, + persistence: Persistence, + purpose: EnvironmentPurpose, + environments_dir: &Path, + ) -> Self { + let mut environment = Self { + driver, + package_name: package_name.to_string(), + package_identifier: package_identifier.to_string(), + source_dir: source_dir.to_path_buf(), + root_dir: PathBuf::new(), + distro, + persistence, + purpose, + }; + environment.root_dir = environments_dir.join(environment.id()); + environment + } + + /// Short stable Environment id derived from the identity tuple. + pub fn id(&self) -> String { + let source_dir = self.source_dir.to_string_lossy(); + let mut id = environment_fingerprint(&[ + source_dir.as_ref(), + &self.package_name, + &self.package_identifier, + self.distro.distro.as_str(), + &self.distro.codename, + self.purpose.as_str(), + self.driver.as_str(), + ]); + id.truncate(16); + id + } + pub fn identifier(&self) -> String { let base = format!( "{}-{}-{}", @@ -261,13 +347,27 @@ impl Environment { self.root_dir.join("temp") } + /// Directory the Environment module's own staging uses. Build and test + /// trees live in kind-specific directories under the work dir instead. + pub fn staged_source_dir(&self) -> PathBuf { + self.work_dir().join(&self.package_identifier) + } + pub fn create_dirs(&self) -> io::Result<()> { fs::create_dir_all(self.work_dir())?; fs::create_dir_all(self.temp_dir())?; + fs::create_dir_all(self.staged_source_dir())?; Ok(()) } } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ResourceStatus { + Present, + Absent, + Unreachable, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EnvironmentMetadata { pub environment: Environment, @@ -313,8 +413,6 @@ pub trait EnvironmentDriver { Ok(()) } - fn cleanup(&self) -> anyhow::Result<()>; - fn interactive_shell(&self, cwd: &Path) -> io::Result<()>; fn driver_type(&self) -> DriverType; @@ -327,6 +425,13 @@ pub trait EnvironmentDriver { fn reused_environment(&self) -> bool { true } + + /// Whether the Driver resource for this Environment still exists. + fn probe_resource(&self) -> ResourceStatus; + + /// Destroy the Driver resource even if this is a Persistent driver. + /// Succeeds when the resource is already gone. + fn destroy_resource(&self) -> anyhow::Result<()>; } /// A live environment driver, created for one build/test run. @@ -360,14 +465,6 @@ impl EnvironmentDriver for Driver { } } - fn cleanup(&self) -> anyhow::Result<()> { - match self { - Self::Docker(d) => d.cleanup(), - Self::Bare(d) => d.cleanup(), - Self::Lxd(d) => d.cleanup(), - } - } - fn interactive_shell(&self, cwd: &Path) -> io::Result<()> { match self { Self::Docker(d) => d.interactive_shell(cwd), @@ -407,6 +504,22 @@ impl EnvironmentDriver for Driver { Self::Lxd(d) => d.reused_environment(), } } + + fn probe_resource(&self) -> ResourceStatus { + match self { + Self::Docker(d) => d.probe_resource(), + Self::Bare(d) => d.probe_resource(), + Self::Lxd(d) => d.probe_resource(), + } + } + + fn destroy_resource(&self) -> anyhow::Result<()> { + match self { + Self::Docker(d) => d.destroy_resource(), + Self::Bare(d) => d.destroy_resource(), + Self::Lxd(d) => d.destroy_resource(), + } + } } pub fn create_driver( @@ -454,6 +567,23 @@ pub fn create_driver( } } +/// The Driver metadata [`create_driver`] will produce, computed without +/// touching the Driver. Recorded before creation so a run that dies mid-setup +/// still leaves enough in the registry to find and destroy the resource. +pub fn planned_driver_metadata( + environment: &Environment, + driver_config: &DriverConfig, + overrides: &DriverOverrides, +) -> HashMap { + match environment.driver { + DriverType::Docker => DriverDocker::planned_metadata(environment), + DriverType::Bare => HashMap::new(), + DriverType::Lxd | DriverType::Incus => { + DriverLxd::planned_metadata(environment, driver_config, &overrides.lxd) + } + } +} + pub fn create_driver_from_metadata( driver_config: &DriverConfig, metadata: &EnvironmentMetadata, @@ -487,31 +617,27 @@ pub fn create_driver_from_metadata( /// Remove `root` from the host. If files are owned by a container user the host /// cannot delete, delete them from inside that environment first. Never requires /// host root. -pub fn remove_environment_root(root: &Path, driver_config: &DriverConfig) -> anyhow::Result<()> { +pub fn remove_environment_root( + root: &Path, + driver_config: &DriverConfig, + metadata: Option<&EnvironmentMetadata>, +) -> anyhow::Result<()> { if !root.exists() { return Ok(()); } match fs::remove_dir_all(root) { Ok(()) => Ok(()), Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { - let metadata_path = root.join("environment.json"); - if !metadata_path.is_file() { + let Some(metadata) = metadata else { return Err(e).with_context(|| { format!( - "failed to remove {} (permission denied) and no environment.json is present to delete files from inside the environment", + "failed to remove {} (permission denied) and no Driver metadata is available to delete files from inside the environment", root.display() ) }); - } - let file = fs::OpenOptions::new() - .read(true) - .open(&metadata_path) - .with_context(|| format!("failed to open {}", metadata_path.display()))?; - let metadata: EnvironmentMetadata = - serde_json::from_reader(std::io::BufReader::new(&file)) - .with_context(|| format!("failed to parse {}", metadata_path.display()))?; + }; let driver = - create_driver_from_metadata(driver_config, &metadata).with_context(|| { + create_driver_from_metadata(driver_config, metadata).with_context(|| { format!( "failed to reattach to the environment at {} to delete privileged files", root.display() @@ -566,6 +692,7 @@ mod tests { Environment { driver: DriverType::Docker, package_identifier: package_identifier.to_string(), + source_dir: PathBuf::from("/src"), root_dir: PathBuf::from("/tmp"), distro: DistroVersion { distro: Distro::Debian, @@ -573,7 +700,7 @@ mod tests { version: "15".to_string(), is_devel: false, }, - persistent: false, + persistence: Persistence::No, package_name: "debmagic".to_string(), purpose: EnvironmentPurpose::Build, } @@ -600,6 +727,10 @@ mod tests { environment.identifier(), sample_environment("debmagic-0.0.1~alpha2").identifier() ); + assert_ne!( + environment.id(), + sample_environment("debmagic-0.0.1~alpha2").id() + ); } #[test] @@ -607,6 +738,7 @@ mod tests { let json = r#"{ "driver": "docker", "package_identifier": "pkg-1.0", + "source_dir": "/src", "root_dir": "/tmp/build", "distro": { "distro": "Debian", "codename": "forky", "version": "15" } }"#; diff --git a/packages/debmagic/src/env.rs b/packages/debmagic/src/env.rs new file mode 100644 index 00000000..ed3617d8 --- /dev/null +++ b/packages/debmagic/src/env.rs @@ -0,0 +1,177 @@ +use anyhow::{Context, bail}; + +use crate::config::Config; +use crate::environment::{ + DestroyWhen, EnvironmentStatus, Registry, assess_environment, claim_and_destroy, + select_unique_environment, +}; +use crate::package::load_package; + +pub async fn list_environments(config: &Config) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + println!("No Environments in the registry."); + return Ok(()); + }; + let environments = registry.list().await?; + if environments.is_empty() { + println!("No Environments in the registry."); + return Ok(()); + } + + println!( + "{:<16} {:<6} {:<8} {:<18} {:<10} {:<10} {:<6} {:<11} SOURCE", + "ID", "PURPOSE", "DRIVER", "PACKAGE", "DISTRO", "PERSIST", "STATUS", "ATTACH" + ); + for registered in environments { + let assessed = assess_environment(®istry, ®istered, &config.driver).await?; + let env = ®istered.environment; + println!( + "{:<16} {:<6} {:<8} {:<18} {:<10} {:<10} {:<6} {:<11} {}", + env.id(), + env.purpose.as_str(), + env.driver.as_str(), + env.package_identifier, + env.distro.codename, + env.persistence.as_str(), + assessed.status.as_str(), + assessed.attachments, + env.source_dir.display(), + ); + } + Ok(()) +} + +/// Destroy every Stale Environment on the machine. +pub async fn clean_environments(config: &Config) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + println!("Cleaned 0 Stale Environment(s); skipped 0."); + return Ok(()); + }; + let environments = registry.list().await?; + let mut cleaned = 0; + let mut skipped = 0; + for registered in environments { + let id = registered.environment.id(); + let status = match assess_environment(®istry, ®istered, &config.driver).await { + Ok(assessed) => assessed.status, + Err(error) => { + // Fail closed: never destroy what we cannot assess. + eprintln!("skipping Environment {id}: {error}"); + skipped += 1; + continue; + } + }; + match status { + EnvironmentStatus::Unreachable => { + eprintln!("skipping unreachable Environment {id}"); + skipped += 1; + } + EnvironmentStatus::Stale => { + match claim_and_destroy(®istry, &config.driver, &id, DestroyWhen::StillStale) + .await + { + Ok(true) => cleaned += 1, + Ok(false) => { + eprintln!("skipping Environment {id}: no longer Stale"); + skipped += 1; + } + Err(error) => { + eprintln!("skipping Environment {id}: {error}"); + skipped += 1; + } + } + } + EnvironmentStatus::Live => {} + } + } + println!("Cleaned {cleaned} Stale Environment(s); skipped {skipped}."); + Ok(()) +} + +/// Destroy the given Environments, even healthy ones. Like `docker rm`, a +/// failure does not stop the others. +pub async fn remove_environments( + config: &Config, + ids: &[String], + force: bool, +) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + bail!("no Environment with id {}", ids.join(", ")); + }; + let mut failed = 0; + for id in ids { + match remove_one(®istry, config, id, force).await { + Ok(()) => println!("Destroyed Environment {id}"), + Err(error) => { + eprintln!("Error: {error:#}"); + failed += 1; + } + } + } + if failed > 0 { + bail!("{failed} of {} Environment(s) not removed", ids.len()); + } + Ok(()) +} + +async fn remove_one( + registry: &Registry, + config: &Config, + id: &str, + force: bool, +) -> anyhow::Result<()> { + if registry.get(id).await?.is_none() { + bail!("no Environment with id {id}"); + } + let when = if force { + DestroyWhen::Always + } else { + DestroyWhen::Reachable + }; + if !claim_and_destroy(registry, &config.driver, id, when).await? { + bail!( + "Environment {id} is Unreachable; make the Driver queryable and retry, \ + or pass --force to destroy it without Driver cooperation" + ); + } + Ok(()) +} + +pub async fn shell_environment( + config: &Config, + source_dir: &std::path::Path, + id: Option<&str>, +) -> anyhow::Result<()> { + let Some(registry) = Registry::open_default_if_exists().await? else { + match id { + Some(id) => bail!("no Environment with id {id}"), + None => bail!("no Environment found for this Source tree"), + } + }; + let registered = match id { + Some(id) => registry + .get(id) + .await? + .with_context(|| format!("no Environment with id {id}"))?, + None => { + let package = load_package(source_dir)?; + let candidates = registry.list_for_source_tree(source_dir).await?; + let package_identifier = format!("{}-{}", package.name(), package.version()); + select_unique_environment(&candidates, &package_identifier)?.clone() + } + }; + + crate::environment::shell_environment(®istry, &config.driver, ®istered).await +} + +/// When no explicit id is given, resolve the Source tree from cwd / `--source-dir`. +/// +/// Canonicalized: Environment ids and registry lookups key on the `source_dir` +/// string, so all entry points must agree on one spelling per tree. +pub fn resolve_shell_source_dir( + source_dir: Option<&std::path::Path>, + cwd: &std::path::Path, +) -> anyhow::Result { + let dir = std::fs::canonicalize(source_dir.unwrap_or(cwd))?; + load_package(&dir).map(|_| dir) +} diff --git a/packages/debmagic/src/environment/mod.rs b/packages/debmagic/src/environment/mod.rs new file mode 100644 index 00000000..4fa5368f --- /dev/null +++ b/packages/debmagic/src/environment/mod.rs @@ -0,0 +1,1713 @@ +mod process; +mod registry; + +pub use registry::{InvocationKind, RecordedInvocation, RegisteredEnvironment, Registry}; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, bail}; +use process::Process; +use sqlx::sqlite::SqliteConnection; + +use crate::driver::{ + Driver, Environment, EnvironmentDriver, EnvironmentMetadata, Persistence, ResourceStatus, + config::{DriverConfig, DriverOverrides}, + create_driver_from_metadata, planned_driver_metadata, remove_environment_root, +}; + +/// How [`ClaimedEnvironment::acquire`] treats the Host root of an Environment +/// it reuses (Persistence `on-failure` or `always`). A Host root that is not +/// reused always starts empty. +pub enum HostRootPolicy { + Reset, + /// Keep the tree when `manifest` exists in it, so an Incremental build + /// continues from the previous run's state. + KeepIncremental { + manifest: std::path::PathBuf, + }, + /// Keep the whole Host root. The caller has already decided an incremental + /// manifest for this run exists. + Keep, + /// Delete only this stage directory, so a sibling build kind sharing the + /// Environment keeps its tree. + ResetStage { + stage_dir: std::path::PathBuf, + }, +} + +/// An Environment this process has claimed in the Environment registry, with +/// its Host root prepared and its Driver created. Only [`claim_command`] +/// hands one out, and it always finishes the claim. +pub struct ClaimedEnvironment<'a> { + registry: &'a Registry, + driver_config: &'a DriverConfig, + environment: Environment, + driver: Driver, + /// `on-failure` keeps the Environment only when the build or TestRun + /// command itself failed. Set before [`Self::finish`]. + keep_after_failure: bool, +} + +impl<'a> ClaimedEnvironment<'a> { + /// Claim `environment`, prepare its Host root, run `stage` to fill it + /// (after the Environment directories exist), and create its Driver. + /// + /// A fresh Host root is staged before the Driver is created, because + /// container Drivers bind-mount it. A reused one gets its Driver first, + /// so the Driver can reset the root from inside the Environment. + async fn acquire( + registry: &'a Registry, + environment: Environment, + driver_config: &'a DriverConfig, + overrides: &DriverOverrides, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + ) -> anyhow::Result { + let planned = planned_driver_metadata(&environment, driver_config, overrides); + let previous = begin_environment(registry, &environment, Process::current(), &planned) + .await? + .map(|row| row.metadata()); + + if environment.persistence.reuses() && environment.root_dir.exists() { + let claimed = + Self::create_driver(registry, environment, driver_config, overrides).await?; + if let Err(error) = claimed.prepare_reused_root(root_policy, stage) { + return claimed.abort(error).await; + } + return Ok(claimed); + } + + let prepared = + remove_environment_root(&environment.root_dir, driver_config, previous.as_ref()) + .and_then(|()| { + environment + .create_dirs() + .context("failed to create Environment directories") + }) + .and_then(|()| stage(&environment, None)); + if let Err(error) = prepared { + abandon_claim(registry, &environment, driver_config, false).await; + return Err(error); + } + Self::create_driver(registry, environment, driver_config, overrides).await + } + + pub fn environment(&self) -> &Environment { + &self.environment + } + + pub fn driver(&self) -> &Driver { + &self.driver + } + + async fn finish(self) -> anyhow::Result<()> { + finish_environment( + self.registry, + &self.environment, + &self.driver, + self.driver_config, + self.keep_after_failure, + ) + .await + } + + /// Finish after `error` stopped the run, keeping `error` as the result. + async fn abort(self, error: anyhow::Error) -> anyhow::Result { + let id = self.environment.id(); + if let Err(teardown) = self.finish().await { + eprintln!("Warning: failed to tear down Environment {id}: {teardown:#}"); + } + Err(error) + } + + async fn create_driver( + registry: &'a Registry, + environment: Environment, + driver_config: &'a DriverConfig, + overrides: &DriverOverrides, + ) -> anyhow::Result { + let driver = match crate::driver::create_driver(&environment, driver_config, overrides) { + Ok(driver) => driver, + Err(error) => { + abandon_claim(registry, &environment, driver_config, true).await; + return Err( + error.context(format!("failed to create {:?} driver", environment.driver)) + ); + } + }; + // Best effort: the claim already recorded the planned Driver metadata, + // which is enough to find the resource. + if let Err(error) = record_driver_metadata(registry, &environment, &driver).await { + eprintln!("Warning: failed to record Driver metadata: {error:#}"); + } + Ok(Self { + registry, + driver_config, + environment, + driver, + keep_after_failure: false, + }) + } + + fn prepare_reused_root( + &self, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + ) -> anyhow::Result<()> { + enum Action { + Reset, + Keep, + ResetStage(std::path::PathBuf), + } + let action = match root_policy { + HostRootPolicy::Reset => Action::Reset, + HostRootPolicy::KeepIncremental { manifest } => { + if manifest.is_file() { + Action::Keep + } else { + Action::Reset + } + } + HostRootPolicy::Keep => Action::Keep, + HostRootPolicy::ResetStage { stage_dir } => Action::ResetStage(stage_dir), + }; + match action { + Action::Reset => { + self.driver + .reset_root() + .context("failed to reset the Host root of the reused Environment")?; + } + Action::ResetStage(stage_dir) => { + if stage_dir.exists() { + self.driver + .run_command_checked( + &["find", ".", "-mindepth", "1", "-delete"], + &stage_dir, + true, + &[], + ) + .context("failed to reset the source stage directory")?; + } + } + Action::Keep => { + if !self.driver.reused_environment() { + println!("Keeping incremental build tree in a fresh build environment"); + } + } + } + self.environment + .create_dirs() + .context("failed to create Environment directories")?; + stage(&self.environment, Some(&self.driver)) + } +} + +async fn record_driver_metadata( + registry: &Registry, + environment: &Environment, + driver: &Driver, +) -> anyhow::Result<()> { + let mut conn = registry.connection().await?; + registry::set_driver_metadata(&mut conn, &environment.id(), &driver.driver_metadata()).await +} + +/// What a command finished with, once an Environment is claimed. +/// +/// `Err` from the command callback is not a completion: nothing is recorded, +/// and the Environment is finished with a warning on failure. +pub struct CommandCompletion { + /// Returned after the Invocation is recorded and the claim is finished. + /// A failed build is a completion whose value is `Err`. + pub value: anyhow::Result, + /// Whether the Invocation succeeded. + pub success: bool, + /// Whether the build or TestRun command itself failed. Persistence + /// `on-failure` keeps the Environment, and the inspection hint is printed. + /// Export, signing, and a strict skip are not this. + pub shell_worthy: bool, + /// Whether a finish error fails the command. Otherwise it is a warning + /// and `value` is still returned. + pub finish_error_fails_command: bool, + /// Exported `.changes` for a successful binary build. Absent for a source + /// build, a TestRun, and a failed binary build. + pub changes_path: Option, + /// Lines printed before the inspection hint when `shell_worthy`. + pub failure_lines: Vec, +} + +/// Hint printed when a failed build or TestRun kept its Environment. +pub fn inspection_hint(environment_id: &str) -> String { + format!("Get a shell with `debmagic env shell {environment_id}`") +} + +/// Claim `environment`, stage it, run `command`, and on a completion record +/// the Invocation, maybe print the inspection hint, and finish. +/// +/// The caller builds the Environment and opens the registry. Distro, Driver, +/// Environment id, package name, and source directory on the Invocation come +/// from the Environment. +#[allow(clippy::too_many_arguments)] +pub async fn claim_command( + registry: &Registry, + environment: Environment, + driver_config: &DriverConfig, + overrides: &DriverOverrides, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + kind: InvocationKind, + package_version: &str, + command: impl FnOnce(&ClaimedEnvironment<'_>) -> anyhow::Result>, +) -> anyhow::Result { + let mut claimed = ClaimedEnvironment::acquire( + registry, + environment, + driver_config, + overrides, + root_policy, + stage, + ) + .await?; + let completion = match command(&claimed) { + Ok(completion) => completion, + Err(error) => return claimed.abort(error).await, + }; + + let environment = claimed.environment(); + let environment_id = environment.id(); + let purpose = environment.purpose.as_str(); + let invocation = RecordedInvocation { + kind, + source_dir: environment.source_dir.clone(), + package_name: environment.package_name.clone(), + package_version: package_version.to_string(), + distro: environment.distro.clone(), + driver: Some(environment.driver), + success: completion.success, + changes_path: completion.changes_path.clone(), + environment_id: Some(environment_id.clone()), + }; + if let Err(error) = registry.record_invocation(&invocation).await { + eprintln!("Warning: failed to record Invocation: {error:#}"); + } + + if completion.shell_worthy { + for line in &completion.failure_lines { + eprintln!("{line}"); + } + if environment.persistence.reuses() { + eprintln!("{}", inspection_hint(&environment_id)); + claimed.keep_after_failure = true; + } + } + + let finish_error_fails = completion.finish_error_fails_command; + let value = completion.value; + match claimed.finish().await { + Ok(()) => value, + Err(error) if finish_error_fails => { + Err(error.context(format!("failed to clean up {purpose} environment"))) + } + Err(error) => { + eprintln!("Failed to clean up {purpose} environment: {error:#}"); + value + } + } +} + +/// Register `environment` as claimed by `owner` with the Driver metadata +/// planned for it, refusing when a live different process already owns it +/// or a live Attachment is open on it (the run would reset the Host root, or +/// replace the container, under the shell). +/// +/// Returns the row as it was before, so the caller can still reach a Driver +/// resource recorded under the previous metadata. +async fn begin_environment( + registry: &Registry, + environment: &Environment, + owner: Process, + planned_driver_metadata: &HashMap, +) -> anyhow::Result> { + let id = environment.id(); + let mut tx = registry.write().await?; + let previous = registry::get_environment(&mut tx, &id).await?; + if let Some(previous) = &previous { + if let Some(holder) = previous.owner + && holder.pid != owner.pid + && holder.is_alive() + { + bail!( + "Environment {id} is already in use by process {}", + holder.pid + ); + } + let attachments = live_attachment_count(&mut tx, &id).await?; + if attachments > 0 { + bail!( + "Environment {id} has {attachments} live Attachment(s) \ + (`debmagic env shell`); exit them first" + ); + } + } + registry::upsert_environment(&mut tx, environment, Some(owner)).await?; + registry::set_driver_metadata(&mut tx, &id, planned_driver_metadata).await?; + tx.commit().await?; + Ok(previous) +} + +/// Verify that an Environment has no live Attachments and no live owner, +/// then mark it as being destroyed by this process, returning its row. +/// Until released or deleted, this blocks new `env shell` Attachments +/// (refused while `destroying`) and `begin_environment` (live owner). +async fn claim_for_destroy(registry: &Registry, id: &str) -> anyhow::Result { + let me = Process::current(); + let mut tx = registry.write().await?; + let row = registry::get_environment(&mut tx, id) + .await? + .with_context(|| format!("no Environment with id {id}"))?; + if live_attachment_count(&mut tx, id).await? > 0 { + bail!("Environment {id} has live Attachments; stop them first"); + } + if let Some(holder) = row.owner + && holder.pid != me.pid + && holder.is_alive() + { + if row.destroying { + bail!( + "Environment {id} is already being destroyed by process {}", + holder.pid + ); + } + bail!("Environment {id} still has a creating command running"); + } + registry::set_claim(&mut tx, id, Some(me), true).await?; + tx.commit().await?; + Ok(row) +} + +async fn owned_by_me(conn: &mut SqliteConnection, id: &str) -> anyhow::Result { + let me = Process::current(); + Ok(registry::get_environment(conn, id) + .await? + .and_then(|row| row.owner) + .is_some_and(|owner| owner.pid == me.pid)) +} + +/// Drop this process's hold on an Environment, if it still has it. +async fn release(registry: &Registry, id: &str) -> anyhow::Result<()> { + let mut tx = registry.write().await?; + if owned_by_me(&mut tx, id).await? { + registry::set_claim(&mut tx, id, None, false).await?; + } + tx.commit().await?; + Ok(()) +} + +/// Delete the registry row, if this process still holds it. +async fn delete_if_owned(registry: &Registry, id: &str) -> anyhow::Result<()> { + let mut tx = registry.write().await?; + if owned_by_me(&mut tx, id).await? { + registry::delete_environment(&mut tx, id).await?; + } + tx.commit().await?; + Ok(()) +} + +/// Live Attachments of an Environment, deleting those whose shell is gone. +async fn live_attachment_count(conn: &mut SqliteConnection, id: &str) -> anyhow::Result { + let mut live = 0; + for attachment in registry::attachments(&mut *conn, id).await? { + if attachment.process.is_alive() { + live += 1; + } else { + registry::delete_attachment(&mut *conn, attachment.id).await?; + } + } + Ok(live) +} + +async fn reap_attachments(registry: &Registry, id: &str) -> anyhow::Result { + let mut tx = registry.write().await?; + let live = live_attachment_count(&mut tx, id).await?; + tx.commit().await?; + Ok(live) +} + +/// Wait until no live Attachments remain. With `mark_destroying`, mark the +/// Environment as being destroyed in that same transaction, so no new +/// Attachment slips in before the teardown. +async fn wait_for_detach( + registry: &Registry, + id: &str, + mark_destroying: bool, +) -> anyhow::Result<()> { + let mut announced = false; + loop { + let mut tx = registry.write().await?; + let live = live_attachment_count(&mut tx, id).await?; + if live == 0 && mark_destroying { + if !owned_by_me(&mut tx, id).await? { + bail!("Environment {id} is no longer owned by this process"); + } + registry::set_claim(&mut tx, id, Some(Process::current()), true).await?; + } + tx.commit().await?; + if live == 0 { + return Ok(()); + } + if !announced { + println!("Waiting for all attached shells to exit..."); + announced = true; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +/// Attach `process` to an Environment, refusing one that is gone or being +/// destroyed. +async fn attach(registry: &Registry, id: &str, process: Process) -> anyhow::Result { + let mut tx = registry.write().await?; + match registry::get_environment(&mut tx, id).await? { + Some(row) if !row.destroying => {} + _ => bail!("Environment {id} is gone or being destroyed"), + } + let attachment = registry::insert_attachment(&mut tx, id, process).await?; + tx.commit().await?; + Ok(attachment) +} + +/// Undo a claim when acquiring failed before a Driver existed. A failed +/// Driver creation may have left a resource under the planned metadata, so +/// then the row stays (unowned, hence Stale) for `debmagic env clean`. +async fn abandon_claim( + registry: &Registry, + environment: &Environment, + driver_config: &DriverConfig, + driver_attempted: bool, +) { + let id = environment.id(); + let destroy = environment.persistence != Persistence::Always + && !driver_attempted + && match remove_environment_root(&environment.root_dir, driver_config, None) { + Ok(()) => true, + Err(error) => { + eprintln!("Warning: failed to remove the Host root of {id}: {error:#}"); + false + } + }; + let released = if destroy { + delete_if_owned(registry, &id).await + } else { + release(registry, &id).await + }; + if let Err(error) = released { + eprintln!("Warning: failed to release Environment {id} in the registry: {error:#}"); + } +} + +async fn finish_environment( + registry: &Registry, + environment: &Environment, + driver: &Driver, + driver_config: &DriverConfig, + keep_after_failure: bool, +) -> anyhow::Result<()> { + let id = environment.id(); + let keep = match environment.persistence { + Persistence::Always => true, + Persistence::OnFailure => keep_after_failure, + Persistence::No => false, + }; + if keep { + wait_for_detach(registry, &id, false).await?; + return release(registry, &id).await; + } + + // Keep ownership while destroying: concurrent `begin_environment` calls + // see a live owner and refuse, and marking `destroying` blocks new + // Attachments, so nothing can start using the Environment mid-teardown. + wait_for_detach(registry, &id, true).await?; + let metadata = EnvironmentMetadata { + environment: environment.clone(), + driver_metadata: driver.driver_metadata(), + }; + destroy_claimed(registry, &metadata, Some(driver), driver_config, false).await +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnvironmentStatus { + Live, + Stale, + Unreachable, +} + +impl EnvironmentStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Live => "live", + Self::Stale => "stale", + Self::Unreachable => "unreachable", + } + } +} + +/// Live, Stale, or Unreachable, plus the live Attachment count. Assessing +/// reaps dead Attachments. +pub struct AssessedEnvironment { + pub status: EnvironmentStatus, + pub attachments: usize, +} + +/// Derive whether a registered Environment is Live, Stale, or Unreachable, +/// and how many live Attachments it has. +pub async fn assess_environment( + registry: &Registry, + registered: &RegisteredEnvironment, + driver_config: &DriverConfig, +) -> anyhow::Result { + // Fail closed: a registry error must never look like "no Attachments", + // since this status drives destruction. + let attachments = reap_attachments(registry, ®istered.environment.id()).await?; + let leftover = registered.environment.persistence == Persistence::No + && registered.owner.is_none_or(|owner| !owner.is_alive()) + && attachments == 0; + let root_gone = !registered.environment.root_dir.exists(); + + let status = match create_driver_from_metadata(driver_config, ®istered.metadata()) { + Err(_) if leftover || root_gone => EnvironmentStatus::Stale, + Err(_) => EnvironmentStatus::Unreachable, + Ok(driver) => match driver.probe_resource() { + ResourceStatus::Unreachable => EnvironmentStatus::Unreachable, + ResourceStatus::Absent => EnvironmentStatus::Stale, + ResourceStatus::Present if leftover || root_gone => EnvironmentStatus::Stale, + ResourceStatus::Present => EnvironmentStatus::Live, + }, + }; + Ok(AssessedEnvironment { + status, + attachments, + }) +} + +/// Open a shell on an Environment that is already chosen and Live. Holds an +/// Attachment for the duration of the shell and releases it afterwards. +pub async fn shell_environment( + registry: &Registry, + driver_config: &DriverConfig, + registered: &RegisteredEnvironment, +) -> anyhow::Result<()> { + shell_while_attached(registry, driver_config, registered, open_interactive_shell).await +} + +/// Assess, hold an Attachment around `during`, then release it. `during` is +/// the interactive shell in production. Tests pass their own check so they +/// exercise this same path without spawning `$SHELL` on the terminal. +async fn shell_while_attached( + registry: &Registry, + driver_config: &DriverConfig, + registered: &RegisteredEnvironment, + during: impl FnOnce(&DriverConfig, &RegisteredEnvironment) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + let id = registered.environment.id(); + match assess_environment(registry, registered, driver_config) + .await? + .status + { + EnvironmentStatus::Unreachable => bail!("Environment {id} is Unreachable"), + EnvironmentStatus::Stale => { + bail!("Environment {id} is Stale; run `debmagic env rm {id}`") + } + EnvironmentStatus::Live => {} + } + + let attachment = attach(registry, &id, Process::current()).await?; + let result = during(driver_config, registered); + let detached = + async { registry::delete_attachment(&mut *registry.connection().await?, attachment).await }; + result.and(detached.await) +} + +fn open_interactive_shell( + driver_config: &DriverConfig, + registered: &RegisteredEnvironment, +) -> anyhow::Result<()> { + let driver = create_driver_from_metadata(driver_config, ®istered.metadata())?; + driver + .interactive_shell(&shell_cwd(®istered.environment)) + .map_err(|error| anyhow::anyhow!("interactive shell failed: {error}"))?; + Ok(()) +} + +/// Where an interactive shell starts. Binary and source builds stage into +/// sibling trees; the binary tree is the iteration workflow, with the source +/// tree as fallback. Environments staged only by the Environment module +/// itself use [`Environment::staged_source_dir`]. +fn shell_cwd(environment: &Environment) -> std::path::PathBuf { + let binary = environment + .work_dir() + .join(format!("{}-binary", environment.package_identifier)); + let source = environment + .work_dir() + .join(format!("{}-source", environment.package_identifier)); + if binary.is_dir() { + binary + } else if source.is_dir() { + source + } else { + environment.staged_source_dir() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DestroyWhen { + /// `env clean`: only what is still Stale once claimed. + StillStale, + /// `env rm `: anything but Unreachable. + Reachable, + /// `env rm --force`: also without Driver cooperation; Driver + /// failures only warn. + Always, +} + +/// Claim the Environment for destruction (refusing live Attachments and a +/// live owner), then assess the row read by the claim: between the caller's +/// assessment and the claim, a run may have recreated it. Destroys the +/// Driver resource, Host root, and registry row when `when` still holds; +/// otherwise releases the claim and returns `false`. +/// +/// The Driver commands run outside the claim's transaction, which would +/// otherwise lock every other debmagic out of the registry meanwhile; the +/// `destroying` mark keeps the Environment reserved instead. +pub async fn claim_and_destroy( + registry: &Registry, + driver_config: &DriverConfig, + id: &str, + when: DestroyWhen, +) -> anyhow::Result { + let mut claimed = claim_for_destroy(registry, id).await?; + // The claim verified the previous owner was gone; we are only holding + // the row, which must not make it look in use. + claimed.owner = None; + let status = match assess_environment(registry, &claimed, driver_config).await { + Ok(assessed) => assessed.status, + Err(error) => { + let _ = release(registry, id).await; + return Err(error); + } + }; + let proceed = match when { + DestroyWhen::StillStale => status == EnvironmentStatus::Stale, + DestroyWhen::Reachable => status != EnvironmentStatus::Unreachable, + DestroyWhen::Always => true, + }; + if !proceed { + release(registry, id).await?; + return Ok(false); + } + + let force = when == DestroyWhen::Always; + let metadata = claimed.metadata(); + let driver = match create_driver_from_metadata(driver_config, &metadata) { + Ok(driver) => Some(driver), + Err(error) if force => { + eprintln!( + "Warning: cannot reattach to the Driver of {id} ({error}); its resource may leak" + ); + None + } + Err(error) => { + return Err(error.context(format!( + "cannot reattach to the Driver of Environment {id}; \ + pass --force to destroy it without Driver cooperation" + ))); + } + }; + destroy_claimed(registry, &metadata, driver.as_ref(), driver_config, force).await?; + Ok(true) +} + +/// Destroy an Environment this process has marked `destroying`: its Driver +/// resource, Host root, and registry row, in that order. Any failure keeps +/// the claim, so the row stays `destroying` (no new Attachments) and still +/// records the resource for a later `env rm`. With `force`, a failed +/// resource destroy only warns. +async fn destroy_claimed( + registry: &Registry, + metadata: &EnvironmentMetadata, + driver: Option<&Driver>, + driver_config: &DriverConfig, + force: bool, +) -> anyhow::Result<()> { + let id = metadata.environment.id(); + if let Some(driver) = driver { + if let Err(error) = driver.reset_root() { + eprintln!("Warning: failed to reset the Host root of {id} before destroy: {error}"); + } + if let Err(error) = driver.destroy_resource() { + if !force { + return Err(error.context(format!( + "failed to destroy the Driver resource of Environment {id}" + ))); + } + eprintln!("Warning: failed to destroy the Driver resource of {id}: {error}"); + } + } + remove_environment_root( + &metadata.environment.root_dir, + driver_config, + Some(metadata), + )?; + delete_if_owned(registry, &id).await +} + +/// 0 / 1 / 2+ selection for `debmagic env shell` with no explicit id. +/// +/// Environments of the Source tree's current `package_identifier` win over +/// those left behind by earlier versions (each version is its own +/// Environment), so a changelog bump does not make the choice ambiguous. +pub fn select_unique_environment<'a>( + environments: &'a [RegisteredEnvironment], + current_package_identifier: &str, +) -> anyhow::Result<&'a RegisteredEnvironment> { + let current: Vec<&RegisteredEnvironment> = environments + .iter() + .filter(|registered| { + registered.environment.package_identifier == current_package_identifier + }) + .collect(); + let candidates: Vec<&RegisteredEnvironment> = if current.is_empty() { + environments.iter().collect() + } else { + current + }; + match candidates.as_slice() { + [] => bail!("no Environment found for this Source tree"), + [only] => Ok(*only), + many => { + let mut message = String::from( + "more than one Environment exists for this Source tree; pass an Environment id:\n", + ); + for registered in many { + message.push_str(&format!( + " {} {} {} {} {}\n", + registered.environment.id(), + registered.environment.purpose.as_str(), + registered.environment.driver.as_str(), + registered.environment.distro.codename, + registered.environment.package_identifier, + )); + } + bail!("{}", message.trim_end()); + } + } +} + +/// Latest Invocation per DistroVersion. `invocations` must be newest-first. +fn matching_binary_changes<'a>( + invocations: &'a [RecordedInvocation], + distro_filter: Option<&str>, +) -> Vec<&'a RecordedInvocation> { + let mut seen = std::collections::HashSet::new(); + invocations + .iter() + .filter(|invocation| { + distro_filter.is_none_or(|codename| invocation.distro.codename == codename) + }) + .filter(|invocation| { + seen.insert(( + invocation.distro.distro.as_str().to_string(), + invocation.distro.codename.clone(), + )) + }) + .collect() +} + +/// 0 / 1 / 2+ DistroVersions among binary-build Invocations with existing `.changes`. +fn select_unique_binary_changes( + invocations: &[RecordedInvocation], + distro_filter: Option<&str>, +) -> anyhow::Result { + match matching_binary_changes(invocations, distro_filter).as_slice() { + [] => { + if let Some(codename) = distro_filter { + bail!( + "no binary-build Invocation with a .changes file for distro {codename}; run `debmagic build binary` first" + ); + } + bail!( + "no prior binary build found; run `debmagic build binary` first or pass --changes" + ) + } + [only] => Ok((*only).clone()), + many => { + let mut message = String::from( + "more than one DistroVersion has a binary-build Invocation; pass --distro:\n", + ); + for invocation in many { + message.push_str(&format!(" {}\n", invocation.distro.codename)); + } + bail!("{}", message.trim_end()); + } + } +} + +/// Prior binary-build Invocation used to default driver and distro. +/// +/// `--changes` overrides only the artifact path, so history is still consulted. +/// An explicit `.changes` with no matching Invocation is not an error (the +/// caller then requires `--driver` and `--distro`). Several DistroVersions +/// still are. +pub fn prior_binary_invocation( + invocations: &[RecordedInvocation], + distro_filter: Option<&str>, + changes_explicit: bool, +) -> anyhow::Result> { + let matching = matching_binary_changes(invocations, distro_filter); + if changes_explicit && matching.is_empty() { + return Ok(None); + } + select_unique_binary_changes(invocations, distro_filter).map(Some) +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use debmagic_common::distro::{Distro, DistroVersion}; + + use super::registry::tests::{sample_environment, temp_registry, upsert}; + use super::*; + use crate::build_intent::BuildKind; + use crate::driver::{DriverType, EnvironmentPurpose}; + + fn dead_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let pid = child.id(); + child.wait().unwrap(); + pid + } + + fn registered(environment: Environment) -> RegisteredEnvironment { + RegisteredEnvironment { + environment, + driver_metadata: HashMap::new(), + owner: None, + destroying: false, + } + } + + #[tokio::test] + async fn begin_environment_refuses_live_owner_and_allows_takeover() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let none = HashMap::new(); + + // pid 1 is always alive; a different process must not take over. + begin_environment(®istry, &environment, Process::of(1), &none).await?; + let error = begin_environment(®istry, &environment, Process::current(), &none) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("already in use by process 1"), "{error}"); + + // A dead owner can be taken over. + upsert(®istry, &environment, Some(Process::of(dead_pid()))).await; + begin_environment(®istry, &environment, Process::current(), &none).await?; + + // The same process re-beginning is idempotent. + begin_environment(®istry, &environment, Process::current(), &none).await?; + Ok(()) + } + + #[tokio::test] + async fn begin_environment_refuses_live_attachments() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + attach(®istry, &environment.id(), Process::current()).await?; + + let error = begin_environment(®istry, &environment, Process::current(), &HashMap::new()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("live Attachment"), "{error}"); + Ok(()) + } + + #[tokio::test] + async fn begin_environment_records_planned_metadata_and_returns_previous() -> anyhow::Result<()> + { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let first = HashMap::from([("project".to_string(), "old".to_string())]); + let second = HashMap::from([("project".to_string(), "new".to_string())]); + + let me = Process::current(); + assert!( + begin_environment(®istry, &environment, me, &first) + .await? + .is_none() + ); + let previous = begin_environment(®istry, &environment, me, &second) + .await? + .expect("previous row"); + assert_eq!(previous.driver_metadata, first); + assert_eq!( + registry + .get(&environment.id()) + .await? + .unwrap() + .driver_metadata, + second + ); + Ok(()) + } + + #[tokio::test] + async fn released_destroy_claim_allows_attachments_again() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let id = environment.id(); + upsert(®istry, &environment, None).await; + claim_for_destroy(®istry, &id).await?; + release(®istry, &id).await?; + + attach(®istry, &id, Process::current()).await?; + assert_eq!(registry.get(&id).await?.unwrap().owner, None); + Ok(()) + } + + #[tokio::test] + async fn claim_for_destroy_blocks_attachments_and_delete_removes_row() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let id = environment.id(); + upsert(®istry, &environment, None).await; + + claim_for_destroy(®istry, &id).await?; + let error = attach(®istry, &id, Process::current()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("gone or being destroyed"), "{error}"); + + delete_if_owned(®istry, &id).await?; + assert!(registry.get(&id).await?.is_none()); + Ok(()) + } + + #[tokio::test] + async fn claim_for_destroy_refuses_live_attachments() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + attach(®istry, &environment.id(), Process::current()).await?; + + let error = claim_for_destroy(®istry, &environment.id()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("live Attachments"), "{error}"); + Ok(()) + } + + #[test] + fn unique_shell_selection_is_zero_one_many() { + let build = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let test = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Test); + let one = vec![registered(build.clone())]; + assert!(select_unique_environment(&one, "pkg-1.0").is_ok()); + assert!(select_unique_environment(&[], "pkg-1.0").is_err()); + let two = vec![registered(build), registered(test)]; + let error = select_unique_environment(&two, "pkg-1.0") + .unwrap_err() + .to_string(); + assert!(error.contains("more than one Environment")); + } + + #[test] + fn shell_selection_prefers_the_current_package_version() { + let version = |identifier: &str| { + registered(Environment::new( + DriverType::Docker, + "pkg", + identifier, + Path::new("/src/a"), + DistroVersion::new(Distro::Debian, "trixie", "13"), + Persistence::Always, + EnvironmentPurpose::Build, + Path::new("/envs"), + )) + }; + let both = vec![version("pkg-1.0"), version("pkg-1.1")]; + let picked = select_unique_environment(&both, "pkg-1.1").unwrap(); + assert_eq!(picked.environment.package_identifier, "pkg-1.1"); + + // No Environment of the current version: fall back to all of them. + let error = select_unique_environment(&both, "pkg-2.0") + .unwrap_err() + .to_string(); + assert!(error.contains("more than one Environment")); + } + + fn recorded_binary(codename: &str, changes: &str) -> RecordedInvocation { + RecordedInvocation { + kind: InvocationKind::Build(BuildKind::Binary), + source_dir: PathBuf::from("/src"), + package_name: "pkg".to_string(), + package_version: "1.0".to_string(), + distro: DistroVersion::new(Distro::Debian, codename, "13"), + driver: Some(DriverType::Docker), + success: true, + changes_path: Some(PathBuf::from(changes)), + environment_id: None, + } + } + + #[test] + fn binary_changes_selection_splits_on_distro() { + let trixie = recorded_binary("trixie", "/out/trixie.changes"); + let older_trixie = recorded_binary("trixie", "/out/old.changes"); + let sid = recorded_binary("sid", "/out/sid.changes"); + let found = [trixie, older_trixie, sid]; + + let error = select_unique_binary_changes(&found, None) + .unwrap_err() + .to_string(); + assert!(error.contains("more than one DistroVersion")); + let picked = select_unique_binary_changes(&found, Some("trixie")).unwrap(); + assert_eq!( + picked.changes_path.as_deref(), + Some(Path::new("/out/trixie.changes")) + ); + } + + #[test] + fn explicit_changes_still_reads_a_prior_invocation() { + let trixie = recorded_binary("trixie", "/out/trixie.changes"); + let sid = recorded_binary("sid", "/out/sid.changes"); + + let prior = prior_binary_invocation(std::slice::from_ref(&trixie), None, true) + .unwrap() + .unwrap(); + assert_eq!(prior.driver, Some(DriverType::Docker)); + assert_eq!(prior.distro.codename, "trixie"); + + assert!(prior_binary_invocation(&[], None, true).unwrap().is_none()); + + let error = prior_binary_invocation(&[trixie, sid], None, true) + .unwrap_err() + .to_string(); + assert!(error.contains("more than one DistroVersion")); + + let missing = prior_binary_invocation(&[], None, false).unwrap_err(); + assert!(missing.to_string().contains("no prior binary build")); + } +} + +#[cfg(test)] +mod teardown_tests { + use std::path::PathBuf; + + use debmagic_common::distro::{Distro, DistroVersion}; + + use super::registry::tests::upsert; + use super::*; + use crate::driver::{DriverType, EnvironmentPurpose}; + + struct Fixture { + dir: PathBuf, + registry: Registry, + } + + impl Fixture { + async fn new() -> Self { + let dir = + std::env::temp_dir().join(format!("debmagic-lifecycle-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let registry = Registry::open(&dir.join("db.sqlite")).await.unwrap(); + Self { dir, registry } + } + + /// A registered Bare-driver Environment whose Host root exists. + async fn environment(&self, persistence: Persistence, owner: Option) -> Environment { + let environment = Environment::new( + DriverType::Bare, + "pkg", + "pkg-1.0", + &self.dir.join("src"), + DistroVersion::new(Distro::Debian, "trixie", "13"), + persistence, + EnvironmentPurpose::Build, + &self.dir.join("envs"), + ); + environment.create_dirs().unwrap(); + upsert(&self.registry, &environment, owner.map(Process::of)).await; + environment + } + + async fn row(&self, environment: &Environment) -> Option { + self.registry.get(&environment.id()).await.unwrap() + } + + async fn assess(&self, environment: &Environment) -> AssessedEnvironment { + let registered = self.row(environment).await.unwrap(); + assess_environment(&self.registry, ®istered, &DriverConfig::default()) + .await + .unwrap() + } + + async fn claim_and_destroy( + &self, + environment: &Environment, + when: DestroyWhen, + ) -> anyhow::Result { + claim_and_destroy( + &self.registry, + &DriverConfig::default(), + &environment.id(), + when, + ) + .await + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + fn dead_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let pid = child.id(); + child.wait().unwrap(); + pid + } + + #[tokio::test] + async fn non_persistent_environment_with_dead_owner_is_stale() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No, Some(dead_pid())).await; + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Stale + ); + } + + #[tokio::test] + async fn non_persistent_environment_with_live_owner_is_live() { + let fixture = Fixture::new().await; + let environment = fixture + .environment(Persistence::No, Some(std::process::id())) + .await; + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Live + ); + } + + #[tokio::test] + async fn persistent_environment_with_root_and_resource_is_live() { + let fixture = Fixture::new().await; + let environment = fixture + .environment(Persistence::Always, Some(dead_pid())) + .await; + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Live + ); + } + + #[tokio::test] + async fn environment_without_host_root_is_stale() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + std::fs::remove_dir_all(&environment.root_dir).unwrap(); + assert_eq!( + fixture.assess(&environment).await.status, + EnvironmentStatus::Stale + ); + } + + #[tokio::test] + async fn still_stale_releases_the_claim_when_no_longer_stale() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + + assert!( + !fixture + .claim_and_destroy(&environment, DestroyWhen::StillStale) + .await? + ); + + let row = fixture.row(&environment).await.unwrap(); + assert_eq!(row.owner, None); + assert!(environment.root_dir.exists()); + // Released means no longer `destroying`, so Attachments are allowed. + attach(&fixture.registry, &environment.id(), Process::current()).await?; + Ok(()) + } + + #[tokio::test] + async fn still_stale_destroys_a_stale_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No, Some(dead_pid())).await; + + assert!( + fixture + .claim_and_destroy(&environment, DestroyWhen::StillStale) + .await? + ); + + assert!(fixture.row(&environment).await.is_none()); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn reachable_destroys_a_live_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + + assert!( + fixture + .claim_and_destroy(&environment, DestroyWhen::Reachable) + .await? + ); + + assert!(fixture.row(&environment).await.is_none()); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn claim_is_refused_while_an_attachment_is_live() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + attach(&fixture.registry, &environment.id(), Process::current()).await?; + + let result = fixture + .claim_and_destroy(&environment, DestroyWhen::Always) + .await; + + assert!(result.is_err()); + assert!(fixture.row(&environment).await.is_some()); + assert!(environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn assess_reports_live_attachment_count() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + let assessed = fixture.assess(&environment).await; + assert_eq!(assessed.status, EnvironmentStatus::Live); + assert_eq!(assessed.attachments, 0); + + attach(&fixture.registry, &environment.id(), Process::current()).await?; + assert_eq!(fixture.assess(&environment).await.attachments, 1); + + attach( + &fixture.registry, + &environment.id(), + Process::of(dead_pid()), + ) + .await?; + assert_eq!( + fixture.assess(&environment).await.attachments, + 1, + "dead shells are reaped" + ); + Ok(()) + } + + #[tokio::test] + async fn shell_holds_and_releases_attachment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + let registered = fixture.row(&environment).await.unwrap(); + + let mut attached = None; + shell_while_attached( + &fixture.registry, + &DriverConfig::default(), + ®istered, + |_driver_config, registered| { + attached = Some(registered.environment.id()); + Ok(()) + }, + ) + .await?; + assert_eq!(attached, Some(environment.id())); + + let assessed = fixture.assess(&environment).await; + assert_eq!(assessed.status, EnvironmentStatus::Live); + assert_eq!(assessed.attachments, 0); + Ok(()) + } + + #[tokio::test] + async fn shell_on_stale_environment_leaves_no_attachment() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always, None).await; + std::fs::remove_dir_all(&environment.root_dir).unwrap(); + let registered = fixture.row(&environment).await.unwrap(); + + let error = shell_environment(&fixture.registry, &DriverConfig::default(), ®istered) + .await + .unwrap_err(); + assert!(error.to_string().contains("Stale")); + + let assessed = fixture.assess(&environment).await; + assert_eq!(assessed.status, EnvironmentStatus::Stale); + assert_eq!(assessed.attachments, 0); + } +} + +#[cfg(test)] +mod claimed_environment_tests { + use super::*; + use crate::build_intent::BuildKind; + use crate::driver::{DriverType, EnvironmentPurpose}; + use debmagic_common::distro::{Distro, DistroVersion}; + use std::path::{Path, PathBuf}; + + struct Fixture { + dir: PathBuf, + registry: Registry, + driver_config: DriverConfig, + overrides: DriverOverrides, + } + + impl Fixture { + async fn new() -> Self { + let dir = + std::env::temp_dir().join(format!("debmagic-lifecycle-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let registry = Registry::open(&dir.join("db.sqlite")).await.unwrap(); + Self { + dir, + registry, + driver_config: DriverConfig::default(), + overrides: DriverOverrides::default(), + } + } + + fn environment(&self, persistence: Persistence) -> Environment { + Environment::new( + DriverType::Bare, + "pkg", + "pkg-1.0", + &self.dir.join("src"), + DistroVersion::new(Distro::Debian, "trixie", "13"), + persistence, + EnvironmentPurpose::Build, + &self.dir.join("environments"), + ) + } + + async fn acquire( + &self, + environment: Environment, + root_policy: HostRootPolicy, + stage: impl FnOnce(&Environment, Option<&Driver>) -> anyhow::Result<()>, + ) -> anyhow::Result> { + ClaimedEnvironment::acquire( + &self.registry, + environment, + &self.driver_config, + &self.overrides, + root_policy, + stage, + ) + .await + } + + async fn owner(&self, environment: &Environment) -> Option> { + self.registry + .get(&environment.id()) + .await + .unwrap() + .map(|row| row.owner.map(|owner| owner.pid)) + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + fn write_staged_file(environment: &Environment) -> anyhow::Result<()> { + std::fs::write(environment.staged_source_dir().join("staged"), "")?; + Ok(()) + } + + fn staged_file(environment: &Environment) -> PathBuf { + environment.staged_source_dir().join("staged") + } + + #[tokio::test] + async fn finish_tears_down_non_persistent_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let claimed = fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| write_staged_file(environment), + ) + .await?; + assert!(staged_file(claimed.environment()).is_file()); + assert_eq!( + fixture.owner(&environment).await, + Some(Some(std::process::id())) + ); + + claimed.finish().await?; + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + Ok(()) + } + + #[tokio::test] + async fn staging_failure_releases_claim_and_host_root() { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let result = fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| { + write_staged_file(environment)?; + anyhow::bail!("staging failed") + }, + ) + .await; + assert!(result.is_err()); + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + } + + #[tokio::test] + async fn persistent_environment_stays_registered_without_owner() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always); + fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| write_staged_file(environment), + ) + .await? + .finish() + .await?; + assert!(staged_file(&environment).is_file()); + assert_eq!(fixture.owner(&environment).await, Some(None)); + Ok(()) + } + + #[tokio::test] + async fn reused_host_root_is_kept_only_for_incremental_with_manifest() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always); + let manifest = environment.root_dir.join("manifest"); + let keep = || HostRootPolicy::KeepIncremental { + manifest: manifest.clone(), + }; + let no_stage = |_: &Environment, _: Option<&Driver>| Ok(()); + + fixture + .acquire(environment.clone(), keep(), |environment, _| { + write_staged_file(environment) + }) + .await? + .finish() + .await?; + fixture + .acquire(environment.clone(), keep(), no_stage) + .await? + .finish() + .await?; + assert!(!staged_file(&environment).exists(), "no manifest: reset"); + + fixture + .acquire(environment.clone(), keep(), |environment, _| { + write_staged_file(environment) + }) + .await? + .finish() + .await?; + std::fs::write(&manifest, "")?; + fixture + .acquire(environment.clone(), keep(), no_stage) + .await? + .finish() + .await?; + assert!(staged_file(&environment).is_file(), "manifest: kept"); + + fixture + .acquire(environment.clone(), HostRootPolicy::Reset, no_stage) + .await? + .finish() + .await?; + assert!( + !staged_file(&environment).exists(), + "Reset ignores manifest" + ); + assert!(Path::new(&environment.staged_source_dir()).is_dir()); + Ok(()) + } + + #[tokio::test] + async fn staging_failure_on_reused_root_releases_claim() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::Always); + fixture + .acquire( + environment.clone(), + HostRootPolicy::Reset, + |environment, _| write_staged_file(environment), + ) + .await? + .finish() + .await?; + let result = fixture + .acquire(environment.clone(), HostRootPolicy::Reset, |_, _| { + anyhow::bail!("staging failed") + }) + .await; + assert!(result.is_err()); + assert_eq!(fixture.owner(&environment).await, Some(None)); + Ok(()) + } + + const SOURCE_BUILD: InvocationKind = InvocationKind::Build(BuildKind::Source); + + async fn claim( + fixture: &Fixture, + environment: Environment, + command: impl FnOnce(&ClaimedEnvironment<'_>) -> anyhow::Result>, + ) -> anyhow::Result<()> { + claim_command( + &fixture.registry, + environment, + &fixture.driver_config, + &fixture.overrides, + HostRootPolicy::Reset, + |_, _| Ok(()), + SOURCE_BUILD, + "9.9", + command, + ) + .await + } + + fn completion(value: anyhow::Result<()>) -> CommandCompletion<()> { + let success = value.is_ok(); + CommandCompletion { + value, + success, + shell_worthy: !success, + finish_error_fails_command: success, + changes_path: None, + failure_lines: Vec::new(), + } + } + + #[tokio::test] + async fn claim_command_records_a_successful_completion() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + claim(&fixture, environment.clone(), |_| Ok(completion(Ok(())))).await?; + let rows = fixture + .registry + .invocations_for_source(&environment.source_dir) + .await?; + assert_eq!(rows.len(), 1); + assert!(rows[0].success); + assert_eq!(rows[0].kind, SOURCE_BUILD); + assert_eq!(rows[0].package_version, "9.9"); + assert_eq!(rows[0].driver, Some(DriverType::Bare)); + assert_eq!(rows[0].source_dir, environment.source_dir); + // the Environment row is gone, the Invocation stays + assert_eq!(rows[0].environment_id, None); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn claim_command_records_a_failed_completion() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let error = claim(&fixture, environment.clone(), |_| { + Ok(completion(Err(anyhow::anyhow!("build failed")))) + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("build failed")); + let rows = fixture + .registry + .invocations_for_source(&environment.source_dir) + .await?; + assert_eq!(rows.len(), 1); + assert!(!rows[0].success); + assert_eq!(rows[0].package_version, "9.9"); + assert!(!environment.root_dir.exists()); + Ok(()) + } + + #[tokio::test] + async fn claim_command_tears_down_and_records_nothing_when_the_command_stops() + -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::No); + let error = claim(&fixture, environment.clone(), |_| anyhow::bail!("stopped")) + .await + .unwrap_err(); + assert!(error.to_string().contains("stopped")); + let rows = fixture + .registry + .invocations_for_source(&environment.source_dir) + .await?; + assert!(rows.is_empty()); + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + Ok(()) + } + + #[tokio::test] + async fn on_failure_success_tears_the_environment_down() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::OnFailure); + claim(&fixture, environment.clone(), |_| Ok(completion(Ok(())))).await?; + assert!(!environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, None); + Ok(()) + } + + #[tokio::test] + async fn on_failure_command_failure_keeps_the_environment() -> anyhow::Result<()> { + let fixture = Fixture::new().await; + let environment = fixture.environment(Persistence::OnFailure); + let error = claim(&fixture, environment.clone(), |_| { + Ok(completion(Err(anyhow::anyhow!("build failed")))) + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("build failed")); + assert!(environment.root_dir.exists()); + assert_eq!(fixture.owner(&environment).await, Some(None)); + Ok(()) + } + + #[test] + fn inspection_hint_names_the_environment() { + assert_eq!( + inspection_hint("bld-abc"), + "Get a shell with `debmagic env shell bld-abc`" + ); + } +} diff --git a/packages/debmagic/src/environment/process.rs b/packages/debmagic/src/environment/process.rs new file mode 100644 index 00000000..07539411 --- /dev/null +++ b/packages/debmagic/src/environment/process.rs @@ -0,0 +1,81 @@ +/// A process as the Environment registry records it: the pid plus its start +/// time, so a reused pid does not pass for the recorded process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Process { + pub pid: u32, + /// `None` when unreadable (non-Linux, or the process is already gone). + pub start: Option, +} + +impl Process { + pub fn current() -> Self { + Self::of(std::process::id()) + } + + pub fn of(pid: u32) -> Self { + Self { + pid, + start: start_time(pid), + } + } + + /// Whether the recorded process still runs. Falls back to plain liveness + /// when the start time cannot be compared. + pub fn is_alive(self) -> bool { + if !pid_is_alive(self.pid) { + return false; + } + match (self.start, start_time(self.pid)) { + (Some(recorded), Some(current)) => recorded == current, + _ => true, + } + } +} + +fn pid_is_alive(pid: u32) -> bool { + if pid == 0 { + return false; + } + let result = unsafe { libc::kill(pid as i32, 0) }; + if result == 0 { + return true; + } + std::io::Error::last_os_error() + .raw_os_error() + .is_none_or(|code| code != libc::ESRCH) +} + +/// Field 22 of `/proc//stat`. +fn start_time(pid: u32) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // comm (field 2) is parenthesized and may itself contain spaces or + // parens; the remaining fields start after the last ')'. + let after_comm = stat.rsplit_once(')')?.1; + // starttime is field 22 overall, i.e. index 19 after comm. + after_comm.split_whitespace().nth(19)?.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn current_process_is_alive() { + assert!(Process::current().is_alive()); + } + + #[test] + fn exited_process_is_not_alive() { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let process = Process::of(child.id()); + child.wait().unwrap(); + assert!(!process.is_alive()); + } + + #[test] + fn reused_pid_is_not_the_recorded_process() { + let mut current = Process::current(); + current.start = current.start.map(|start| start + 1); + assert!(!current.is_alive()); + } +} diff --git a/packages/debmagic/src/environment/registry.rs b/packages/debmagic/src/environment/registry.rs new file mode 100644 index 00000000..4e7f8dd3 --- /dev/null +++ b/packages/debmagic/src/environment/registry.rs @@ -0,0 +1,1114 @@ +//! Storage of the Environment registry: rows of Environments, their +//! Attachments, and Invocations. Row functions take a connection, so the +//! parent module composes them into transactions around its lifecycle rules. + +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, bail}; +use sqlx::{ + migrate::{MigrateError, Migrator}, + pool::PoolConnection, + sqlite::{Sqlite, SqliteConnectOptions, SqliteConnection, SqlitePool, SqlitePoolOptions}, +}; + +use super::process::Process; +use crate::build_intent::BuildKind; +use crate::driver::{ + DriverType, Environment, EnvironmentMetadata, EnvironmentPurpose, Persistence, +}; +use debmagic_common::distro::{Distro, DistroVersion}; + +macro_rules! select_environments { + ($rest:literal) => { + concat!( + "select driver, package_name, package_identifier, source_dir, root_dir, + distro_family, distro_codename, distro_version, distro_is_devel, + persistence, purpose, owner_pid, owner_pid_start, destroying, driver_metadata + from environments ", + $rest + ) + }; +} + +macro_rules! select_invocations { + ($rest:literal) => { + concat!( + "select id, kind, source_dir, package_name, package_version, + distro_family, distro_codename, distro_version, distro_is_devel, + driver, success, changes_path, environment_id + from invocations ", + $rest + ) + }; +} + +static MIGRATOR: Migrator = sqlx::migrate!("src/migrations"); + +const BUSY_TIMEOUT: Duration = Duration::from_secs(5); + +pub type Transaction = sqlx::Transaction<'static, Sqlite>; + +#[derive(Debug)] +struct IncompatibleSchema { + version: i64, +} + +impl std::fmt::Display for IncompatibleSchema { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "the Environment registry has schema version {}, which this debmagic does not know; upgrade debmagic", + self.version + ) + } +} + +impl std::error::Error for IncompatibleSchema {} + +fn is_incompatible_schema(error: &anyhow::Error) -> bool { + error + .chain() + .any(|cause| cause.downcast_ref::().is_some()) +} + +fn is_busy(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(is_busy_error) + }) +} + +/// SQLITE_BUSY or SQLITE_LOCKED; extended result codes add the high bits. +fn is_busy_error(error: &sqlx::Error) -> bool { + let sqlx::Error::Database(error) = error else { + return false; + }; + error + .code() + .and_then(|code| code.parse::().ok()) + .is_some_and(|code| matches!(code & 0xff, 5 | 6)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InvocationKind { + Build(BuildKind), + Test, +} + +impl InvocationKind { + fn as_str(self) -> &'static str { + match self { + Self::Build(BuildKind::Binary) => "binary_build", + Self::Build(BuildKind::Source) => "source_build", + Self::Test => "test_run", + } + } + + fn from_name(name: &str) -> anyhow::Result { + Ok(match name { + "binary_build" => Self::Build(BuildKind::Binary), + "source_build" => Self::Build(BuildKind::Source), + "test_run" => Self::Test, + other => bail!("unknown Invocation kind '{other}'"), + }) + } +} + +#[derive(Debug, Clone)] +pub struct RecordedInvocation { + pub kind: InvocationKind, + pub source_dir: PathBuf, + pub package_name: String, + pub package_version: String, + pub distro: DistroVersion, + pub driver: Option, + pub success: bool, + pub changes_path: Option, + pub environment_id: Option, +} + +#[derive(Debug, Clone)] +pub struct RegisteredEnvironment { + pub environment: Environment, + pub(super) driver_metadata: HashMap, + /// The command holding the Environment. + pub(super) owner: Option, + pub(super) destroying: bool, +} + +impl RegisteredEnvironment { + pub(super) fn metadata(&self) -> EnvironmentMetadata { + EnvironmentMetadata { + environment: self.environment.clone(), + driver_metadata: self.driver_metadata.clone(), + } + } +} + +pub(super) struct Attachment { + pub id: i64, + pub process: Process, +} + +pub struct Registry { + pool: SqlitePool, +} + +impl Registry { + async fn connect(options: SqliteConnectOptions) -> anyhow::Result { + // One connection: no caller holds two at once, and SQLite serializes + // writers anyway. Never closed while idle, so an in-memory registry + // lives as long as the process. + let pool = SqlitePoolOptions::new() + .max_connections(1) + .idle_timeout(None) + .max_lifetime(None) + .connect_with(options.busy_timeout(BUSY_TIMEOUT).foreign_keys(true)) + .await?; + Ok(Self { pool }) + } + + pub async fn open(path: &Path) -> anyhow::Result { + Self::open_file(path) + .await + .with_context(|| format!("opening the Environment registry {} failed", path.display())) + } + + async fn open_file(path: &Path) -> anyhow::Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {} failed", parent.display()))?; + } + let options = SqliteConnectOptions::new() + .filename(path) + .create_if_missing(true); + let registry = Self::connect(options).await?; + registry.enable_wal().await?; + registry.migrate().await?; + Ok(registry) + } + + /// Open `path` when the file already exists. Does not create the file or + /// its parent directory. + pub async fn open_if_exists(path: &Path) -> anyhow::Result> { + if !path.is_file() { + return Ok(None); + } + Self::open(path).await.map(Some) + } + + /// Open the on-disk registry when it already exists. `debmagic env` reads + /// use this so listing or cleaning does not create an empty database. + pub async fn open_default_if_exists() -> anyhow::Result> { + Self::open_if_exists(&crate::data_dir::registry_path()?).await + } + + async fn open_memory() -> anyhow::Result { + let registry = Self::connect(SqliteConnectOptions::new().in_memory(true)).await?; + registry.migrate().await?; + Ok(registry) + } + + async fn warn_and_open_memory(error: &anyhow::Error) -> Self { + eprintln!( + "Warning: cannot open the Environment registry ({error:#}); \ + continuing without persistent bookkeeping" + ); + Self::open_memory() + .await + .expect("an in-memory registry cannot fail to open") + } + + /// Open `path`, or a process-local in-memory registry when the file cannot + /// be used. Still an error: + /// - a schema newer than this binary — falling back would hide + /// Environments the newer debmagic already recorded; + /// - a registry that is busy or locked — it works, another debmagic holds + /// it, and bypassing it would skip the "already in use" check that keeps + /// two runs off the same Environment. + async fn open_or_ephemeral(path: &Path) -> anyhow::Result { + match Self::open(path).await { + Ok(registry) => Ok(registry), + Err(error) if is_incompatible_schema(&error) || is_busy(&error) => Err(error), + Err(error) => Ok(Self::warn_and_open_memory(&error).await), + } + } + + /// For build/test bookkeeping, which must not fail the actual work: see + /// [`Self::open_or_ephemeral`]. `debmagic env` commands *are* registry + /// operations and use [`Self::open_default_if_exists`]. + pub async fn open_default_or_ephemeral() -> anyhow::Result { + match crate::data_dir::registry_path() { + Ok(path) => Self::open_or_ephemeral(&path).await, + Err(error) => Ok(Self::warn_and_open_memory(&error).await), + } + } + + /// Switch to WAL, retrying on SQLITE_BUSY. Changing the journal mode + /// upgrades a shared lock, and SQLite reports that conflict immediately + /// instead of invoking the busy handler, so concurrent first opens must + /// retry by hand. + async fn enable_wal(&self) -> anyhow::Result<()> { + let deadline = Instant::now() + BUSY_TIMEOUT; + loop { + match sqlx::query("pragma journal_mode = wal") + .execute(&self.pool) + .await + { + Ok(_) => return Ok(()), + Err(error) if is_busy_error(&error) && Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err(error) => return Err(error).context("enabling WAL failed"), + } + } + } + + async fn migrate(&self) -> anyhow::Result<()> { + // The SQLite migrator does not lock, so concurrent first opens are + // serialized by running it inside our own write transaction. + let mut tx = self.write().await?; + match MIGRATOR.run(&mut *tx).await { + Ok(()) => {} + Err(MigrateError::VersionMissing(version)) => { + return Err(IncompatibleSchema { version }.into()); + } + Err(MigrateError::VersionMismatch(version)) => bail!( + "schema version {version} of the Environment registry differs from this debmagic's; \ + delete the registry to recreate it" + ), + Err(error) => return Err(error).context("migrating the schema failed"), + } + let violations = sqlx::query("pragma foreign_key_check") + .fetch_all(&mut *tx) + .await + .context("checking foreign keys failed")?; + if !violations.is_empty() { + bail!( + "the Environment registry has {} foreign key violations", + violations.len() + ); + } + tx.commit().await?; + Ok(()) + } + + /// A write transaction. IMMEDIATE takes the write lock up front, so a + /// read-check-write sequence cannot interleave with another debmagic. + pub(super) async fn write(&self) -> anyhow::Result { + self.pool + .begin_with("begin immediate") + .await + .context("starting an Environment registry transaction failed") + } + + /// The connection for a single statement outside a transaction. Release + /// it before calling another registry function: there is only one. + pub(super) async fn connection(&self) -> anyhow::Result> { + self.pool + .acquire() + .await + .context("connecting to the Environment registry failed") + } + + pub async fn get(&self, id: &str) -> anyhow::Result> { + get_environment(&mut *self.connection().await?, id).await + } + + pub async fn list(&self) -> anyhow::Result> { + let rows = sqlx::query_as::<_, EnvironmentRow>(select_environments!( + "order by package_name, source_dir, purpose, driver" + )) + .fetch_all(&mut *self.connection().await?) + .await + .context("listing the Environments failed")?; + rows.into_iter().map(TryInto::try_into).collect() + } + + pub async fn list_for_source_tree( + &self, + source_dir: &Path, + ) -> anyhow::Result> { + let rows = sqlx::query_as::<_, EnvironmentRow>(select_environments!( + "where source_dir = ? order by purpose, driver, distro_codename" + )) + .bind(path_to_string(source_dir)) + .fetch_all(&mut *self.connection().await?) + .await + .context("listing the Environments of the Source tree failed")?; + rows.into_iter().map(TryInto::try_into).collect() + } + + pub async fn record_invocation(&self, invocation: &RecordedInvocation) -> anyhow::Result<()> { + let mut tx = self.write().await?; + sqlx::query( + "insert into invocations ( + kind, source_dir, package_name, package_version, + distro_family, distro_codename, distro_version, distro_is_devel, + driver, success, changes_path, environment_id, created_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(invocation.kind.as_str()) + .bind(path_to_string(&invocation.source_dir)) + .bind(&invocation.package_name) + .bind(&invocation.package_version) + .bind(invocation.distro.distro.as_str()) + .bind(&invocation.distro.codename) + .bind(&invocation.distro.version) + .bind(invocation.distro.is_devel) + .bind(invocation.driver.map(DriverType::as_str)) + .bind(invocation.success) + .bind(invocation.changes_path.as_deref().map(path_to_string)) + .bind(&invocation.environment_id) + .bind(unix_time(SystemTime::now())) + .execute(&mut *tx) + .await + .context("inserting the Invocation failed")?; + prune_invocations(&mut tx).await?; + tx.commit().await?; + Ok(()) + } + + /// Successful binary-build Invocations for this Source tree whose + /// `.changes` still exists, newest first. + pub async fn binary_build_changes_for_tree( + &self, + source_dir: &Path, + ) -> anyhow::Result> { + // Housekeeping only: the filter below skips missing files anyway. + if let Err(error) = self.prune_invocations().await { + eprintln!("Warning: failed to prune Invocation history: {error:#}"); + } + let rows = sqlx::query_as::<_, InvocationRow>(select_invocations!( + "where source_dir = ? and kind = ? and success = 1 order by created_at desc, id desc" + )) + .bind(path_to_string(source_dir)) + .bind(InvocationKind::Build(BuildKind::Binary).as_str()) + .fetch_all(&mut *self.connection().await?) + .await + .context("reading binary-build Invocations failed")?; + let mut found = Vec::new(); + for row in rows { + let invocation = RecordedInvocation::try_from(row)?; + if invocation + .changes_path + .as_ref() + .is_some_and(|path| path.is_file()) + { + found.push(invocation); + } + } + Ok(found) + } + + async fn prune_invocations(&self) -> anyhow::Result<()> { + let mut tx = self.write().await?; + prune_invocations(&mut tx).await?; + tx.commit().await?; + Ok(()) + } + + /// Every Invocation recorded for this Source tree, newest first. + #[cfg(test)] + pub(super) async fn invocations_for_source( + &self, + source_dir: &Path, + ) -> anyhow::Result> { + let rows = sqlx::query_as::<_, InvocationRow>(select_invocations!( + "where source_dir = ? order by created_at desc, id desc" + )) + .bind(path_to_string(source_dir)) + .fetch_all(&mut *self.connection().await?) + .await?; + rows.into_iter().map(TryInto::try_into).collect() + } +} + +/// Drop Invocation rows that can no longer affect a lookup. +/// +/// Keeps the newest row per kind, Source tree, and DistroVersion. A +/// binary-build row is kept only when it succeeded and its `.changes` +/// file still exists, so a later failure does not erase the last usable +/// artifact and a deleted artifact does not linger. +async fn prune_invocations(conn: &mut SqliteConnection) -> anyhow::Result<()> { + let rows = sqlx::query_as::<_, InvocationRow>(select_invocations!( + "order by created_at desc, id desc" + )) + .fetch_all(&mut *conn) + .await + .context("reading the Invocation history failed")?; + let binary_build = InvocationKind::Build(BuildKind::Binary).as_str(); + let mut seen = HashSet::new(); + for row in rows { + let usable = row.kind != binary_build + || (row.success + && row + .changes_path + .as_deref() + .is_some_and(|path| Path::new(path).is_file())); + let key = ( + row.kind, + row.source_dir, + row.distro_family, + row.distro_codename, + ); + if !(usable && seen.insert(key)) { + sqlx::query("delete from invocations where id = ?") + .bind(row.id) + .execute(&mut *conn) + .await + .context("pruning the Invocation history failed")?; + } + } + Ok(()) +} + +pub(super) async fn get_environment( + conn: &mut SqliteConnection, + id: &str, +) -> anyhow::Result> { + sqlx::query_as::<_, EnvironmentRow>(select_environments!("where id = ?")) + .bind(id) + .fetch_optional(conn) + .await + .context("reading the Environment from the registry failed")? + .map(TryInto::try_into) + .transpose() +} + +/// Insert or replace `environment`, keeping its Driver metadata, apt +/// upgrade time, Attachments, and Invocations. +pub(super) async fn upsert_environment( + conn: &mut SqliteConnection, + environment: &Environment, + owner: Option, +) -> anyhow::Result<()> { + sqlx::query( + "insert into environments ( + id, driver, package_name, package_identifier, source_dir, root_dir, + distro_family, distro_codename, distro_version, distro_is_devel, + persistence, purpose, owner_pid, owner_pid_start + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict(id) do update set + driver = excluded.driver, + package_name = excluded.package_name, + package_identifier = excluded.package_identifier, + source_dir = excluded.source_dir, + root_dir = excluded.root_dir, + distro_family = excluded.distro_family, + distro_codename = excluded.distro_codename, + distro_version = excluded.distro_version, + distro_is_devel = excluded.distro_is_devel, + persistence = excluded.persistence, + purpose = excluded.purpose, + owner_pid = excluded.owner_pid, + owner_pid_start = excluded.owner_pid_start, + destroying = 0", + ) + .bind(environment.id()) + .bind(environment.driver.as_str()) + .bind(&environment.package_name) + .bind(&environment.package_identifier) + .bind(path_to_string(&environment.source_dir)) + .bind(path_to_string(&environment.root_dir)) + .bind(environment.distro.distro.as_str()) + .bind(&environment.distro.codename) + .bind(&environment.distro.version) + .bind(environment.distro.is_devel) + .bind(environment.persistence.as_str()) + .bind(environment.purpose.as_str()) + .bind(owner.map(|owner| i64::from(owner.pid))) + .bind(owner.and_then(|owner| owner.start)) + .execute(conn) + .await + .context("writing the Environment to the registry failed")?; + Ok(()) +} + +pub(super) async fn set_driver_metadata( + conn: &mut SqliteConnection, + id: &str, + metadata: &HashMap, +) -> anyhow::Result<()> { + let updated = sqlx::query("update environments set driver_metadata = ? where id = ?") + .bind(serde_json::to_string(metadata)?) + .bind(id) + .execute(conn) + .await + .context("writing the Driver metadata failed")?; + if updated.rows_affected() == 0 { + bail!("Environment {id} is not in the registry"); + } + Ok(()) +} + +pub(super) async fn set_claim( + conn: &mut SqliteConnection, + id: &str, + owner: Option, + destroying: bool, +) -> anyhow::Result<()> { + sqlx::query( + "update environments set owner_pid = ?, owner_pid_start = ?, destroying = ? where id = ?", + ) + .bind(owner.map(|owner| i64::from(owner.pid))) + .bind(owner.and_then(|owner| owner.start)) + .bind(destroying) + .bind(id) + .execute(conn) + .await + .context("writing the claim of the Environment failed")?; + Ok(()) +} + +pub(super) async fn delete_environment( + conn: &mut SqliteConnection, + id: &str, +) -> anyhow::Result<()> { + sqlx::query("delete from environments where id = ?") + .bind(id) + .execute(conn) + .await + .context("deleting the Environment from the registry failed")?; + Ok(()) +} + +pub(super) async fn attachments( + conn: &mut SqliteConnection, + environment_id: &str, +) -> anyhow::Result> { + let rows: Vec<(i64, i64, Option)> = + sqlx::query_as("select id, pid, pid_start from attachments where environment_id = ?") + .bind(environment_id) + .fetch_all(conn) + .await + .context("reading the Attachments failed")?; + Ok(rows + .into_iter() + .map(|(id, pid, start)| Attachment { + id, + process: Process { + pid: pid as u32, + start, + }, + }) + .collect()) +} + +pub(super) async fn insert_attachment( + conn: &mut SqliteConnection, + environment_id: &str, + process: Process, +) -> anyhow::Result { + let inserted = + sqlx::query("insert into attachments (environment_id, pid, pid_start) values (?, ?, ?)") + .bind(environment_id) + .bind(i64::from(process.pid)) + .bind(process.start) + .execute(conn) + .await + .context("recording the Attachment failed")?; + Ok(inserted.last_insert_rowid()) +} + +pub(super) async fn delete_attachment(conn: &mut SqliteConnection, id: i64) -> anyhow::Result<()> { + sqlx::query("delete from attachments where id = ?") + .bind(id) + .execute(conn) + .await + .context("removing the Attachment failed")?; + Ok(()) +} + +fn path_to_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn unix_time(time: SystemTime) -> i64 { + time.duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or(0) +} + +#[derive(sqlx::FromRow)] +struct EnvironmentRow { + driver: String, + package_name: String, + package_identifier: String, + source_dir: String, + root_dir: String, + distro_family: String, + distro_codename: String, + distro_version: String, + distro_is_devel: bool, + persistence: String, + purpose: String, + owner_pid: Option, + owner_pid_start: Option, + destroying: bool, + driver_metadata: String, +} + +impl TryFrom for RegisteredEnvironment { + type Error = anyhow::Error; + + fn try_from(row: EnvironmentRow) -> anyhow::Result { + Ok(Self { + environment: Environment { + driver: DriverType::from_name(&row.driver)?, + package_name: row.package_name, + package_identifier: row.package_identifier, + source_dir: PathBuf::from(row.source_dir), + root_dir: PathBuf::from(row.root_dir), + distro: DistroVersion { + distro: Distro::parse(&row.distro_family), + codename: row.distro_codename, + version: row.distro_version, + is_devel: row.distro_is_devel, + }, + persistence: Persistence::from_name(&row.persistence)?, + purpose: EnvironmentPurpose::from_name(&row.purpose)?, + }, + driver_metadata: serde_json::from_str(&row.driver_metadata) + .context("parsing the Driver metadata failed")?, + owner: row.owner_pid.map(|pid| Process { + pid: pid as u32, + start: row.owner_pid_start, + }), + destroying: row.destroying, + }) + } +} + +#[derive(sqlx::FromRow)] +struct InvocationRow { + id: i64, + kind: String, + source_dir: String, + package_name: String, + package_version: String, + distro_family: String, + distro_codename: String, + distro_version: String, + distro_is_devel: bool, + driver: Option, + success: bool, + changes_path: Option, + environment_id: Option, +} + +impl TryFrom for RecordedInvocation { + type Error = anyhow::Error; + + fn try_from(row: InvocationRow) -> anyhow::Result { + Ok(Self { + kind: InvocationKind::from_name(&row.kind)?, + source_dir: PathBuf::from(row.source_dir), + package_name: row.package_name, + package_version: row.package_version, + distro: DistroVersion { + distro: Distro::parse(&row.distro_family), + codename: row.distro_codename, + version: row.distro_version, + is_devel: row.distro_is_devel, + }, + driver: row + .driver + .as_deref() + .map(DriverType::from_name) + .transpose()?, + success: row.success, + changes_path: row.changes_path.map(PathBuf::from), + environment_id: row.environment_id, + }) + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::*; + + pub(in crate::environment) async fn temp_registry() -> (PathBuf, Registry) { + let dir = std::env::temp_dir().join(format!("debmagic-registry-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let registry = Registry::open(&dir.join("db.sqlite")).await.unwrap(); + (dir, registry) + } + + pub(in crate::environment) fn sample_environment( + source_dir: &Path, + purpose: EnvironmentPurpose, + ) -> Environment { + Environment::new( + DriverType::Docker, + "pkg", + "pkg-1.0", + source_dir, + DistroVersion::new(Distro::Debian, "trixie", "13"), + Persistence::Always, + purpose, + &source_dir.join("envs"), + ) + } + + pub(in crate::environment) async fn upsert( + registry: &Registry, + environment: &Environment, + owner: Option, + ) { + let mut conn = registry.connection().await.unwrap(); + upsert_environment(&mut conn, environment, owner) + .await + .unwrap(); + } + + fn invocation( + kind: InvocationKind, + source_dir: &Path, + version: &str, + distro: &DistroVersion, + success: bool, + changes_path: Option<&Path>, + ) -> RecordedInvocation { + RecordedInvocation { + kind, + source_dir: source_dir.to_path_buf(), + package_name: "pkg".to_string(), + package_version: version.to_string(), + distro: distro.clone(), + driver: Some(DriverType::Docker), + success, + changes_path: changes_path.map(Path::to_path_buf), + environment_id: None, + } + } + + const BINARY: InvocationKind = InvocationKind::Build(BuildKind::Binary); + + async fn invocation_count(registry: &Registry) -> i64 { + sqlx::query_scalar("select count(*) from invocations") + .fetch_one(&mut *registry.connection().await.unwrap()) + .await + .unwrap() + } + + #[test] + fn environment_id_differs_across_source_trees_and_purposes() { + let first = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let second = sample_environment(Path::new("/src/b"), EnvironmentPurpose::Build); + let test = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Test); + assert_ne!(first.id(), second.id()); + assert_ne!(first.id(), test.id()); + assert_eq!( + first.id(), + sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build).id() + ); + assert_eq!(first.id().len(), 16); + } + + #[tokio::test] + async fn upsert_and_list_round_trip() -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, Some(Process::of(1))).await; + let listed = registry.list().await?; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].environment.id(), environment.id()); + assert_eq!(listed[0].environment.source_dir, PathBuf::from("/src/a")); + assert_eq!(listed[0].environment.persistence, Persistence::Always); + assert_eq!(listed[0].owner.map(|owner| owner.pid), Some(1)); + Ok(()) + } + + #[tokio::test] + async fn unknown_enum_values_are_rejected() { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + let error = sqlx::query("update environments set purpose = 'deploy'") + .execute(&mut *registry.connection().await.unwrap()) + .await + .unwrap_err(); + assert!(error.to_string().contains("CHECK"), "{error}"); + } + + #[tokio::test] + async fn deleting_an_environment_drops_its_attachments_and_keeps_invocations() + -> anyhow::Result<()> { + let (_dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + let id = environment.id(); + upsert(®istry, &environment, None).await; + let mut conn = registry.connection().await?; + insert_attachment(&mut conn, &id, Process::current()).await?; + drop(conn); + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + let mut recorded = invocation( + InvocationKind::Test, + Path::new("/src/a"), + "1.0", + &distro, + true, + None, + ); + recorded.environment_id = Some(id.clone()); + registry.record_invocation(&recorded).await?; + + let mut conn = registry.connection().await?; + delete_environment(&mut conn, &id).await?; + assert!(attachments(&mut conn, &id).await?.is_empty()); + drop(conn); + let invocations = registry.invocations_for_source(Path::new("/src/a")).await?; + assert_eq!(invocations.len(), 1); + assert_eq!(invocations[0].environment_id, None); + Ok(()) + } + + #[tokio::test] + async fn invocation_records_full_distro_version() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let changes = dir.join("pkg.changes"); + std::fs::write(&changes, "changes")?; + let distro = DistroVersion { + distro: Distro::Custom("yocto".to_string()), + codename: "kirkstone".to_string(), + version: "4.0".to_string(), + is_devel: true, + }; + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + &distro, + true, + Some(&changes), + )) + .await?; + + let found = registry.binary_build_changes_for_tree(&source).await?; + assert_eq!(found.len(), 1); + assert_eq!(found[0].distro, distro); + assert_eq!(found[0].kind, BINARY); + Ok(()) + } + + #[tokio::test] + async fn binary_changes_lookup_skips_missing_files() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let trixie = dir.join("trixie.changes"); + std::fs::write(&trixie, "changes")?; + let missing = dir.join("gone.changes"); + let distro_trixie = DistroVersion::new(Distro::Debian, "trixie", "13"); + let distro_sid = DistroVersion::new(Distro::Debian, "sid", ""); + for (distro, changes) in [ + (&distro_trixie, &missing), + (&distro_trixie, &trixie), + (&distro_sid, &trixie), + ] { + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + distro, + true, + Some(changes), + )) + .await?; + } + + assert_eq!(invocation_count(®istry).await, 2); + let found = registry.binary_build_changes_for_tree(&source).await?; + let codenames: Vec<_> = found.iter().map(|i| i.distro.codename.as_str()).collect(); + assert_eq!(codenames, ["sid", "trixie"]); + Ok(()) + } + + #[tokio::test] + async fn invocation_history_keeps_the_newest_usable_binary_build() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let older = dir.join("older.changes"); + let newer = dir.join("newer.changes"); + std::fs::write(&older, "old")?; + std::fs::write(&newer, "new")?; + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + &distro, + true, + Some(&older), + )) + .await?; + let mut lxd = invocation(BINARY, &source, "1.1", &distro, true, Some(&newer)); + lxd.driver = Some(DriverType::Lxd); + registry.record_invocation(&lxd).await?; + registry + .record_invocation(&invocation(BINARY, &source, "1.2", &distro, false, None)) + .await?; + + assert_eq!(invocation_count(®istry).await, 1); + let found = registry.binary_build_changes_for_tree(&source).await?; + assert_eq!(found.len(), 1); + assert_eq!(found[0].changes_path.as_deref(), Some(newer.as_path())); + assert_eq!(found[0].driver, Some(DriverType::Lxd)); + assert_eq!(found[0].package_version, "1.1"); + Ok(()) + } + + #[tokio::test] + async fn deleting_a_changes_file_prunes_that_invocation() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let changes = dir.join("pkg.changes"); + std::fs::write(&changes, "changes")?; + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + registry + .record_invocation(&invocation( + BINARY, + &source, + "1.0", + &distro, + true, + Some(&changes), + )) + .await?; + std::fs::remove_file(&changes)?; + + assert!( + registry + .binary_build_changes_for_tree(&source) + .await? + .is_empty() + ); + assert_eq!(invocation_count(®istry).await, 0); + Ok(()) + } + + #[tokio::test] + async fn non_binary_invocations_collapse_to_the_newest() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let source = dir.join("src"); + let distro = DistroVersion::new(Distro::Debian, "trixie", "13"); + for version in ["1.0", "1.1"] { + registry + .record_invocation(&invocation( + InvocationKind::Test, + &source, + version, + &distro, + true, + None, + )) + .await?; + } + let rows = registry.invocations_for_source(&source).await?; + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].package_version, "1.1"); + Ok(()) + } + + #[tokio::test] + async fn open_if_exists_does_not_create_the_registry() -> anyhow::Result<()> { + let path = std::env::temp_dir().join(format!( + "debmagic-missing-{}/db.sqlite", + uuid::Uuid::new_v4() + )); + assert!(Registry::open_if_exists(&path).await?.is_none()); + assert!(!path.parent().unwrap().exists()); + Ok(()) + } + + #[tokio::test] + async fn reopening_keeps_the_rows() -> anyhow::Result<()> { + let (dir, registry) = temp_registry().await; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + registry.pool.close().await; + + let registry = Registry::open(&dir.join("db.sqlite")).await?; + let listed = registry.list().await?; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].environment.id(), environment.id()); + Ok(()) + } + + async fn add_future_migration(path: &Path) { + let registry = Registry::open(path).await.unwrap(); + sqlx::query( + "insert into _sqlx_migrations (version, description, success, checksum, execution_time) + values (99, 'future', 1, x'00', 0)", + ) + .execute(®istry.pool) + .await + .unwrap(); + registry.pool.close().await; + } + + #[tokio::test] + async fn newer_schema_is_refused() { + let (dir, registry) = temp_registry().await; + registry.pool.close().await; + let path = dir.join("db.sqlite"); + add_future_migration(&path).await; + + let Err(error) = Registry::open(&path).await else { + panic!("expected a newer schema to be refused"); + }; + let rendered = format!("{error:#}"); + assert!(rendered.contains("upgrade debmagic"), "{rendered}"); + assert!(rendered.contains("99"), "{rendered}"); + + let Err(error) = Registry::open_or_ephemeral(&path).await else { + panic!("expected the fallback not to hide a newer schema"); + }; + assert!(is_incompatible_schema(&error), "{error:#}"); + } + + #[tokio::test] + async fn open_or_ephemeral_falls_back_when_the_file_is_unusable() -> anyhow::Result<()> { + let (dir, _registry) = temp_registry().await; + // a directory is not a database + let registry = Registry::open_or_ephemeral(&dir).await?; + let environment = sample_environment(Path::new("/src/a"), EnvironmentPurpose::Build); + upsert(®istry, &environment, None).await; + assert_eq!(registry.list().await?.len(), 1); + Ok(()) + } + + /// Each opener gets its own runtime, like separate debmagic processes. + #[test] + fn concurrent_first_opens_all_succeed() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-registry-{}", uuid::Uuid::new_v4())); + let path = dir.join("db.sqlite"); + let barrier = std::sync::Barrier::new(8); + std::thread::scope(|scope| { + let openers: Vec<_> = (0..8) + .map(|_| { + scope.spawn(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + barrier.wait(); + runtime.block_on(Registry::open(&path)).map(|_| ()) + }) + }) + .collect(); + openers + .into_iter() + .try_for_each(|opener| opener.join().expect("opener thread panicked")) + }) + } +} diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 9224fc8e..1cbd5191 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -1,10 +1,9 @@ -use std::env; use std::process::ExitCode; use anyhow::Context; use clap::{CommandFactory, Parser}; -use crate::cli::{BuildTarget, Cli, Commands, ConfigCommands, UpstreamCommands}; +use crate::cli::{BuildTarget, Cli, Commands, ConfigCommands, EnvCommands, UpstreamCommands}; pub mod build; pub mod build_intent; @@ -13,7 +12,10 @@ pub mod changes; pub mod cli; pub mod config; pub mod control; +pub mod data_dir; pub mod driver; +pub mod env; +pub mod environment; pub mod output; pub mod package; pub mod requests; @@ -42,7 +44,7 @@ async fn run() -> anyhow::Result { let cli = Cli::parse(); output::init_color(cli.color); - let current_dir = env::current_dir()?; + let current_dir = std::env::current_dir()?; match &cli.command { Commands::Build(args) => { let (build_args, debug_symbols, kind, include_orig, run_test) = match &args.target { @@ -81,7 +83,6 @@ async fn run() -> anyhow::Result { clean: build_args.clean, source_sync: build_args.source_sync, host_arch_variant: build_args.host_arch_variant.clone(), - shell_on_failure: build_args.shell_on_failure, driver_overrides: build_args.driver_overrides(), }, distro: build_args.distro.clone(), @@ -92,13 +93,41 @@ async fn run() -> anyhow::Result { }) .await?; } - Commands::Shell(args) => { - build::shell( - ¤t_dir, - args.common.source_dir.as_deref(), - cli.config.as_deref(), - )?; - } + Commands::Environment(args) => match &args.command { + EnvCommands::List => { + let config = config::Config::load(None, cli.config.as_deref())?; + env::list_environments(&config).await?; + } + EnvCommands::Clean => { + let config = config::Config::load(None, cli.config.as_deref())?; + env::clean_environments(&config).await?; + } + EnvCommands::Rm { ids, force } => { + let config = config::Config::load(None, cli.config.as_deref())?; + env::remove_environments(&config, ids, *force).await?; + } + EnvCommands::Shell { id, common } => { + if let Some(id) = id { + let config = config::Config::load(None, cli.config.as_deref())?; + env::shell_environment(&config, ¤t_dir, Some(id)).await?; + } else { + let source_dir = common.source_dir.as_deref().unwrap_or(¤t_dir); + let source_dir = match env::resolve_shell_source_dir( + Some(source_dir), + ¤t_dir, + ) { + Ok(dir) => dir, + Err(_) => { + anyhow::bail!( + "not a Source tree; pass an Environment id (see `debmagic env list`)" + ); + } + }; + let config = config::Config::load(Some(&source_dir), cli.config.as_deref())?; + env::shell_environment(&config, &source_dir, None).await?; + } + } + }, Commands::Test(args) => { let intent = test::resolve_test_intent(test::TestIntentInput { fallback_dir: current_dir.clone(), @@ -109,12 +138,13 @@ async fn run() -> anyhow::Result { strict: args.strict, changes: args.changes.clone(), allow_host_test: args.allow_host_test, - shell_on_failure: args.shell_on_failure, distro: args.distro.clone(), driver_overrides: args.driver_overrides(), })?; - let outcome = test::run_test(&intent).context("running tests failed")?; + let outcome = test::run_test(&intent) + .await + .context("running tests failed")?; return Ok(match outcome { test::TestOutcome::Passed => ExitCode::SUCCESS, test::TestOutcome::Failed => ExitCode::from(1), diff --git a/packages/debmagic/src/migrations/0001_registry.sql b/packages/debmagic/src/migrations/0001_registry.sql new file mode 100644 index 00000000..527cefcd --- /dev/null +++ b/packages/debmagic/src/migrations/0001_registry.sql @@ -0,0 +1,55 @@ +-- SQLite has no enums: each enum column checks the values of the Rust +-- conversion named above it. + +create table environments ( + -- Environment::id + id text primary key, + -- DriverType::as_str / DriverType::from_name + driver text not null check (driver in ('docker', 'bare', 'lxd', 'incus')), + package_name text not null, + package_identifier text not null, + source_dir text not null, + root_dir text not null, + -- Distro::as_str / Distro::parse, free-form for custom distros + distro_family text not null, + distro_codename text not null, + distro_version text not null, + distro_is_devel integer not null check (distro_is_devel in (0, 1)), + -- Persistence::as_str / Persistence::from_name + persistence text not null check (persistence in ('no', 'on-failure', 'always')), + -- EnvironmentPurpose::as_str / EnvironmentPurpose::from_name + purpose text not null check (purpose in ('build', 'test')), + -- the command holding the Environment, see Process + owner_pid integer, + owner_pid_start integer, + destroying integer not null default 0 check (destroying in (0, 1)), + -- JSON object from Driver::driver_metadata + driver_metadata text not null default '{}' +) strict; + +-- `debmagic env shell` sessions, see Process +create table attachments ( + id integer primary key autoincrement, + environment_id text not null references environments(id) on delete cascade, + pid integer not null, + pid_start integer +) strict; + +create table invocations ( + id integer primary key autoincrement, + -- InvocationKind::as_str / InvocationKind::from_name + kind text not null check (kind in ('binary_build', 'source_build', 'test_run')), + source_dir text not null, + package_name text not null, + package_version text not null, + distro_family text not null, + distro_codename text not null, + distro_version text not null, + distro_is_devel integer not null check (distro_is_devel in (0, 1)), + driver text check (driver in ('docker', 'bare', 'lxd', 'incus')), + success integer not null check (success in (0, 1)), + changes_path text, + environment_id text references environments(id) on delete set null, + -- unix time + created_at integer not null +) strict; diff --git a/packages/debmagic/src/package.rs b/packages/debmagic/src/package.rs index 9635d682..267247ee 100644 --- a/packages/debmagic/src/package.rs +++ b/packages/debmagic/src/package.rs @@ -157,7 +157,9 @@ fn custom_distro_from_base_images( } } -fn lookup_distro(name: &str, mode: DistroResolveMode<'_>) -> anyhow::Result { +/// Resolve a distro codename (built-in or a custom suite from the Driver's +/// `base_images` map) to a full [`DistroVersion`]. +pub fn lookup_distro(name: &str, mode: DistroResolveMode<'_>) -> anyhow::Result { if let DistroResolveMode::Container { base_images, .. } = mode { validate_base_images_keys(base_images.keys().map(|k| k.as_str()))?; } diff --git a/packages/debmagic/src/test/intent.rs b/packages/debmagic/src/test/intent.rs index 1221a967..cbb5c139 100644 --- a/packages/debmagic/src/test/intent.rs +++ b/packages/debmagic/src/test/intent.rs @@ -4,7 +4,7 @@ use anyhow::Context; use crate::{ config::Config, - driver::{DriverType, config::DriverOverrides}, + driver::{DriverType, Persistence, config::DriverOverrides}, }; /// Inputs for resolving a [`TestIntent`]. @@ -15,11 +15,10 @@ pub struct TestIntentInput { pub source_dir: Option, pub config_file: Option, pub driver: Option, - pub persistent: Option, + pub persistent: Option, pub strict: bool, pub changes: Option, pub allow_host_test: bool, - pub shell_on_failure: Option, pub distro: Option, pub driver_overrides: DriverOverrides, } @@ -34,14 +33,15 @@ pub struct TestIntent { pub strict: bool, pub changes: Option, pub allow_host_test: bool, - pub shell_on_failure: bool, pub distro: Option, pub config: Config, pub driver_overrides: DriverOverrides, } pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result { - let source_dir = std::path::absolute(input.source_dir.unwrap_or(input.fallback_dir)) + // Canonicalized: Environment ids and registry lookups key on the + // source_dir string, so all entry points must agree on one spelling. + let source_dir = std::fs::canonicalize(input.source_dir.unwrap_or(input.fallback_dir)) .context("resolving source dir failed")?; let mut config = Config::load(Some(&source_dir), input.config_file.as_deref())?; @@ -56,15 +56,12 @@ pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result None }; - let shell_on_failure = input.shell_on_failure.unwrap_or(config.shell_on_failure); - Ok(TestIntent { source_dir, driver: input.driver, strict: input.strict, changes, allow_host_test: input.allow_host_test, - shell_on_failure, distro: input.distro, config, driver_overrides: input.driver_overrides, @@ -96,7 +93,6 @@ mod tests { strict: false, changes: None, allow_host_test: false, - shell_on_failure: None, distro: None, driver_overrides: DriverOverrides { apt_mirror: None, @@ -116,32 +112,10 @@ mod tests { fn resolve_applies_persistent_override() -> anyhow::Result<()> { let dir = std::env::temp_dir(); let mut input = base_input(dir); - input.persistent = Some(false); - - let intent = resolve_test_intent(input)?; - assert!(!intent.config.driver.persistent); - Ok(()) - } - - #[test] - fn resolve_passes_through_strict() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.strict = true; - - let intent = resolve_test_intent(input)?; - assert!(intent.strict); - Ok(()) - } - - #[test] - fn resolve_passes_through_shell_on_failure() -> anyhow::Result<()> { - let dir = std::env::temp_dir(); - let mut input = base_input(dir); - input.shell_on_failure = Some(true); + input.persistent = Some(Persistence::No); let intent = resolve_test_intent(input)?; - assert!(intent.shell_on_failure); + assert_eq!(intent.config.driver.persistent, Persistence::No); Ok(()) } @@ -150,7 +124,7 @@ mod tests { let dir = std::env::temp_dir(); let intent = resolve_test_intent(base_input(dir.clone()))?; assert!(intent.source_dir.is_absolute()); - assert_eq!(intent.source_dir, std::path::absolute(&dir)?); + assert_eq!(intent.source_dir, std::fs::canonicalize(&dir)?); Ok(()) } } diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index 9a33a019..c2ca0f8b 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -1,23 +1,23 @@ use std::{ fs, io, - io::{BufReader, IsTerminal, stdout}, path::{Path, PathBuf}, }; use super::intent::TestIntent; -use crate::build::artifacts::{self, copy_changes_artifacts, copy_dir_all}; -use crate::build::source::stage_dir; -use crate::build::source::stage_source_tree; +use crate::build::artifacts::{copy_changes_artifacts, copy_dir_all}; +use crate::build::source::{stage_dir, stage_source_tree}; use crate::build_intent::BuildKind; -use crate::driver::{ - Driver, DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, EnvironmentPurpose, - IsolationCapability, - config::{DriverConfig, DriverOverrides}, - create_driver, remove_environment_root, +use crate::driver::{DriverType, Environment, EnvironmentDriver, EnvironmentPurpose}; +use crate::driver::{IsolationCapability, config::DriverConfig}; +use crate::environment::{ + CommandCompletion, HostRootPolicy, InvocationKind, RecordedInvocation, Registry, claim_command, + prior_binary_invocation, }; +use crate::package::distro_resolve_mode_for_driver; use crate::package::load_package; use crate::subprocess::Capture; use anyhow::{Context, anyhow, bail}; +use debian_control::lossless::changes::Changes; use debmagic_common::distro::DistroVersion; use debmagic_common::package::SourcePackage; @@ -38,28 +38,6 @@ pub enum TestOutcome { StrictFailure, } -struct TestRun { - environment: Environment, - driver: Driver, -} - -fn get_build_root_and_identifier( - temp_build_dir: &Path, - package: &SourcePackage, -) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", package.name(), package.version()); - let build_root = temp_build_dir.join(&package_identifier); - (package_identifier, build_root) -} - -fn test_build_root(build_root: &Path) -> PathBuf { - let package_identifier = build_root - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - build_root.with_file_name(format!("{package_identifier}-test")) -} - fn map_autopkgtest_exit(exit_code: i32, strict: bool) -> TestOutcome { if exit_code == AUTOPKGTEST_EXIT_PASS { return TestOutcome::Passed; @@ -85,99 +63,95 @@ fn map_autopkgtest_exit(exit_code: i32, strict: bool) -> TestOutcome { TestOutcome::Passed } -fn lookup_distro(name: &str) -> anyhow::Result { - debmagic_common::distro::get_distro_version(name) - .ok_or_else(|| anyhow!("unknown distro codename '{name}'")) +/// Resolve an explicit `--distro` override like a build would: built-in +/// suites, plus custom suites declared in the chosen Driver's `base_images`. +fn lookup_distro_override( + name: &str, + driver: DriverType, + driver_config: &DriverConfig, +) -> anyhow::Result { + let mode = distro_resolve_mode_for_driver( + driver, + &driver_config.docker.base_images, + &driver_config.lxd.base_images, + ); + crate::package::lookup_distro(name, mode) } -fn load_environment_metadata(root: &Path) -> anyhow::Result { - let metadata_path = root.join("environment.json"); - if !metadata_path.is_file() { - bail!("No environment.json found"); - } - let file = fs::OpenOptions::new().read(true).open(&metadata_path)?; - let reader = BufReader::new(&file); - serde_json::from_reader(reader).with_context(|| { - format!( - "Failed to read environment metadata from {} - invalid json", - metadata_path.display() - ) +/// Whether any suite in a `.changes` `Distribution` field (space-separated, +/// codenames or aliases like `unstable`) names `distro`. +fn distribution_matches(distribution: &str, distro: &DistroVersion) -> bool { + distribution.split_whitespace().any(|suite| { + suite == distro.codename + || debmagic_common::distro::get_distro_version(suite) + .is_some_and(|resolved| resolved.codename == distro.codename) }) } -impl TestRun { - fn create( - environment: &Environment, - driver_config: &DriverConfig, - driver_overrides: &DriverOverrides, - ) -> anyhow::Result { - let driver = create_driver(environment, driver_config, driver_overrides) - .context(format!("failed to create {:?} driver", environment.driver))?; - Ok(Self { - environment: environment.clone(), - driver, - }) - } - - fn write_metadata(&self) -> anyhow::Result<()> { - let metadata = EnvironmentMetadata { - environment: self.environment.clone(), - driver_metadata: self.driver.driver_metadata(), - }; - let path = self.environment.root_dir.join("environment.json"); - let json = serde_json::to_string_pretty(&metadata) - .context("Failed to serialize environment metadata")?; - fs::write(path, json)?; - Ok(()) - } -} - -fn prepare_test_env( +/// Point out where the prior binary-build Invocation that supplies the +/// defaults disagrees with what is about to be tested. +fn warn_on_prior_build_mismatch( intent: &TestIntent, - environment: &Environment, package: &SourcePackage, + invocation: &RecordedInvocation, changes_path: &Path, -) -> anyhow::Result { - let test_root = &environment.root_dir; - - if intent.config.driver.persistent && test_root.exists() { - let test_run = - TestRun::create(environment, &intent.config.driver, &intent.driver_overrides) - .context(format!("failed to create {:?} driver", environment.driver))?; - test_run - .driver - .reset_root() - .context("failed to reset persistent test directory")?; - environment - .create_dirs() - .context("failed to create test directories")?; - stage_source_tree( - environment, - BuildKind::Binary, - package, - intent.config.source_sync_mode, - false, - )?; - copy_changes_artifacts(changes_path, &environment.work_dir())?; - return Ok(test_run); +) { + let current_version = package.version().to_string(); + if intent.changes.is_none() && invocation.package_version != current_version { + eprintln!( + "Warning: testing {} {} from the last successful binary build; \ + debian/changelog is now at {current_version}", + package.name(), + invocation.package_version + ); } + if intent.changes.is_some() + && intent.distro.is_none() + && let Ok(changes) = Changes::from_file(changes_path) + && let Some(distribution) = changes.distribution() + && !distribution_matches(&distribution, &invocation.distro) + { + eprintln!( + "Warning: {} targets {distribution}, but the test defaults to {} from the \ + prior binary build; pass --distro to override", + changes_path.display(), + invocation.distro.codename + ); + } +} - remove_environment_root(test_root, &intent.config.driver)?; +/// Written into every exported test directory; only directories carrying it +/// are ever replaced, so an unrelated folder next to the `.changes` survives. +const TEST_OUTPUT_MARKER: &str = ".debmagic-test-output"; + +/// `/.test` — named after the `.changes` so it cannot +/// clash with a generic folder (e.g. a package's own `test/`) when the +/// output directory is the source tree or its parent. +fn exported_test_dir(changes_path: &Path) -> anyhow::Result { + let stem = changes_path + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let parent = changes_path.parent().unwrap_or_else(|| Path::new(".")); + Ok(parent.join(format!("{stem}.test"))) +} - environment - .create_dirs() - .context("failed to create test directories")?; - stage_source_tree( - environment, - BuildKind::Binary, - package, - intent.config.source_sync_mode, - false, - )?; - copy_changes_artifacts(changes_path, &environment.work_dir())?; +fn ensure_exported_test_dir_replaceable(dir: &Path) -> anyhow::Result<()> { + if dir.exists() && !dir.join(TEST_OUTPUT_MARKER).is_file() { + bail!( + "{} exists but was not written by debmagic; move it away and retry", + dir.display() + ); + } + Ok(()) +} - let test_run = TestRun::create(environment, &intent.config.driver, &intent.driver_overrides)?; - Ok(test_run) +fn replace_exported_test_dir(dir: &Path) -> anyhow::Result<()> { + ensure_exported_test_dir_replaceable(dir)?; + if !dir.exists() { + return Ok(()); + } + fs::remove_dir_all(dir).with_context(|| format!("failed to remove {}", dir.display())) } fn print_autopkgtest_notices(exit_code: i32, summary_path: &Path) { @@ -228,10 +202,24 @@ fn autopkgtest_isolation_args(isolation: IsolationCapability) -> Vec<&'static st } } -pub fn run_test(intent: &TestIntent) -> anyhow::Result { +pub async fn run_test(intent: &TestIntent) -> anyhow::Result { let package = load_package(&intent.source_dir)?; - let (package_identifier, build_root) = - get_build_root_and_identifier(&intent.config.temp_build_dir, &package); + let registry = Registry::open_default_or_ephemeral().await?; + + // `--changes` overrides only the artifact path. Driver and distro still + // default to the prior binary-build Invocation when one exists. + let invocations = registry + .binary_build_changes_for_tree(&intent.source_dir) + .await + .unwrap_or_else(|error| { + eprintln!("Warning: cannot read prior binary builds from the registry: {error}"); + Vec::new() + }); + let prior_invocation = prior_binary_invocation( + &invocations, + intent.distro.as_deref(), + intent.changes.is_some(), + )?; let changes_path = if let Some(ref explicit) = intent.changes { if !explicit.is_file() { @@ -239,27 +227,26 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { } explicit.clone() } else { - let metadata_path = build_root.join("environment.json"); - if !metadata_path.is_file() { - bail!( - "no prior build found at {}; run `debmagic build binary` first", - build_root.display() - ); - } - // tests install built binaries, so a binary build's .changes is what - // to export — not a leftover source one from the same persistent dir - artifacts::find_changes_file(&build_root.join("work"), BuildKind::Binary)? - }; - - let prior_build = if build_root.join("environment.json").is_file() { - Some(load_environment_metadata(&build_root)?) - } else { - None + prior_invocation + .as_ref() + .and_then(|invocation| invocation.changes_path.clone()) + .context( + "no prior binary build found; run `debmagic build binary` first or pass --changes", + )? }; + if let Some(invocation) = prior_invocation.as_ref() { + warn_on_prior_build_mismatch(intent, &package, invocation, &changes_path); + } + let exported_test_dir = exported_test_dir(&changes_path)?; + ensure_exported_test_dir_replaceable(&exported_test_dir)?; let driver = intent .driver - .or_else(|| prior_build.as_ref().map(|metadata| metadata.environment.driver)) + .or_else(|| { + prior_invocation + .as_ref() + .and_then(|invocation| invocation.driver) + }) .ok_or_else(|| { anyhow!( "no driver specified and no prior build found; pass --driver or run `debmagic build binary` first" @@ -274,159 +261,193 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { } let distro = if let Some(ref override_distro) = intent.distro { - lookup_distro(override_distro)? - } else if let Some(ref metadata) = prior_build { - metadata.environment.distro.clone() + lookup_distro_override(override_distro, driver, &intent.config.driver)? + } else if let Some(ref invocation) = prior_invocation { + // The Invocation recorded the full DistroVersion, so custom suites + // from a Driver's base_images work without re-resolving. + invocation.distro.clone() } else { bail!( - "no prior build metadata found; pass --distro when using --changes without a build root" + "no prior build metadata found; pass --distro when using --changes without a prior Invocation" ); }; - let test_root = test_build_root(&build_root); - - let environment = Environment { + let environment = Environment::new( driver, - package_name: package.name().to_string(), - package_identifier, - root_dir: test_root.clone(), + package.name(), + &format!("{}-{}", package.name(), package.version()), + &intent.source_dir, distro, - persistent: intent.config.driver.persistent, - purpose: EnvironmentPurpose::Test, - }; - - let test_run = prepare_test_env(intent, &environment, &package, &changes_path) - .context("failed to prepare test environment")?; - test_run - .write_metadata() - .context("failed to write test metadata")?; - - crate::output::stage("Installing autopkgtest"); - let staged_source_dir = stage_dir(&environment, BuildKind::Binary); - test_run.driver.run_command_checked( - &["apt-get", "install", "-y", "autopkgtest"], - &staged_source_dir, - true, - &[("DEBIAN_FRONTEND", "noninteractive")], - )?; - - let work_dir = environment.work_dir(); - let changes_filename = changes_path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; - let source_tree_name = staged_source_dir - .file_name() - .and_then(|name| name.to_str()) - .context("stage dir has no file name")?; - let autopkgtest_out_host = test_root.join("autopkgtest-out"); - if autopkgtest_out_host.exists() { - fs::remove_dir_all(&autopkgtest_out_host)?; - } - let output_dir_arg = "../autopkgtest-out"; - let summary_arg = "../autopkgtest-out/summary"; - - // Binary-only builds have no .dsc in the .changes; pass the staged source - // tree alongside the .changes so debian/tests/ is found without rebuilding - // (-B). See autopkgtest(1) "TESTING A DEBIAN PACKAGE" (.changes + tree). - // IsolationCapabilities become `--ignore-restrictions` (autopkgtest args, - // not virt-null `--fake-capability`; see autopkgtest_isolation_args). - let output_dir_flag = format!("--output-dir={output_dir_arg}"); - let summary_flag = format!("--summary={summary_arg}"); - let source_tree_arg = format!("{source_tree_name}/"); - let isolation_args = autopkgtest_isolation_args(test_run.driver.isolation_capability()); - let mut autopkgtest_cmd = vec!["autopkgtest", "-B", "--no-auto-control"]; - autopkgtest_cmd.extend(isolation_args); - autopkgtest_cmd.extend([ - output_dir_flag.as_str(), - summary_flag.as_str(), - changes_filename, - source_tree_arg.as_str(), - "--", - "null", - ]); - - crate::output::stage(&format!("Running autopkgtest for {}", package.name())); - let exit_code = test_run - .driver - .run_command(&autopkgtest_cmd, &work_dir, true, &[], Capture::NONE) - .map(|result| result.exit_code) - .unwrap_or(-1); - - let summary_path = autopkgtest_out_host.join("summary"); - print_autopkgtest_notices(exit_code, &summary_path); - - let export_root = prior_build - .as_ref() - .map(|_| build_root.as_path()) - .unwrap_or_else(|| changes_path.parent().unwrap()); - let exported_test_dir = export_root.join("test"); - if exported_test_dir.exists() { - fs::remove_dir_all(&exported_test_dir)?; - } - copy_dir_all(&autopkgtest_out_host, &exported_test_dir).with_context(|| { - format!( - "failed to copy test output to {}", - exported_test_dir.display() - ) - })?; - println!("Test output written to {}", exported_test_dir.display()); - - let outcome = map_autopkgtest_exit(exit_code, intent.strict); - - if outcome == TestOutcome::Failed { - eprintln!("Tests failed (autopkgtest exit code {exit_code})."); - eprintln!("Test logs: {}", exported_test_dir.display()); - if intent.shell_on_failure && stdout().is_terminal() { - eprintln!("Dropping into shell..."); - if let Err(shell_error) = test_run.driver.interactive_shell(&staged_source_dir) { - eprintln!("Dropping into shell failed: {shell_error}"); + intent.config.driver.persistent, + EnvironmentPurpose::Test, + &intent.config.environments_dir, + ); + let version = package.version().to_string(); + + claim_command( + ®istry, + environment, + &intent.config.driver, + &intent.driver_overrides, + HostRootPolicy::Reset, + |environment, _| { + stage_source_tree( + environment, + BuildKind::Binary, + &package, + intent.config.source_sync_mode, + false, + )?; + copy_changes_artifacts(&changes_path, &environment.work_dir()) + }, + InvocationKind::Test, + &version, + |claimed| { + crate::output::stage("Installing autopkgtest"); + let staged_source_dir = stage_dir(claimed.environment(), BuildKind::Binary); + claimed.driver().run_command_checked( + &["apt-get", "install", "-y", "autopkgtest"], + &staged_source_dir, + true, + &[("DEBIAN_FRONTEND", "noninteractive")], + )?; + + let work_dir = claimed.environment().work_dir(); + let changes_filename = changes_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let source_tree_name = staged_source_dir + .file_name() + .and_then(|name| name.to_str()) + .context("stage dir has no file name")?; + let autopkgtest_out_host = claimed.environment().root_dir.join("autopkgtest-out"); + if autopkgtest_out_host.exists() { + fs::remove_dir_all(&autopkgtest_out_host)?; } - } else if intent.shell_on_failure { - eprintln!( - "--shell-on-failure is set but stdout is not a TTY; skipping interactive shell" - ); - } else { - eprintln!("Re-run with --shell-on-failure to inspect the test environment"); - } - } - - if !environment.persistent { - // Clear container-owned files from the bind mount before destroying - // the container; otherwise the host user cannot remove them later. - if let Err(e) = test_run.driver.reset_root() { - eprintln!("Warning: failed to reset test root before cleanup: {e}"); - } - } - - if let Err(cleanup_error) = test_run.driver.cleanup() { - eprintln!("Failed to clean up test environment: {cleanup_error}"); - } - - Ok(outcome) + let output_dir_arg = "../autopkgtest-out"; + let summary_arg = "../autopkgtest-out/summary"; + + // Binary-only builds have no .dsc in the .changes; pass the staged source + // tree alongside the .changes so debian/tests/ is found without rebuilding + // (-B). See autopkgtest(1) "TESTING A DEBIAN PACKAGE" (.changes + tree). + // IsolationCapabilities become `--ignore-restrictions` (autopkgtest args, + // not virt-null `--fake-capability`; see autopkgtest_isolation_args). + let output_dir_flag = format!("--output-dir={output_dir_arg}"); + let summary_flag = format!("--summary={summary_arg}"); + let source_tree_arg = format!("{source_tree_name}/"); + let isolation_args = + autopkgtest_isolation_args(claimed.driver().isolation_capability()); + let mut autopkgtest_cmd = vec!["autopkgtest", "-B", "--no-auto-control"]; + autopkgtest_cmd.extend(isolation_args); + autopkgtest_cmd.extend([ + output_dir_flag.as_str(), + summary_flag.as_str(), + changes_filename, + source_tree_arg.as_str(), + "--", + "null", + ]); + + crate::output::stage(&format!("Running autopkgtest for {}", package.name())); + let exit_code = claimed + .driver() + .run_command(&autopkgtest_cmd, &work_dir, true, &[], Capture::NONE) + .map(|result| result.exit_code) + .unwrap_or(-1); + + let summary_path = autopkgtest_out_host.join("summary"); + print_autopkgtest_notices(exit_code, &summary_path); + + replace_exported_test_dir(&exported_test_dir)?; + copy_dir_all(&autopkgtest_out_host, &exported_test_dir).with_context(|| { + format!( + "failed to copy test output to {}", + exported_test_dir.display() + ) + })?; + fs::write(exported_test_dir.join(TEST_OUTPUT_MARKER), "")?; + println!("Test output written to {}", exported_test_dir.display()); + + let outcome = map_autopkgtest_exit(exit_code, intent.strict); + let failure_lines = if outcome == TestOutcome::Failed { + vec![ + format!("Tests failed (autopkgtest exit code {exit_code})."), + format!("Test logs: {}", exported_test_dir.display()), + ] + } else { + Vec::new() + }; + Ok(CommandCompletion { + value: Ok(outcome), + success: outcome == TestOutcome::Passed, + shell_worthy: outcome == TestOutcome::Failed, + finish_error_fails_command: false, + changes_path: None, + failure_lines, + }) + }, + ) + .await } #[cfg(test)] mod tests { use super::*; + use crate::driver::Persistence; - fn sample_package() -> SourcePackage { - SourcePackage::from_files([( - "changelog", - "pkg (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", - )]) - .unwrap() + #[test] + fn exported_test_dir_is_named_after_the_changes_file() -> anyhow::Result<()> { + assert_eq!( + exported_test_dir(Path::new("/out/pkg_1.0-1_amd64.changes"))?, + PathBuf::from("/out/pkg_1.0-1_amd64.test") + ); + Ok(()) } #[test] - fn test_build_root_appends_test_suffix() { - let (_, build_root) = - get_build_root_and_identifier(Path::new("/tmp/debmagic"), &sample_package()); - assert_eq!(build_root, PathBuf::from("/tmp/debmagic/pkg-1.0-1")); - assert_eq!( - test_build_root(&build_root), - PathBuf::from("/tmp/debmagic/pkg-1.0-1-test") + fn foreign_test_dir_is_never_removed() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-test-{}", uuid::Uuid::new_v4())); + let foreign = dir.join("pkg.test"); + fs::create_dir_all(&foreign)?; + fs::write(foreign.join("keep"), "user data")?; + assert!(replace_exported_test_dir(&foreign).is_err()); + assert!(foreign.join("keep").is_file()); + + fs::write(foreign.join(TEST_OUTPUT_MARKER), "")?; + replace_exported_test_dir(&foreign)?; + assert!(!foreign.exists()); + fs::remove_dir_all(&dir)?; + Ok(()) + } + + #[test] + fn test_environment_id_differs_from_build() { + let distro = DistroVersion::new(debmagic_common::distro::Distro::Debian, "trixie", "13"); + let envs = PathBuf::from("/envs"); + let source_dir = PathBuf::from("/src"); + let build = Environment::new( + DriverType::Docker, + "pkg", + "pkg-1.0-1", + &source_dir, + distro.clone(), + Persistence::Always, + EnvironmentPurpose::Build, + &envs, + ); + let test = Environment::new( + DriverType::Docker, + "pkg", + "pkg-1.0-1", + &source_dir, + distro, + Persistence::Always, + EnvironmentPurpose::Test, + &envs, ); + assert_ne!(build.id(), test.id()); + assert_ne!(build.root_dir, test.root_dir); } #[test] diff --git a/packages/debmagic/tests/assets/config1.toml b/packages/debmagic/tests/assets/config1.toml index 542e4aca..e4af8717 100644 --- a/packages/debmagic/tests/assets/config1.toml +++ b/packages/debmagic/tests/assets/config1.toml @@ -1,6 +1,6 @@ [driver] default = "docker" -persistent = true +persistent = "always" [driver.docker] base_images = { "debian:trixie" = "some-debian-trixie-image:latest" }