From 311103c81e333396a4175c696aed969c29d0c450 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 10:39:48 +0000 Subject: [PATCH 01/43] Add plan for running smoke tests against a built Quarto version Analysis of the current test harness (in-process quarto() invocation, json-stream log capture, smoke-all/_quarto.tests dispatch, ff-matrix bucket routing, release CI artifacts) plus a proposal: - Add a subprocess 'binary mode' seam (QUARTO_TEST_BIN) in tests/test.ts reusing the existing --log/--log-format json-stream contract - Pair built binaries with a repo checkout at the matching v tag to avoid harness/binary version skew - Parameterize test-smokes.yml (dev | release | artifact install modes) - Mode A: scheduled workflow testing the GitHub prerelease - Mode B: smoke a built artifact inside create-release.yml - Phased roadmap starting with the feature-format-matrix bucket Co-Authored-By: Claude Fable 5 --- dev-docs/smoke-tests-built-version-plan.md | 314 +++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 dev-docs/smoke-tests-built-version-plan.md diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md new file mode 100644 index 0000000000..f5acb85dff --- /dev/null +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -0,0 +1,314 @@ +# Plan: Running Smoke Tests Against a Built Quarto + +Status: **proposal / plan** — no implementation yet. + +Goal: be able to run the existing smoke test suites — at minimum `smoke-all` +and the feature-format matrix — against a **built** Quarto (a GitHub +prerelease/release, or an installer artifact produced earlier in the same CI +run), instead of only the dev source tree. + +--- + +## 1. How testing works today (findings) + +### 1.1 The harness runs Quarto in-process + +- `testQuartoCmd` does **not** spawn a `quarto` binary. It imports the CLI + entry point and calls it as a function inside the Deno test process: + - `tests/test.ts:13` — `import { quarto } from "../src/quarto.ts";` + - `tests/test.ts:159-162` — `await Promise.race([quarto([cmd, ...args], undefined, context?.env), timeout])` +- `tests/run-tests.sh` launches the repo-bundled Deno with + `--importmap=src/import_map.json`, `--allow-all`, `--unstable-kv --unstable-ffi` + (`run-tests.sh:57,63,164`) and exports source-tree paths: + `QUARTO_BIN_PATH=package/dist/bin` (`:47`), + `QUARTO_SHARE_PATH=src/resources` (`:60`), `QUARTO_DEBUG=true` (`:61`). +- The only subprocess seam today is `quartoDevCmd()` (`tests/utils.ts:244`, + returns `"quarto"`/`"quarto.cmd"` from `PATH`), used by a handful of + passthrough/Playwright tests — not by the render harness. + +### 1.2 Output capture and verification are already execution-agnostic + +- Before each test, the harness redirects Quarto's logger to a temp file in + `json-stream` format (`tests/test.ts:243-249`). After execution it parses + the file line-by-line into `ExecuteOutput { msg, level, levelName }` + (`test.ts:176-180, 386-392`) and hands that array to every verifier. +- `noErrors` / `noErrorsOrWarnings` / `shouldError` / `printsMessage` only + inspect those log records (`tests/verify.ts:107-205`). +- Errors thrown by `execute()` are caught and converted into ERROR log + records before verification (`test.ts:262-266`), so verifiers never depend + on exceptions from the in-process call. +- All file-content verifiers (`ensureHtmlElements`, `ensureFileRegexMatches`, + `ensureLatexFileRegexMatches`, docx/pptx/pdf/jats/odt checks, snapshots…) + read produced files off disk and do not care how Quarto ran. +- Crucially, the **CLI already exposes the same logging pipeline as flags**: + `--log --log-format json-stream --log-level info` + (`src/core/log.ts:28,66-67,116,447`). A subprocess can therefore produce a + byte-compatible log file for the existing verifiers. + +### 1.3 smoke-all specifics + +- `tests/smoke/smoke-all.test.ts` globs `docs/smoke-all/**/*.{md,qmd,ipynb}` + (`:378-386`), parses `_quarto.tests` metadata with **dev-source code** + (`readYamlFromMarkdown` from `src/core/yaml.ts`, `jupyterNotebookToMarkdown` + from `src/command/convert/jupyter.ts`, `:399-401`), maps spec keys to + verifiers via an inline `verifyMap` (`:189-218`), and calls + `testQuartoCmd("render", [input, "--to", format], ...)` (`:462`). It also + calls `quarto(["render", projectPath])` directly for project pre-renders + (`:434`). +- `tests/utils.ts` (`outputForInput`, `findProjectOutputDir`, …) re-implements + Quarto's output-path rules in harness code (`utils.ts:100-208`). +- The YAML-intelligence bootstrap (`initYamlIntelligenceResourcesFromFilesystem`, + `setInitializer`/`initState`, `smoke-all.test.ts:64-66,463-466`) exists only + because Quarto runs in-process. + +### 1.4 Feature-format matrix = a smoke-all bucket + +- `.github/workflows/test-ff-matrix.yml:44-50` reuses `test-smokes.yml` with + `buckets: '[ "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" ]'`. + The bucket loop calls `./run-tests.sh "$file"` which routes `.qmd` args to + `smoke-all.test.ts` (`run-tests.sh:132-164`). The 0/1/2 quality ratings are + display-only (`create_table.py`); CI enforces only the `_quarto.tests` specs. +- So: **anything that makes smoke-all work against a built binary makes the + feature-format matrix work too.** + +### 1.5 CI today + +- `test-smokes.yml` sets up all language deps inline (R/renv, Python/uv, + Julia, TinyTeX, Playwright, …) and then installs the **dev tree** via the + `quarto-dev` composite action (`configure.sh`; asserts version `99.9.9` — + `.github/workflows/actions/quarto-dev/action.yml:45-51`). +- `create-release.yml` runs nightly, builds per-OS artifacts + (`quarto--linux-amd64.tar.gz`, msi/zip, pkg/tar.gz, deb/rpm), uploads + them as **workflow artifacts**, publishes them as **GitHub Release assets** + (prerelease by default), and tags `v` on the exact commit built + (`create-release.yml:88-100,678-703`). +- The release pipeline already has minimal "test the built artifact" jobs + (`test-tarball-linux`, `test-zip-win`, `test-zip-mac`): download artifact, + extract, `quarto check` / `--version` (`create-release.yml:310-348,431-477,548-585`). + These are natural extension points. +- `test-smokes-parallel.yml` already uses `quarto-dev/quarto-actions/setup@v2` + with `version: pre-release` — but only to compute test buckets, not to run + tests (`test-smokes-parallel.yml:59-75`). + +--- + +## 2. Strategy + +Two structural insights drive the design: + +1. **The coupling to the dev tree is concentrated in one seam** — the body of + `test.execute()` (the in-process `quarto()` call plus programmatic logger + init). The entire verifier layer, the `_quarto.tests` dispatch, and the + failure-reporting machinery consume only (a) the json-stream log file and + (b) files on disk. Both can be produced identically by a subprocess using + `--log/--log-format/--log-level`. + +2. **Version skew is avoidable by construction.** Every published (pre)release + is tagged `v` on the commit it was built from. If CI checks out + the repo **at the tag matching the binary under test**, the harness code, + test documents, schemas, and `utils.ts` path-derivation logic match the + binary exactly. The harness may keep importing `src/` for *parsing and + path math* — only *execution* moves to the binary. This keeps the change + small and low-risk. (Full decoupling via `quarto inspect` is a later, + optional phase — see Phase 4.) + +So the recommended architecture is: **add a "binary mode" to the existing +harness (subprocess execution behind an env var), keep everything else as-is, +and always pair a built binary with the repo checkout at its own tag.** + +--- + +## 3. Design: binary mode in the harness + +### 3.1 The execution seam + +Introduce `QUARTO_TEST_BIN` (absolute path to a built `quarto` / +`quarto.cmd`). In `tests/test.ts`: + +- When unset → current behavior (in-process `quarto()`), zero change for the + default dev workflow. +- When set → `test.execute()` spawns the binary via `Deno.Command`: + + ``` + \ + --log --log-format json-stream --log-level info + ``` + + - Reuse the temp log path already created at `test.ts:243`; skip (or + no-op) the in-process `initializeLogger` for the subprocess side, but + keep `logError(e)` able to append harness-side failures (spawn errors, + timeouts) as ERROR records so existing reporting keeps working. + - Do **not** throw on non-zero exit: the binary writes its own ERROR + records to the log, which is exactly what `noErrors`/`shouldError` + already consume. (Mirrors today's catch-and-log at `test.ts:262-266`.) + - Timeout: on expiry, kill the child process (an improvement over today, + where a timed-out in-process render keeps running). + - `context.env`: pass as subprocess env overlay instead of the + in-process env juggling in `quarto()` (`src/quarto.ts:163-217`). + - cwd: pass `Deno.cwd()` (the harness still does `Deno.chdir` for + `context.cwd`, so inheriting cwd is sufficient). + +- **Environment hygiene (critical):** the installed launcher *inherits* + `QUARTO_SHARE_PATH` if set (`package/scripts/common/quarto:102-108`), and + `run-tests.sh` exports it pointing at `src/resources`. In binary mode the + subprocess env must **drop/unset** `QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, + and `QUARTO_DEBUG` so the binary resolves its own installed layout. + +### 3.2 smoke-all adjustments + +- Route the direct project pre-render `quarto(["render", projectPath])` + (`smoke-all.test.ts:434`) through the same seam (small helper + `runQuarto(args)` used by both call sites). +- Metadata parsing, format guessing, and `verifyMap` dispatch stay unchanged + (they run in the harness process against the matching checkout — see §2.2). +- The YAML-intelligence bootstrap can stay; it is harness-process-only and + harmless in binary mode. + +### 3.3 Runner plumbing + +- `run-tests.sh` / `run-tests.ps1`: accept `--bin ` (or just honor an + exported `QUARTO_TEST_BIN`). When set: + - still resolve the repo Deno + import map to run the *harness* (a repo + checkout is always required — it holds the tests); + - skip exporting `QUARTO_SHARE_PATH` / `QUARTO_DEBUG`; + - print a banner with ` --version` so logs are unambiguous about what + was tested. + +### 3.4 Tests that can't run in binary mode + +Most `tests/smoke/**/*.test.ts` are pure `testQuartoCmd` and will just work. +A minority poke internals (unit tests, tests importing quarto APIs to compute +expectations, `quarto run` TS scripts relying on dev Deno, env-var tests). +Mechanism: + +- Add `TestContext.requiresDevQuarto?: boolean`; the `test()` wrapper sets + Deno's `ignore` when binary mode is active. +- Unit tests (`tests/unit/`) are dev-only by definition — excluded wholesale + in binary mode. +- Start by targeting only **smoke-all + feature-format matrix** (bucket + invocation), where this problem is near-zero, and grow coverage from there. + +--- + +## 4. CI integration + +Two complementary modes; both funnel through a parameterized +`test-smokes.yml`. + +### 4.0 Parameterize `test-smokes.yml` + +New `workflow_call` inputs: + +- `quarto-install`: `dev` (default) | `release` | `artifact` +- `quarto-version`: version/tag string (for `release`), e.g. `1.10.23` or + `pre-release` +- `quarto-artifact-name`: workflow artifact to download (for `artifact`) + +Setup step becomes conditional: + +- `dev` → existing `quarto-dev` composite action (unchanged default). +- `release` → `quarto-dev/quarto-actions/setup@v2` with the given version + (already used elsewhere in this repo: `test-smokes-parallel.yml:59-62`). +- `artifact` → `actions/download-artifact`, extract zip/tarball, add `bin` + to `PATH` (same recipe as `create-release.yml:310-348`). + +For `release`/`artifact`: skip the `99.9.9` dev assertion, instead assert +`quarto --version` equals the expected version, and export +`QUARTO_TEST_BIN=$(command -v quarto)` for the run-tests steps. All +language-dependency setup (R/Python/Julia/TinyTeX/…) is shared and unchanged. + +### 4.1 Mode A — scheduled run against the GitHub prerelease (recommended first) + +New workflow `test-smokes-built.yml`: + +1. Trigger: `workflow_dispatch` (inputs: version, bucket) + weekly `schedule`. + Optionally later: `release: [prereleased, released]` event for + run-on-every-nightly. +2. Resolve version: input, or "latest prerelease" via + `https://quarto.org/docs/download/_prerelease.json` / `gh release list`. +3. **Checkout `refs/tags/v`** → harness matches binary (§2.2). +4. Call `test-smokes.yml` with `quarto-install: release`, + `quarto-version: `, and + `buckets: '[ "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" ]'` + initially (expand to smoke-all buckets once stable). + +Pros: zero impact on the release pipeline's duration/reliability; tests the +exact bits users install (post signing/packaging); easy to backfill any +released version for bisecting. + +### 4.2 Mode B — artifact as prerequisite inside `create-release.yml` + +Extend the existing `test-tarball-linux` job (or add a sibling +`smoke-test-tarball-linux`) to call the parameterized `test-smokes.yml` with +`quarto-install: artifact`, `quarto-artifact-name: "Deb Zip"`. Same commit by +construction — no tag gymnastics needed. + +Recommendations to keep the nightly pipeline healthy: + +- Start with the **feature-format-matrix bucket only** (bounded, well-specified + suite) on **linux-amd64 only**. +- Make it **non-blocking** initially (`continue-on-error: true` + a visible + summary), promote to blocking once the skip-list has settled. +- Full smoke-all in the release pipeline is probably too slow/flaky for a + nightly release gate; keep the full suite in Mode A instead. + +--- + +## 5. Phased roadmap + +**Phase 0 — spike (≈1 day).** Hack the seam locally (no plumbing polish): +download the current prerelease tarball, set `QUARTO_TEST_BIN`, run ~20–30 +representative smoke-all docs + a couple of ff-matrix files. Catalog failure +classes (env leakage, path derivation drift, dev-only assumptions). This +validates the log-file contract end-to-end before investing in CI. + +**Phase 1 — harness binary mode.** Implement §3 properly: seam in `test.ts`, +`runQuarto` helper for smoke-all, env hygiene, `requiresDevQuarto`, +`run-tests.sh`/`.ps1` plumbing. Default behavior unchanged; add a small CI +sanity job (or manual checklist) that runs a tiny binary-mode bucket to keep +the mode from rotting. + +**Phase 2 — Mode A workflow.** Parameterize `test-smokes.yml` (§4.0), add +`test-smokes-built.yml` running the ff-matrix bucket weekly against the +latest prerelease, Linux first, then Windows. Triage failures into: real +product bugs (jackpot — this is the point of the project), harness +assumptions (fix), dev-only tests (skip-list). + +**Phase 3 — broaden.** Run full smoke-all buckets in Mode A; wire Mode B +(ff-matrix bucket, non-blocking) into `create-release.yml`; consider the +`release` event trigger so every nightly prerelease gets a built-version run +without touching the release pipeline itself. + +**Phase 4 — optional decoupling.** If testing binaries whose version diverges +from the harness checkout ever matters (e.g. running main's test suite +against last stable), replace the remaining `src/` imports in the smoke-all +driver: `quarto inspect` for metadata/format discovery, `@std/fs` glob, and +derive output paths from actual render results instead of `outputForInput`. +This is a bigger refactor with drift risk and is **not** needed for the +matched-tag strategy. + +--- + +## 6. Risks & open questions + +- **`QUARTO_DEBUG` behavior differences**: dev runs with `QUARTO_DEBUG=true`; + a built binary runs without it. Some tests may implicitly depend on debug + behaviors (stack traces in logs, error formatting). Phase 0 will surface + these. +- **Version-string assumptions**: anything expecting `99.9.9` (or dev share + paths) must be conditionalized (`quarto-dev` action assertion, any tests + matching version output). +- **Windows**: subprocess must invoke `quarto.cmd` (or use `cmd /c`); + quoting of args with spaces needs care in the `Deno.Command` seam. +- **Per-test subprocess startup cost** (~a few hundred ms × hundreds of + smoke-all docs) is real but small relative to render time; acceptable. +- **Parallel test files + env vars**: binary mode passes env per-subprocess, + which is actually *safer* than today's process-global `Deno.env` mutation. +- **Skip-list drift**: `requiresDevQuarto` annotations need a home in review + guidelines so new dev-only tests get tagged at authoring time. +- **Decision needed — cadence & blocking-ness**: weekly scheduled (Mode A) + vs. every nightly prerelease vs. inside `create-release.yml` (Mode B), and + whether failures should ever block publishing. Recommendation: Mode A + weekly non-blocking first; revisit after a month of signal. +- **Decision needed — scope**: start with ff-matrix bucket only (recommended) + or jump straight to full smoke-all. From dd0d0f0571ea00c05dd8bfc909e1ebbf27dcf83b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 10:49:07 +0000 Subject: [PATCH 02/43] Revise built-version test plan per maintainer decisions - Weekly schedule plus manual workflow_dispatch trigger - Scope is full smoke-all (ff-matrix bucket as optional extra) - Primary mode is preventive build-then-test: the workflow builds the linux-amd64 dist itself (same configure.sh + quarto-bld prepare-dist steps as create-release.yml make-tarball) and tests that artifact in the same run, so harness and binary share one commit - Published-(pre)release testing becomes a secondary dispatch input for Windows coverage and version backfill - create-release.yml gating deferred until the mode has a track record Co-Authored-By: Claude Fable 5 --- dev-docs/smoke-tests-built-version-plan.md | 146 +++++++++++++-------- 1 file changed, 92 insertions(+), 54 deletions(-) diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index f5acb85dff..72dd009104 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -192,8 +192,19 @@ Mechanism: ## 4. CI integration -Two complementary modes; both funnel through a parameterized -`test-smokes.yml`. +> **Decisions recorded (2026-07-17):** weekly schedule **plus manual +> `workflow_dispatch`**; scope is **full smoke-all** (ff-matrix bucket is a +> nice-to-have addition, not the target); and the primary goal is +> **preventive** — test artifacts *as built*, not only published prereleases +> (which is curative, after the fact). Since `create-release.yml` builds the +> linux tarball with just `./configure.sh` + `quarto-bld prepare-dist +> --set-version` + tar of `package/pkg-working` +> (`create-release.yml:124-159`), the test workflow can **build the artifact +> itself with the exact same steps** and test it in the same run — same +> commit by construction, no version-skew handling, and zero changes to the +> release pipeline. + +Modes below funnel through a parameterized `test-smokes.yml`. ### 4.0 Parameterize `test-smokes.yml` @@ -217,48 +228,68 @@ For `release`/`artifact`: skip the `99.9.9` dev assertion, instead assert `QUARTO_TEST_BIN=$(command -v quarto)` for the run-tests steps. All language-dependency setup (R/Python/Julia/TinyTeX/…) is shared and unchanged. -### 4.1 Mode A — scheduled run against the GitHub prerelease (recommended first) - -New workflow `test-smokes-built.yml`: - -1. Trigger: `workflow_dispatch` (inputs: version, bucket) + weekly `schedule`. - Optionally later: `release: [prereleased, released]` event for - run-on-every-nightly. -2. Resolve version: input, or "latest prerelease" via - `https://quarto.org/docs/download/_prerelease.json` / `gh release list`. -3. **Checkout `refs/tags/v`** → harness matches binary (§2.2). -4. Call `test-smokes.yml` with `quarto-install: release`, - `quarto-version: `, and - `buckets: '[ "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" ]'` - initially (expand to smoke-all buckets once stable). - -Pros: zero impact on the release pipeline's duration/reliability; tests the -exact bits users install (post signing/packaging); easy to backfill any -released version for bisecting. - -### 4.2 Mode B — artifact as prerequisite inside `create-release.yml` - -Extend the existing `test-tarball-linux` job (or add a sibling -`smoke-test-tarball-linux`) to call the parameterized `test-smokes.yml` with -`quarto-install: artifact`, `quarto-artifact-name: "Deb Zip"`. Same commit by -construction — no tag gymnastics needed. - -Recommendations to keep the nightly pipeline healthy: - -- Start with the **feature-format-matrix bucket only** (bounded, well-specified - suite) on **linux-amd64 only**. -- Make it **non-blocking** initially (`continue-on-error: true` + a visible - summary), promote to blocking once the skip-list has settled. -- Full smoke-all in the release pipeline is probably too slow/flaky for a - nightly release gate; keep the full suite in Mode A instead. +### 4.1 Mode A — build-then-test workflow (primary) + +New workflow `test-smokes-built.yml`, **weekly `schedule` + manual +`workflow_dispatch`**, testing an artifact built *in the same run* from the +current `main` (preventive — catches packaging/built-layout breakage before +the nightly `create-release.yml` ships it): + +1. **`build-artifact` job** (ubuntu-latest): checkout `main` (record the SHA + as a job output), then reproduce `make-tarball` exactly + (`create-release.yml:136-159`): + `./configure.sh` → `pushd package/src && ./quarto-bld prepare-dist + --set-version --log-level info` → tar `package/pkg-working` → + upload as workflow artifact `built-quarto-linux-amd64`. Version string: + `version.txt` base + a `+test`/run-number marker so it is never confused + with a published build. +2. **`run-smokes` job**: call the parameterized `test-smokes.yml` with + `quarto-install: artifact`, `quarto-artifact-name: + built-quarto-linux-amd64`, **checking out the same SHA** as the build job + (harness = binary commit, zero skew), and **no buckets** → full run, which + includes all of smoke-all (`docs/smoke-all/**`) plus the `.test.ts` + smokes that survive binary mode. Optionally add the ff-matrix bucket + (`dev-docs/feature-format-matrix/qmd-files/**/*.qmd`) as a second job for + extra coverage. +3. Platform: **linux-amd64 first**. The Windows zip requires the signing + pipeline (`make-installer-win`, `create-release.yml:350-429`), so + Windows coverage comes more easily from the published-prerelease mode + below (unsigned local zip build is a possible later enhancement). + +Cost note: the build job is cheap (`configure.sh` + `prepare-dist`, a few +minutes — same as every `create-release` build job); the expensive part is +the smoke suite itself, which already runs daily in dev mode, so a weekly +built-mode run is a modest addition. + +### 4.2 Mode A′ — same workflow, published (pre)release as input + +`test-smokes-built.yml` also takes a `workflow_dispatch` input +`source: build (default) | release`, plus `version` (`pre-release`, +`release`, or an explicit `1.x.y`). With `source: release` it skips the +build job, checks out **`refs/tags/v`** (harness matches binary, +§2.2), and calls `test-smokes.yml` with `quarto-install: release`. Use +cases: verifying the bits users actually install (post signing/packaging), +Windows coverage, and backfilling any historical version when bisecting a +regression report ("did 1.9.12 already have this?"). + +### 4.3 Mode B (later option) — gate inside `create-release.yml` + +Once binary mode is stable and the dev-only skip-list has settled, the same +parameterized `test-smokes.yml` can be called from `create-release.yml` +between build and `publish-release` (artifact `Deb Zip`), turning the weekly +preventive check into a true release gate. Recommendation: keep this out of +scope until Mode A has produced a few weeks of clean signal — a flaky gate +on the nightly pipeline is worse than none. Start `continue-on-error: true` +and possibly with a bounded bucket (ff-matrix) rather than full smoke-all to +keep the nightly duration sane. --- ## 5. Phased roadmap **Phase 0 — spike (≈1 day).** Hack the seam locally (no plumbing polish): -download the current prerelease tarball, set `QUARTO_TEST_BIN`, run ~20–30 -representative smoke-all docs + a couple of ff-matrix files. Catalog failure +build a dist locally (`configure.sh` + `quarto-bld prepare-dist`), set +`QUARTO_TEST_BIN`, run ~20–30 representative smoke-all docs. Catalog failure classes (env leakage, path derivation drift, dev-only assumptions). This validates the log-file contract end-to-end before investing in CI. @@ -268,16 +299,18 @@ validates the log-file contract end-to-end before investing in CI. sanity job (or manual checklist) that runs a tiny binary-mode bucket to keep the mode from rotting. -**Phase 2 — Mode A workflow.** Parameterize `test-smokes.yml` (§4.0), add -`test-smokes-built.yml` running the ff-matrix bucket weekly against the -latest prerelease, Linux first, then Windows. Triage failures into: real -product bugs (jackpot — this is the point of the project), harness -assumptions (fix), dev-only tests (skip-list). +**Phase 2 — Mode A workflow (build-then-test).** Parameterize +`test-smokes.yml` (§4.0); add `test-smokes-built.yml` (weekly + +`workflow_dispatch`) that builds the linux-amd64 dist from `main` and runs +**full smoke-all** against it (§4.1). Triage failures into: real product +bugs caught pre-release (jackpot — this is the point of the project), +harness assumptions (fix), dev-only tests (skip-list). -**Phase 3 — broaden.** Run full smoke-all buckets in Mode A; wire Mode B -(ff-matrix bucket, non-blocking) into `create-release.yml`; consider the -`release` event trigger so every nightly prerelease gets a built-version run -without touching the release pipeline itself. +**Phase 3 — broaden.** Add the `source: release` input path (§4.2) for +published-prerelease testing, Windows coverage, and version backfill; add +the ff-matrix bucket job if wanted. Once several weeks of clean signal +exist, consider Mode B — calling the same suite from `create-release.yml` +as a (initially non-blocking) gate before `publish-release` (§4.3). **Phase 4 — optional decoupling.** If testing binaries whose version diverges from the harness checkout ever matters (e.g. running main's test suite @@ -285,7 +318,7 @@ against last stable), replace the remaining `src/` imports in the smoke-all driver: `quarto inspect` for metadata/format discovery, `@std/fs` glob, and derive output paths from actual render results instead of `outputForInput`. This is a bigger refactor with drift risk and is **not** needed for the -matched-tag strategy. +matched-commit strategy. --- @@ -306,9 +339,14 @@ matched-tag strategy. which is actually *safer* than today's process-global `Deno.env` mutation. - **Skip-list drift**: `requiresDevQuarto` annotations need a home in review guidelines so new dev-only tests get tagged at authoring time. -- **Decision needed — cadence & blocking-ness**: weekly scheduled (Mode A) - vs. every nightly prerelease vs. inside `create-release.yml` (Mode B), and - whether failures should ever block publishing. Recommendation: Mode A - weekly non-blocking first; revisit after a month of signal. -- **Decision needed — scope**: start with ff-matrix bucket only (recommended) - or jump straight to full smoke-all. +- **Full smoke-all duration in one job**: the daily dev-mode `test-smokes.yml` + scheduled run already does a full un-sharded run per OS, so a weekly + built-mode equivalent is feasible; if it proves too slow, reuse the + `run-parallel-tests` bucket matrix from `test-smokes-parallel.yml`. +- ~~Decision needed — cadence & blocking-ness~~ **Decided:** weekly schedule + + manual `workflow_dispatch`; non-blocking (no release gate) until the + mode has a track record (§4.3). +- ~~Decision needed — scope~~ **Decided:** full smoke-all (ff-matrix bucket + optional extra), against an artifact **built in the same workflow run** + (preventive), with published-(pre)release testing as a secondary dispatch + input (curative/backfill). From fd898846bb459e11c6fcee9f3ffac99aa99ff529 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 11:05:57 +0000 Subject: [PATCH 03/43] Add binary-mode classification results for all 133 smoke test files Full sweep of tests/smoke/**/*.test.ts: 107 compatible as-is, 24 adaptable through four mechanical patterns (shared runQuarto dispatch helper, quartoDevCmd honoring QUARTO_TEST_BIN, consolidating hardcoded package/dist/bin spawns, two semantic one-offs), and only 2 genuinely dev-only yaml-intelligence unitTest files that should move to tests/unit/ instead of carrying a requiresDevQuarto flag. Also audits smoke-all documents: docs are binary-clean except ten fixture extensions pinned to quarto-required '>=99.9.0', which hard-error against a real-version binary; recommend relaxing to '>=1.9'. Records a no-reorganization verdict with two greppable lint rules to keep future tests binary-compatible. Co-Authored-By: Claude Fable 5 --- dev-docs/smoke-tests-built-version-plan.md | 136 +++++++++++++++++++-- 1 file changed, 129 insertions(+), 7 deletions(-) diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index 72dd009104..42c2f11ae5 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -176,17 +176,22 @@ Introduce `QUARTO_TEST_BIN` (absolute path to a built `quarto` / ### 3.4 Tests that can't run in binary mode -Most `tests/smoke/**/*.test.ts` are pure `testQuartoCmd` and will just work. -A minority poke internals (unit tests, tests importing quarto APIs to compute -expectations, `quarto run` TS scripts relying on dev Deno, env-var tests). +A full classification sweep of all 133 `tests/smoke/**/*.test.ts` files was +performed (2026-07-17, six parallel agents reading every file; results in +§7). Outcome: **107 compatible as-is, 24 adaptable via a handful of +mechanical patterns, only 2 genuinely dev-only** (both `unitTest()`-based +yaml-intelligence tests that arguably belong in `tests/unit/`). + Mechanism: - Add `TestContext.requiresDevQuarto?: boolean`; the `test()` wrapper sets - Deno's `ignore` when binary mode is active. + Deno's `ignore` when binary mode is active. Given the sweep results this + flag is a rare escape hatch, not a broad annotation campaign. - Unit tests (`tests/unit/`) are dev-only by definition — excluded wholesale - in binary mode. -- Start by targeting only **smoke-all + feature-format matrix** (bucket - invocation), where this problem is near-zero, and grow coverage from there. + in binary mode. The 2 dev-only smoke files should simply **move to + `tests/unit/`** rather than carry the flag (see §7.3). +- The 24 "adapt" files reduce to shared-helper fixes (§7.2), not per-test + work. --- @@ -350,3 +355,120 @@ matched-commit strategy. optional extra), against an artifact **built in the same workflow run** (preventive), with published-(pre)release testing as a secondary dispatch input (curative/backfill). + +--- + +## 7. Classification sweep results (2026-07-17) + +All 133 `tests/smoke/**/*.test.ts` files were read in full and classified +for binary-mode compatibility: **107 compatible / 24 adapt / 2 dev-only**. + +### 7.1 Compatible directories (no changes needed) + +`render` (28/31), `crossref`+`site`+`website` (25/26), `project` (8/8), +`inspect` (5/6), `extensions` (5/7), `yaml`, `ojs`, `use`, `verify`, `jats`, +`book`, `shortcodes`, `search`, `scholar`, `manuscript`, `embed`, `authors`, +`build-ts-extension`, `check`, and more — everything funneling through +`testQuartoCmd` or its wrappers (`testRender`, `testSite`, +`testProjectRender`, `testManuscriptRender`). Their `src/` imports are +expectation/path/cleanup helpers only. `TestContext.env` is already passed +as a parameter (not `Deno.env.set`), so it forwards cleanly to a subprocess +— binary mode actually *improves* isolation for env-dependent tests. + +### 7.2 The 24 "adapt" files — four mechanical patterns + +**(a) Direct `quarto()` import from `src/quarto.ts`** (~14 files; greppable +via `from ".*src/quarto.ts"`). Setup-side pre-renders or multi-step bodies: +`render-freeze`, `render-format-extension`, `render-output-file-collision`, +`crossref/syntax`, `extensions/extension-render-{journals,typst-templates}`, +`convert/issue-12318`, `jupyter/{cache,issue-10097,issue-12374}`, +`engine/invalid-engine-in-project`, `self-contained/stdout`, `issues/9133`, +and `smoke-all.test.ts` itself (project pre-render). Fix: one shared +`runQuarto(args, {env, cwd})` helper dispatching to in-process `quarto()` +or a `QUARTO_TEST_BIN` spawn; three of these are trivial rewrites to plain +`testQuartoCmd`. Notes: `invalid-engine-in-project` asserts on a thrown +Error (convert to exit-code + ERROR-log assertion; its `assertRejects` is +currently not awaited, so it silently passes today — a pre-existing bug); +`issues/9133` reproduces an *intra*-process concurrency bug, so the +two-subprocess version needs a runtime check that it still triggers. + +**(b) PATH-quarto subprocess via `quartoDevCmd()`** (`run/*` ×3, +`lua-unit`, `logging`, `create`): a **one-line fix** — `quartoDevCmd()` +(`tests/utils.ts:244`) returns `QUARTO_TEST_BIN` when set. These tests +already spawn a real binary; several (e.g. `stdlib-run-version`, +`lua-unit`) arguably *belong* in binary mode since they verify the shipped +`quarto run` stdlib/embedded deno. + +**(c) Hardcoded `../package/dist/bin/quarto` spawns** +(`filters/editor-support`, `typst-gather` tests 7–12): consolidate onto the +patched `quartoDevCmd()`. + +**(d) Semantic one-offs**: `env/check.test.ts` hardcodes `Version: 99.9.9` +(compute expectation from the binary, or relax the regex); +`inspect/inspect-standalone-rstudio.test.ts` uses the in-process +`_setIsRStudioForTest` hook (in binary mode, set `RSTUDIO=1` in the child +env instead — simpler than today; the companion "not RStudio" test requires +the harness to spawn with a *clean* env). + +Implementation caveats surfaced by the sweep: +- `testQuartoCmd`'s `cwd` option must keep chdir-ing the **harness** + process too (relative-path verifiers and teardowns depend on it), while + also setting the subprocess cwd. +- Binary mode's `--log/--log-format` injection applies only to + `testQuartoCmd`-driven invocations — never to tests that spawn quarto + themselves and own their flags (`logging/log-level-and-formats` exists + precisely to test those flags). + +### 7.3 Genuinely dev-only (2 files) + +`yaml-intelligence/yaml-intelligence.test.ts` and +`yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts` — +`unitTest()` calls exercising `src/core/lib/yaml-intelligence` internals +with no CLI surface. **Recommendation: move them to `tests/unit/`** (the +other two files in `smoke/yaml-intelligence/` are ordinary render tests and +stay). With that move, *zero* smoke tests need `requiresDevQuarto` today; +the flag remains as an escape hatch for future tests. + +**Reorganization verdict:** a folder/name convention (e.g. `smoke-dev/`, +`*.dev.test.ts`) is not worth it — dev-only-ness is too rare. Caution: +"uses `unitTest()`" is NOT a reliable dev-only signal (`run/*`, +`lua-unit`, `typst-gather` use it as a generic wrapper around subprocess +spawns). The reliable, greppable signals are: (a) `import ... from +".*src/quarto.ts"` in a test file, (b) spawns not routed through +`quartoDevCmd()`. Enforce both as lightweight lint/CI rules so new tests +stay binary-compatible by construction. + +### 7.4 Smoke-all documents audit + +The docs themselves are almost entirely binary-clean: no doc executes +quarto from a code cell, no runtime dev-tree paths, no pre/post-render +scripts in fixture `_quarto.yml`s, and **every `printsMessage` assertion +uses INFO/WARN/ERROR — never DEBUG** — matching a built binary's default +log level. The json-stream record shape from `--log --log-format +json-stream` is identical to what `readExecuteOutput` parses, and +`--quiet` does not affect the file handler. + +**One systemic blocker:** ten fixture extensions declare +`quarto-required: '>=99.9.0'`, which passes only against the dev sentinel +version — a real-version binary hard-errors in `validateExtension` +(`src/extension/extension.ts:776-788`). Affected fixtures: the +`dragonstyle/lipsum` copies under `dashboard/`, `lightbox/`, +`format/html/`, `2023/04/24/`; the brand/typst extension fixtures under +`typst/brand-yaml/typography/`, `typst/font-paths/`, +`brand/typography/remote-font-extension/`, `brand/logo/logo-extension*/`; +and `2023/01/06/input-relative/`. Two options: +- **Relax the fixtures to `'>=1.9'`** (preferred — keeps the binary's real + version visible to everything else), unless a doc specifically tests + version gating; +- or export `QUARTO_FORCE_VERSION=99.9.9` into the child env (honored at + `src/core/quarto.ts:35`) — a blunter tool that masks real version + behavior. + +Driver adaptations beyond plain render: route the `editor-support-crossref` +pseudo-format (2 docs) and the 24 `render-project` pre-renders through the +binary; tolerate non-zero child exit (read the log file regardless — the +binary's top-level handler writes the ERROR record `shouldError` needs). +Runtime one-time checks: `engine/class-override` extensions +(`quarto-required '>=1.9.17'` — fails on older release binaries by design), +`QUARTO_EXECUTE_INFO` / `QUARTO_PROJECT_ROOT` env-var docs, and the +pandoc-args INFO echo docs (`2025/12/09/13775-*`). From 18a9c4dc67145c32412fe31d87ab037e48d472bd Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 11:23:37 +0000 Subject: [PATCH 04/43] Rewrite built-version test plan as grounded v2 design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incorporates four verification passes with file:line evidence: - Git archaeology: the ten quarto-required '>=99.9.0' fixtures are scaffolding artifacts (create-extension template computes the value from the running dev version); no test exercises version gating; all ten are safe to relax to '>=1.9'. QUARTO_FORCE_VERSION=99.9.9 is rejected (masks version gates, flips quarto check into dev mode). - Log pipeline: --log/--log-format json-stream confirmed global and per-record flushed; render failures write ERROR before exit 1; pandoc/typst/run passthroughs are the documented exception. - Env hygiene: full strip/allow/overlay contract with clearEnv; DENO_DIR/QUARTO_DENO added to strip list; dev-mode trap documented (in-repo package/dist/bin/quarto runs TS sources — dist must be extracted outside the checkout, guarded by a 99.9.9 version check). - CI: quarto-dev action must run in every mode (harness runtime); built binary is a PATH override on top, not a replacement; prepare-dist writes only to package/pkg-working so build+test can share a checkout. Windows built layout ships both quarto.cmd (prepare-dist) and the Rust launcher quarto.exe (installer job); release testing targets quarto.exe. Includes concrete seam spec (runQuarto helper), parameterized test-smokes.yml step diff, test-smokes-built.yml sketch, and phased roadmap with acceptance criteria. Co-Authored-By: Claude Fable 5 --- dev-docs/smoke-tests-built-version-plan.md | 977 +++++++++++---------- 1 file changed, 524 insertions(+), 453 deletions(-) diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index 42c2f11ae5..55124fe019 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -1,474 +1,545 @@ # Plan: Running Smoke Tests Against a Built Quarto -Status: **proposal / plan** — no implementation yet. +Status: **detailed design — v2** (grounded; pending adversarial review + Phase 0 spike). -Goal: be able to run the existing smoke test suites — at minimum `smoke-all` -and the feature-format matrix — against a **built** Quarto (a GitHub -prerelease/release, or an installer artifact produced earlier in the same CI -run), instead of only the dev source tree. +Goal: run the existing smoke test suites — the full smoke-all corpus first, +plus the `.test.ts` smokes and the feature-format matrix — against a +**built** Quarto (an artifact produced in the same CI run, or a published +GitHub (pre)release), instead of only the dev source tree. ---- +Decisions (maintainer, 2026-07-17): -## 1. How testing works today (findings) +- Weekly `schedule` **plus** manual `workflow_dispatch`. +- Scope: **full smoke-all** (ff-matrix bucket optional extra). +- Primary mode is **preventive build-then-test**: build the artifact in the + same workflow run and test it — not only after-the-fact testing of + published prereleases (that stays as a secondary dispatch input). +- No release-pipeline gating until the mode has a track record. -### 1.1 The harness runs Quarto in-process +--- -- `testQuartoCmd` does **not** spawn a `quarto` binary. It imports the CLI - entry point and calls it as a function inside the Deno test process: - - `tests/test.ts:13` — `import { quarto } from "../src/quarto.ts";` - - `tests/test.ts:159-162` — `await Promise.race([quarto([cmd, ...args], undefined, context?.env), timeout])` -- `tests/run-tests.sh` launches the repo-bundled Deno with - `--importmap=src/import_map.json`, `--allow-all`, `--unstable-kv --unstable-ffi` - (`run-tests.sh:57,63,164`) and exports source-tree paths: +## 1. How testing works today (established facts) + +- `testQuartoCmd` invokes Quarto **in-process**: it imports `quarto()` from + `src/quarto.ts` (`tests/test.ts:13`) and calls it directly + (`tests/test.ts:159-162`). There is no subprocess and no configurable seam. +- Before each test the harness redirects Quarto's logger to a temp file in + `json-stream` format (`tests/test.ts:243-249`); after execution it parses + the file into `ExecuteOutput { msg, level, levelName }` records + (`test.ts:176-180, 386-392`) consumed by every log verifier + (`noErrors`, `noErrorsOrWarnings`, `shouldError`, `printsMessage` — + `tests/verify.ts:107-205`). Errors thrown by `execute()` are caught and + converted into ERROR log records before verification (`test.ts:262-266`). +- All file-content verifiers read produced output files off disk. +- `tests/run-tests.sh` runs the harness under the repo-bundled Deno with + `--importmap=src/import_map.json` and exports dev paths: `QUARTO_BIN_PATH=package/dist/bin` (`:47`), - `QUARTO_SHARE_PATH=src/resources` (`:60`), `QUARTO_DEBUG=true` (`:61`). -- The only subprocess seam today is `quartoDevCmd()` (`tests/utils.ts:244`, - returns `"quarto"`/`"quarto.cmd"` from `PATH`), used by a handful of - passthrough/Playwright tests — not by the render harness. - -### 1.2 Output capture and verification are already execution-agnostic - -- Before each test, the harness redirects Quarto's logger to a temp file in - `json-stream` format (`tests/test.ts:243-249`). After execution it parses - the file line-by-line into `ExecuteOutput { msg, level, levelName }` - (`test.ts:176-180, 386-392`) and hands that array to every verifier. -- `noErrors` / `noErrorsOrWarnings` / `shouldError` / `printsMessage` only - inspect those log records (`tests/verify.ts:107-205`). -- Errors thrown by `execute()` are caught and converted into ERROR log - records before verification (`test.ts:262-266`), so verifiers never depend - on exceptions from the in-process call. -- All file-content verifiers (`ensureHtmlElements`, `ensureFileRegexMatches`, - `ensureLatexFileRegexMatches`, docx/pptx/pdf/jats/odt checks, snapshots…) - read produced files off disk and do not care how Quarto ran. -- Crucially, the **CLI already exposes the same logging pipeline as flags**: - `--log --log-format json-stream --log-level info` - (`src/core/log.ts:28,66-67,116,447`). A subprocess can therefore produce a - byte-compatible log file for the existing verifiers. - -### 1.3 smoke-all specifics - -- `tests/smoke/smoke-all.test.ts` globs `docs/smoke-all/**/*.{md,qmd,ipynb}` - (`:378-386`), parses `_quarto.tests` metadata with **dev-source code** - (`readYamlFromMarkdown` from `src/core/yaml.ts`, `jupyterNotebookToMarkdown` - from `src/command/convert/jupyter.ts`, `:399-401`), maps spec keys to - verifiers via an inline `verifyMap` (`:189-218`), and calls - `testQuartoCmd("render", [input, "--to", format], ...)` (`:462`). It also - calls `quarto(["render", projectPath])` directly for project pre-renders - (`:434`). -- `tests/utils.ts` (`outputForInput`, `findProjectOutputDir`, …) re-implements - Quarto's output-path rules in harness code (`utils.ts:100-208`). -- The YAML-intelligence bootstrap (`initYamlIntelligenceResourcesFromFilesystem`, - `setInitializer`/`initState`, `smoke-all.test.ts:64-66,463-466`) exists only - because Quarto runs in-process. - -### 1.4 Feature-format matrix = a smoke-all bucket - -- `.github/workflows/test-ff-matrix.yml:44-50` reuses `test-smokes.yml` with - `buckets: '[ "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" ]'`. - The bucket loop calls `./run-tests.sh "$file"` which routes `.qmd` args to - `smoke-all.test.ts` (`run-tests.sh:132-164`). The 0/1/2 quality ratings are - display-only (`create_table.py`); CI enforces only the `_quarto.tests` specs. -- So: **anything that makes smoke-all work against a built binary makes the - feature-format matrix work too.** - -### 1.5 CI today - -- `test-smokes.yml` sets up all language deps inline (R/renv, Python/uv, - Julia, TinyTeX, Playwright, …) and then installs the **dev tree** via the - `quarto-dev` composite action (`configure.sh`; asserts version `99.9.9` — - `.github/workflows/actions/quarto-dev/action.yml:45-51`). -- `create-release.yml` runs nightly, builds per-OS artifacts - (`quarto--linux-amd64.tar.gz`, msi/zip, pkg/tar.gz, deb/rpm), uploads - them as **workflow artifacts**, publishes them as **GitHub Release assets** - (prerelease by default), and tags `v` on the exact commit built - (`create-release.yml:88-100,678-703`). -- The release pipeline already has minimal "test the built artifact" jobs - (`test-tarball-linux`, `test-zip-win`, `test-zip-mac`): download artifact, - extract, `quarto check` / `--version` (`create-release.yml:310-348,431-477,548-585`). - These are natural extension points. -- `test-smokes-parallel.yml` already uses `quarto-dev/quarto-actions/setup@v2` - with `version: pre-release` — but only to compute test buckets, not to run - tests (`test-smokes-parallel.yml:59-75`). - ---- + `QUARTO_SHARE_PATH=src/resources` (`:60`), `QUARTO_DEBUG=true` (`:61`), + `DENO_DIR=package/dist/bin/deno_cache` (`:50-54`). +- `smoke-all.test.ts` globs `docs/smoke-all/**/*.{md,qmd,ipynb}`, parses + `_quarto.tests` with dev-source YAML code, dispatches spec keys to + verifiers via an inline `verifyMap`, and renders via + `testQuartoCmd("render", [input, "--to", format])` (`:462`); project + pre-renders call `quarto(["render", projectPath])` directly (`:434`). +- The **feature-format matrix is a smoke-all bucket**: + `test-ff-matrix.yml:44-50` calls `test-smokes.yml` with a qmd glob that + `run-tests.sh:132-164` routes to `smoke-all.test.ts`. Anything that fixes + smoke-all fixes the matrix. +- `create-release.yml` runs nightly: builds per-OS artifacts, uploads them + as workflow artifacts, publishes GitHub Release assets (prerelease by + default) and tags `v` on the exact commit built + (`create-release.yml:88-100,678-703`). The linux tarball recipe is just + `./configure.sh` → `quarto-bld prepare-dist --set-version ` → tar of + `package/pkg-working` (`create-release.yml:136-159`). ## 2. Strategy -Two structural insights drive the design: - -1. **The coupling to the dev tree is concentrated in one seam** — the body of - `test.execute()` (the in-process `quarto()` call plus programmatic logger - init). The entire verifier layer, the `_quarto.tests` dispatch, and the - failure-reporting machinery consume only (a) the json-stream log file and - (b) files on disk. Both can be produced identically by a subprocess using - `--log/--log-format/--log-level`. - -2. **Version skew is avoidable by construction.** Every published (pre)release - is tagged `v` on the commit it was built from. If CI checks out - the repo **at the tag matching the binary under test**, the harness code, - test documents, schemas, and `utils.ts` path-derivation logic match the - binary exactly. The harness may keep importing `src/` for *parsing and - path math* — only *execution* moves to the binary. This keeps the change - small and low-risk. (Full decoupling via `quarto inspect` is a later, - optional phase — see Phase 4.) - -So the recommended architecture is: **add a "binary mode" to the existing -harness (subprocess execution behind an env var), keep everything else as-is, -and always pair a built binary with the repo checkout at its own tag.** - ---- - -## 3. Design: binary mode in the harness - -### 3.1 The execution seam - -Introduce `QUARTO_TEST_BIN` (absolute path to a built `quarto` / -`quarto.cmd`). In `tests/test.ts`: - -- When unset → current behavior (in-process `quarto()`), zero change for the - default dev workflow. -- When set → `test.execute()` spawns the binary via `Deno.Command`: - - ``` - \ - --log --log-format json-stream --log-level info - ``` - - - Reuse the temp log path already created at `test.ts:243`; skip (or - no-op) the in-process `initializeLogger` for the subprocess side, but - keep `logError(e)` able to append harness-side failures (spawn errors, - timeouts) as ERROR records so existing reporting keeps working. - - Do **not** throw on non-zero exit: the binary writes its own ERROR - records to the log, which is exactly what `noErrors`/`shouldError` - already consume. (Mirrors today's catch-and-log at `test.ts:262-266`.) - - Timeout: on expiry, kill the child process (an improvement over today, - where a timed-out in-process render keeps running). - - `context.env`: pass as subprocess env overlay instead of the - in-process env juggling in `quarto()` (`src/quarto.ts:163-217`). - - cwd: pass `Deno.cwd()` (the harness still does `Deno.chdir` for - `context.cwd`, so inheriting cwd is sufficient). - -- **Environment hygiene (critical):** the installed launcher *inherits* - `QUARTO_SHARE_PATH` if set (`package/scripts/common/quarto:102-108`), and - `run-tests.sh` exports it pointing at `src/resources`. In binary mode the - subprocess env must **drop/unset** `QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, - and `QUARTO_DEBUG` so the binary resolves its own installed layout. - -### 3.2 smoke-all adjustments - -- Route the direct project pre-render `quarto(["render", projectPath])` - (`smoke-all.test.ts:434`) through the same seam (small helper - `runQuarto(args)` used by both call sites). -- Metadata parsing, format guessing, and `verifyMap` dispatch stay unchanged - (they run in the harness process against the matching checkout — see §2.2). -- The YAML-intelligence bootstrap can stay; it is harness-process-only and - harmless in binary mode. - -### 3.3 Runner plumbing - -- `run-tests.sh` / `run-tests.ps1`: accept `--bin ` (or just honor an - exported `QUARTO_TEST_BIN`). When set: - - still resolve the repo Deno + import map to run the *harness* (a repo - checkout is always required — it holds the tests); - - skip exporting `QUARTO_SHARE_PATH` / `QUARTO_DEBUG`; - - print a banner with ` --version` so logs are unambiguous about what - was tested. - -### 3.4 Tests that can't run in binary mode - -A full classification sweep of all 133 `tests/smoke/**/*.test.ts` files was -performed (2026-07-17, six parallel agents reading every file; results in -§7). Outcome: **107 compatible as-is, 24 adaptable via a handful of -mechanical patterns, only 2 genuinely dev-only** (both `unitTest()`-based -yaml-intelligence tests that arguably belong in `tests/unit/`). - -Mechanism: - -- Add `TestContext.requiresDevQuarto?: boolean`; the `test()` wrapper sets - Deno's `ignore` when binary mode is active. Given the sweep results this - flag is a rare escape hatch, not a broad annotation campaign. -- Unit tests (`tests/unit/`) are dev-only by definition — excluded wholesale - in binary mode. The 2 dev-only smoke files should simply **move to - `tests/unit/`** rather than carry the flag (see §7.3). -- The 24 "adapt" files reduce to shared-helper fixes (§7.2), not per-test - work. - ---- - -## 4. CI integration - -> **Decisions recorded (2026-07-17):** weekly schedule **plus manual -> `workflow_dispatch`**; scope is **full smoke-all** (ff-matrix bucket is a -> nice-to-have addition, not the target); and the primary goal is -> **preventive** — test artifacts *as built*, not only published prereleases -> (which is curative, after the fact). Since `create-release.yml` builds the -> linux tarball with just `./configure.sh` + `quarto-bld prepare-dist -> --set-version` + tar of `package/pkg-working` -> (`create-release.yml:124-159`), the test workflow can **build the artifact -> itself with the exact same steps** and test it in the same run — same -> commit by construction, no version-skew handling, and zero changes to the -> release pipeline. - -Modes below funnel through a parameterized `test-smokes.yml`. - -### 4.0 Parameterize `test-smokes.yml` - -New `workflow_call` inputs: - -- `quarto-install`: `dev` (default) | `release` | `artifact` -- `quarto-version`: version/tag string (for `release`), e.g. `1.10.23` or - `pre-release` -- `quarto-artifact-name`: workflow artifact to download (for `artifact`) - -Setup step becomes conditional: - -- `dev` → existing `quarto-dev` composite action (unchanged default). -- `release` → `quarto-dev/quarto-actions/setup@v2` with the given version - (already used elsewhere in this repo: `test-smokes-parallel.yml:59-62`). -- `artifact` → `actions/download-artifact`, extract zip/tarball, add `bin` - to `PATH` (same recipe as `create-release.yml:310-348`). - -For `release`/`artifact`: skip the `99.9.9` dev assertion, instead assert -`quarto --version` equals the expected version, and export -`QUARTO_TEST_BIN=$(command -v quarto)` for the run-tests steps. All -language-dependency setup (R/Python/Julia/TinyTeX/…) is shared and unchanged. - -### 4.1 Mode A — build-then-test workflow (primary) - -New workflow `test-smokes-built.yml`, **weekly `schedule` + manual -`workflow_dispatch`**, testing an artifact built *in the same run* from the -current `main` (preventive — catches packaging/built-layout breakage before -the nightly `create-release.yml` ships it): - -1. **`build-artifact` job** (ubuntu-latest): checkout `main` (record the SHA - as a job output), then reproduce `make-tarball` exactly - (`create-release.yml:136-159`): - `./configure.sh` → `pushd package/src && ./quarto-bld prepare-dist - --set-version --log-level info` → tar `package/pkg-working` → - upload as workflow artifact `built-quarto-linux-amd64`. Version string: - `version.txt` base + a `+test`/run-number marker so it is never confused - with a published build. -2. **`run-smokes` job**: call the parameterized `test-smokes.yml` with - `quarto-install: artifact`, `quarto-artifact-name: - built-quarto-linux-amd64`, **checking out the same SHA** as the build job - (harness = binary commit, zero skew), and **no buckets** → full run, which - includes all of smoke-all (`docs/smoke-all/**`) plus the `.test.ts` - smokes that survive binary mode. Optionally add the ff-matrix bucket - (`dev-docs/feature-format-matrix/qmd-files/**/*.qmd`) as a second job for - extra coverage. -3. Platform: **linux-amd64 first**. The Windows zip requires the signing - pipeline (`make-installer-win`, `create-release.yml:350-429`), so - Windows coverage comes more easily from the published-prerelease mode - below (unsigned local zip build is a possible later enhancement). - -Cost note: the build job is cheap (`configure.sh` + `prepare-dist`, a few -minutes — same as every `create-release` build job); the expensive part is -the smoke suite itself, which already runs daily in dev mode, so a weekly -built-mode run is a modest addition. - -### 4.2 Mode A′ — same workflow, published (pre)release as input - -`test-smokes-built.yml` also takes a `workflow_dispatch` input -`source: build (default) | release`, plus `version` (`pre-release`, -`release`, or an explicit `1.x.y`). With `source: release` it skips the -build job, checks out **`refs/tags/v`** (harness matches binary, -§2.2), and calls `test-smokes.yml` with `quarto-install: release`. Use -cases: verifying the bits users actually install (post signing/packaging), -Windows coverage, and backfilling any historical version when bisecting a -regression report ("did 1.9.12 already have this?"). - -### 4.3 Mode B (later option) — gate inside `create-release.yml` - -Once binary mode is stable and the dev-only skip-list has settled, the same -parameterized `test-smokes.yml` can be called from `create-release.yml` -between build and `publish-release` (artifact `Deb Zip`), turning the weekly -preventive check into a true release gate. Recommendation: keep this out of -scope until Mode A has produced a few weeks of clean signal — a flaky gate -on the nightly pipeline is worse than none. Start `continue-on-error: true` -and possibly with a bounded bucket (ff-matrix) rather than full smoke-all to -keep the nightly duration sane. - ---- - -## 5. Phased roadmap - -**Phase 0 — spike (≈1 day).** Hack the seam locally (no plumbing polish): -build a dist locally (`configure.sh` + `quarto-bld prepare-dist`), set -`QUARTO_TEST_BIN`, run ~20–30 representative smoke-all docs. Catalog failure -classes (env leakage, path derivation drift, dev-only assumptions). This -validates the log-file contract end-to-end before investing in CI. - -**Phase 1 — harness binary mode.** Implement §3 properly: seam in `test.ts`, -`runQuarto` helper for smoke-all, env hygiene, `requiresDevQuarto`, -`run-tests.sh`/`.ps1` plumbing. Default behavior unchanged; add a small CI -sanity job (or manual checklist) that runs a tiny binary-mode bucket to keep -the mode from rotting. - -**Phase 2 — Mode A workflow (build-then-test).** Parameterize -`test-smokes.yml` (§4.0); add `test-smokes-built.yml` (weekly + -`workflow_dispatch`) that builds the linux-amd64 dist from `main` and runs -**full smoke-all** against it (§4.1). Triage failures into: real product -bugs caught pre-release (jackpot — this is the point of the project), -harness assumptions (fix), dev-only tests (skip-list). - -**Phase 3 — broaden.** Add the `source: release` input path (§4.2) for -published-prerelease testing, Windows coverage, and version backfill; add -the ff-matrix bucket job if wanted. Once several weeks of clean signal -exist, consider Mode B — calling the same suite from `create-release.yml` -as a (initially non-blocking) gate before `publish-release` (§4.3). - -**Phase 4 — optional decoupling.** If testing binaries whose version diverges -from the harness checkout ever matters (e.g. running main's test suite -against last stable), replace the remaining `src/` imports in the smoke-all -driver: `quarto inspect` for metadata/format discovery, `@std/fs` glob, and -derive output paths from actual render results instead of `outputForInput`. -This is a bigger refactor with drift risk and is **not** needed for the -matched-commit strategy. - ---- - -## 6. Risks & open questions - -- **`QUARTO_DEBUG` behavior differences**: dev runs with `QUARTO_DEBUG=true`; - a built binary runs without it. Some tests may implicitly depend on debug - behaviors (stack traces in logs, error formatting). Phase 0 will surface - these. -- **Version-string assumptions**: anything expecting `99.9.9` (or dev share - paths) must be conditionalized (`quarto-dev` action assertion, any tests - matching version output). -- **Windows**: subprocess must invoke `quarto.cmd` (or use `cmd /c`); - quoting of args with spaces needs care in the `Deno.Command` seam. -- **Per-test subprocess startup cost** (~a few hundred ms × hundreds of - smoke-all docs) is real but small relative to render time; acceptable. -- **Parallel test files + env vars**: binary mode passes env per-subprocess, - which is actually *safer* than today's process-global `Deno.env` mutation. -- **Skip-list drift**: `requiresDevQuarto` annotations need a home in review - guidelines so new dev-only tests get tagged at authoring time. -- **Full smoke-all duration in one job**: the daily dev-mode `test-smokes.yml` - scheduled run already does a full un-sharded run per OS, so a weekly - built-mode equivalent is feasible; if it proves too slow, reuse the - `run-parallel-tests` bucket matrix from `test-smokes-parallel.yml`. -- ~~Decision needed — cadence & blocking-ness~~ **Decided:** weekly schedule - + manual `workflow_dispatch`; non-blocking (no release gate) until the - mode has a track record (§4.3). -- ~~Decision needed — scope~~ **Decided:** full smoke-all (ff-matrix bucket - optional extra), against an artifact **built in the same workflow run** - (preventive), with published-(pre)release testing as a secondary dispatch - input (curative/backfill). +1. **The dev-tree coupling is concentrated in one seam** — the body of + `test.execute()` (in-process `quarto()` call + programmatic logger init). + The verifier layer and the `_quarto.tests` dispatch consume only the + json-stream log file and files on disk, both reproducible by a + subprocess via `--log --log-format json-stream --log-level info`. +2. **Version skew is avoided by construction.** The primary workflow builds + the artifact from the same SHA the harness checks out. For + published-release testing, check out `refs/tags/v`. The harness + keeps importing `src/` for *parsing and path math* only; *execution* + moves to the binary. + +## 3. Ground truths verified in code (2026-07-17) + +These were verified by dedicated review passes with file:line evidence; +the design in §4–§5 depends on them. + +**Log pipeline (CONFIRMED):** + +- `--log`, `--log-level`, `--log-format` are global options appended to + every command (`src/core/log.ts:49-95`; applied via the `cmdHandler` in + `src/quarto.ts:231-234`), and — decisive — the logger is initialized in + `mainRunner` from a **raw-args parse before any command runs** + (`src/core/main.ts:23-27`, `logOptions` at `src/core/log.ts:97-118`). Env + fallbacks `QUARTO_LOG`, `QUARTO_LOG_LEVEL`, `QUARTO_LOG_FORMAT` exist but + explicit flags win. No command bypasses logger init. +- json-stream writes one `JSON.stringify(logRecord)` per line with + `msg`/`level`/`levelName` (`log.ts:262-263`) — byte-compatible with + `readExecuteOutput`. The file handler **flushes after every record** + (`log.ts:267-273`), so even a hard crash loses nothing already logged. +- On render failure an ERROR record is written before exit 1: errors + propagate to `mainRunner`'s catch → `logError(e)` (`main.ts:70-73`) → + ERROR record in the file; then `exitWithCleanup(1)` (`main.ts:74-81`). + `CommandError` likewise (`src/quarto.ts:202-208`). +- **Exception:** the `pandoc`/`typst`/`run` passthroughs `Deno.exit(code)` + with the child's code and route child stderr directly — a failing + passthrough writes no ERROR record to the log file (`src/quarto.ts:100, + 119, 139`). The tests covering these already use `execProcess` and assert + on stdout/stderr/exit code, so this is a documentation caveat for the + seam, not a blocker. + +**`QUARTO_FORCE_VERSION=99.9.9` is rejected (NUANCED → do not use):** +honored at `src/core/quarto.ts:34-45` but `99.9.9` equals +`kLocalDevelopment`, which (a) satisfies every lower-bound version gate +(extensions `src/extension/extension.ts:776-788`, document +`quarto-required` `src/command/render/render-files.ts:130-144`, engines +`src/execute/engine.ts:62-80`, freezer `src/core/cache/cache.ts:81`) — +masking real "version too old" behavior — and (b) flips `quarto check` +into a dev-mode branch that shells `git rev-parse` in `$QUARTO_ROOT` +(`src/command/check/check.ts:288-302`). The binary must report its real +version; the fixtures get fixed instead (§4.6). + +**The `>=99.9.0` fixtures are scaffolding artifacts (git archaeology):** +`quarto create extension` computes `quarto-required` from the *running* +version truncated to `major.minor.0` +(`src/command/create/artifacts/artifact-shared.ts:143`); on a dev build +(`99.9.9`) that yields exactly `>=99.9.0`. All ten fixtures were authored +on dev checkouts (four separate lipsum vendorings 2023; brand/typst +fixtures 2025–2026). No test anywhere exercises the version-gate error +path (zero test references to `quarto-required` / "incompatible with this +quarto"); sibling extensions in the same `_extensions/` dirs use +`>=1.3.0`, as does upstream `quarto-ext/lipsum` and Quarto's own bundled +copy (`src/resources/extensions/quarto/lipsum/_extension.yml:4`). **All +ten are safe to relax.** + +**Launcher & env behavior:** + +- The installed launcher inherits `QUARTO_SHARE_PATH` and `QUARTO_DEBUG` + if already set (`package/scripts/common/quarto:102-110, 68-70`; + `package/scripts/windows/quarto.cmd:80-81, 47`), recomputes + `QUARTO_ROOT`/`QUARTO_BIN_PATH` unconditionally, and **never sets + `DENO_DIR` outside dev mode** — so a leaked dev `DENO_DIR` reaches the + binary's deno. Strip list follows in §4.2. +- **Dev-mode trap:** the launcher decides dev vs installed by finding a + sibling `src/quarto.ts` relative to its own path + (`common/quarto:22-37`). Therefore `QUARTO_TEST_BIN` pointing at the + in-repo `package/dist/bin/quarto` silently runs **dev mode** (TS + sources, `--check`, dev env defaults) — not the built layout. The dist + must be extracted **outside the git checkout**, and the seam must fail + loudly if ` --version` reports `99.9.9`. +- A genuinely installed binary runs a single esbuild-bundled `quarto.js` + with `--no-check` (`common/quarto:91-121, 205-208`), a real + `share/version` file, inlined Lua filters, and arch-specific + deno/deno_dom (`package/src/common/prepare-dist.ts:128-147, 191-224`). + Binary mode therefore covers bundling/import-resolution errors, missing + share resources, version wiring, and the `--no-check` gap — none of + which dev mode can catch. +- `QUARTO_DEBUG=true` (dev default) only adds stack traces to ERROR `msg` + text (`src/core/log.ts:371-386`) plus dev-only reconfigure/watch paths. + Binary mode runs **without** it — truer to shipped behavior; ERROR + counts are unaffected, only `printsMessage` regexes matching stack text + could differ (none known in smoke-all: audit found all `printsMessage` + levels are INFO/WARN/ERROR with content-based regexes). +- **Windows built layout ships two entry points.** `prepare-dist` copies + `quarto.cmd` into the dist (`copyQuartoScript`, + `package/src/common/configure.ts:143-149`); the `make-installer-win` + job additionally builds and signs the **Rust launcher `quarto.exe`** + (`cargo build`, `package/launcher/src/main.rs`; + `create-release.yml:350-429`), which is what the MSI/zip put on PATH — + i.e. what Windows users actually run. `QUARTO_TEST_BIN` should target + `quarto.exe` for published-release testing (trivial spawn, no `.cmd` + handling); a `prepare-dist`-only Windows artifact has `quarto.cmd`, + which spawns **directly** with `Deno.Command` — no `cmd /c` (the repo's + established pattern: `quartoDevCmd()` `tests/utils.ts:244-246`; npm.cmd + in `src/project/serve/serve.ts:434`). Notes: the Rust launcher has **no + dev-mode branch** (always runs the bundled `quarto.js` — the dev-mode + trap below doesn't apply to it), reads `--version` straight from + `share/version`, and inherits `QUARTO_SHARE_PATH` and `QUARTO_DENO` + from the environment (`path_from_env`, `main.rs:21-23,50-52`) — the + strip list applies to it equally. + +**CI structure:** + +- `run-tests.sh` **never** uses PATH `quarto`; it hardcodes the harness + runtime at `package/dist/bin/tools//deno` + + `package/dist/bin/deno_cache` + `src/import_map.json` + (`run-tests.sh:47,57,164`), all produced only by `configure.sh`. + **Consequence: the `quarto-dev` action must run in every CI mode** — + it provisions the harness runtime; the built binary is added *on top* + (PATH override + `QUARTO_TEST_BIN`), not substituted. (Corrects v1 §4.0, + which framed the setup modes as mutually exclusive.) +- The dev symlink lands in `/usr/local/bin` + (`package/src/common/dependencies.ts:87-127`); a later `$GITHUB_PATH` + prepend shadows it for all subsequent steps, so + `quarto install tinytex/verapdf/chrome-headless-shell` + (`test-smokes.yml:236,243,251`) and PATH-based tests automatically use + the binary under test. `merge-extension-tests` is a plain `cp -r` — mode + independent. +- `prepare-dist` writes only to `package/pkg-working` (disjoint from + `package/dist`; `package/src/common/config.ts:65-89`) and removes only + `package/dist/config` (unused by `run-tests.sh`), so build and test can + share a checkout; separate jobs remain preferable for artifact reuse and + a future OS matrix. `configure.sh` + `prepare-dist` need network (public + URLs) but **no secrets** and no signing for the linux tarball. + +## 4. Design — harness "binary mode" + +Activated by `QUARTO_TEST_BIN=` (on Windows prefer `quarto.exe` — what releases ship, §3). +Unset ⇒ current behavior, byte-for-byte unchanged. + +### 4.1 The execution seam (`tests/test.ts` + a shared helper) + +New helper (e.g. `tests/quarto-cmd.ts`): + +```ts +export function binaryMode(): string | undefined => + Deno.env.get("QUARTO_TEST_BIN") || undefined; + +// Single dispatch point used by test.execute() AND all direct call sites. +export async function runQuarto( + args: string[], + options?: { env?: Record; cwd?: string; + logFile?: string; timeoutMs?: number }, +): Promise +``` + +- **Dev branch** (no `QUARTO_TEST_BIN`): call in-process `quarto(args, + undefined, options?.env)` exactly as today. +- **Binary branch**: `new Deno.Command(bin, { args: [...args, "--log", + logFile, "--log-format", "json-stream", "--log-level", "info"], cwd, + env: , clearEnv: true })`. + - Reuse the temp log path already created at `test.ts:243`; in binary + mode skip `initializeLogger` for capture but keep the harness able to + append its own ERROR records (spawn failure, timeout) to the same file + so existing reporting works. + - **Do not throw on non-zero exit** — the binary already wrote its ERROR + record (§3); mirrors today's catch-and-log at `test.ts:262-266`. + - Timeout: `setTimeout` → `child.kill("SIGKILL" /* windows: kill() */)`, + then append a timeout ERROR record. (Improvement over dev mode, where + a timed-out render keeps running.) + - `--log*` flags are appended **only** here — never for tests that spawn + quarto themselves and own their flags + (`logging/log-level-and-formats.test.ts` tests exactly those flags). +- `testQuartoCmd`'s `cwd` context keeps chdir-ing the harness process + (relative-path verifiers and teardowns depend on it) *and* passes cwd to + the subprocess. +- Startup guard, once per run: if `QUARTO_TEST_BIN` is set, execute + ` --version`; **fail hard** if it reports `99.9.9` (dev-mode trap, + §3) or doesn't match `QUARTO_TEST_EXPECTED_VERSION` when provided. + Print the version banner. + +### 4.2 Env contract for the spawned binary + +With `clearEnv: true`, construct the child env as: + +1. **Base pass-through from ambient env** (needed for engines/toolchains): + `PATH`, `HOME`/`USERPROFILE`, `TMPDIR`/`TEMP`/`TMP`, `LANG`/`LC_*`, + `VIRTUAL_ENV`, `PYTHONPATH`, `R_LIBS*`, `RENV_*`, `JULIA_*`, + `GH_TOKEN`, CI markers (`CI`, `GITHUB_ACTIONS`), proxy vars, and the + toolchain overrides `QUARTO_R`, `QUARTO_PYTHON`, `QUARTO_TYPST`, + `QUARTO_ESBUILD`, `QUARTO_DART_SASS`, `QUARTO_CHROMIUM`, + `QUARTO_TEXLIVE_BINPATH`, `QUARTO_TINYTEX_REPOSITORY`, + `QUARTO_KNITR_RSCRIPT_ARGS` (audit: legitimately CI-environment vars, + not dev-tree vars — stripping them breaks every engine test). +2. **Never pass** (dev-tree identity; the strip list): + `QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, `QUARTO_DEBUG`, `DENO_DIR`, + `QUARTO_DENO`, `DENO_DOM_PLUGIN` (both launchers inherit these if + set — bash `common/quarto:167-179`, Rust `main.rs:50-60`), + `QUARTO_ROOT`, `QUARTO_SRC_PATH`, `QUARTO_FORCE_VERSION`, + `QUARTO_VERSION_REQUIREMENT`, `QUARTO_PROJECT_DIR` (stale-leak guard), + `RSTUDIO` (must be affirmatively absent for the "not RStudio" test). +3. **Overlay `context.env` last** (per-test intent wins): e.g. + `QUARTO_PROFILE`, `QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES/_OUTPUT_FILES`, + `QUARTO_PDF_STANDARD`, `QUARTO_LOG_LEVEL`, `RSTUDIO=1`, `LUA_PATH`. + +The exact allowlist is finalized in Phase 0 when real failures show what's +missing; the shape (allowlist + strip + overlay) is fixed. + +### 4.3 smoke-all driver changes (`tests/smoke/smoke-all.test.ts`) + +- Route the project pre-render (`:434`) and the `editor-support-crossref` + pseudo-format (2 docs) through `runQuarto`. +- Everything else (discovery, `_quarto.tests` parsing, `verifyMap` + dispatch, cleanup, `run:` skip logic) is harness-side and unchanged. +- The YAML-intelligence bootstrap stays (harness-process-only). + +### 4.4 Runner plumbing (`run-tests.sh` / `run-tests.ps1`) + +When `QUARTO_TEST_BIN` is set (or `--bin ` given): + +- Still resolve the repo Deno + import map — the harness always needs them. +- **Do not export** `QUARTO_SHARE_PATH` / `QUARTO_DEBUG` (defense in depth + on top of `clearEnv`; also keeps the harness's own process honest). +- Default no-args test selection must **exclude `unit/`** (dev-only by + definition) — e.g. pass the `smoke/` tree explicitly instead of letting + `deno test` discover everything. +- Print ` --version` banner; refuse `99.9.9` (§4.1 guard). + +### 4.5 Test adaptations (from the 133-file classification, §8) + +- **Shared-helper migrations** (~14 files importing `src/quarto.ts`): + replace direct `quarto()` calls with `runQuarto` (setup pre-renders: + `render-freeze`, `render-format-extension`, + `render-output-file-collision`, `extension-render-{journals, + typst-templates}`; bodies: `crossref/syntax`, `convert/issue-12318`, + `jupyter/cache`, `self-contained/stdout`, `issues/9133`). Trivial + `testQuartoCmd` rewrites where possible (`jupyter/issue-10097`, + `issue-12374`). `engine/invalid-engine-in-project` converts from + `assertRejects` (currently un-awaited — a latent bug) to exit-code + + ERROR-log assertion. +- **`quartoDevCmd()` one-liner** (`tests/utils.ts:244`): return + `QUARTO_TEST_BIN` when set — migrates `run/*`, `lua-unit`, `logging`, + `create` wholesale; consolidate the hardcoded + `../package/dist/bin/quarto` spawns (`filters/editor-support`, + `typst-gather`) onto it. +- **Semantic one-offs**: `env/check.test.ts` — compute expected version + from the binary instead of hardcoding `99.9.9`; + `inspect-standalone-rstudio` — `RSTUDIO=1` via `context.env` in binary + mode (the in-process `_setIsRStudioForTest` hook stays for dev mode). +- **Anti-pattern fix**: `website/drafts-env.test.ts` converts + `Deno.env.set("QUARTO_PROFILE", ...)` to `context.env` (already flagged + by `.claude/rules/testing/test-anti-patterns.md`; would become + cross-subprocess leakage in binary mode). +- **Move, don't flag**: the 2 genuinely dev-only files + (`yaml-intelligence/yaml-intelligence.test.ts`, + `yaml-intelligence-folded-block-strings.test.ts`) move to `tests/unit/`. + `TestContext.requiresDevQuarto` is still added as an escape hatch + (sets Deno `ignore` in binary mode) but starts with zero users. +- **Guard rails** (lightweight lint/CI checks): forbid + `import ... from ".*src/quarto.ts"` under `tests/smoke/`; forbid + quarto spawns not routed through `quartoDevCmd()`/`runQuarto`. + +### 4.6 Fixture fixes + +- Relax the ten `quarto-required: '>=99.9.0'` fixture extensions to + `'>=1.9'` (evidence in §3; matches their own sibling extensions). +- These are safe, standalone commits that can land **before** any harness + work (they are no-ops for dev mode since `99.9.9 >= 1.9`). + +## 5. CI design + +### 5.1 Parameterize `test-smokes.yml` + +New `workflow_call` inputs: `quarto-install` (`dev`|`release`|`artifact`, +default `dev`), `quarto-version` (for `release`), `quarto-artifact-name` +(for `artifact`), and `ref` (checkout ref, so callers can pin harness = +binary commit). + +Step changes — all **additive**, inserted after the existing `quarto-dev` +step (`test-smokes.yml:230`), which now runs **unconditionally in every +mode** (it provisions the harness Deno runtime, §3): + +```yaml +- uses: ./.github/workflows/actions/quarto-dev # ALWAYS (harness runtime) + +- name: Set up release quarto # release mode + if: inputs.quarto-install == 'release' + uses: quarto-dev/quarto-actions/setup@v2 + with: { version: "${{ inputs.quarto-version }}" } + +- name: Download built quarto artifact # artifact mode + if: inputs.quarto-install == 'artifact' + uses: actions/download-artifact@v7 + with: { name: "${{ inputs.quarto-artifact-name }}" } +- name: Install built quarto outside the checkout # artifact mode + if: inputs.quarto-install == 'artifact' + run: | # extract OUTSIDE repo (§3 dev-mode trap) + mkdir -p "$RUNNER_TEMP/quarto-under-test" + tar -xzf built-quarto-*.tar.gz -C "$RUNNER_TEMP/quarto-under-test" --strip-components=1 + echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" + +- name: Pin and verify test target # both non-dev modes + if: inputs.quarto-install != 'dev' + run: | + v="$(quarto --version)" + [ "$v" != "99.9.9" ] || { echo "dev sentinel detected"; exit 1; } + echo "QUARTO_TEST_BIN=$(command -v quarto)" >> "$GITHUB_ENV" +``` + +Everything downstream is untouched: `quarto install tinytex/verapdf/…` +pick up the PATH override automatically; `run-tests.sh` reads +`QUARTO_TEST_BIN` from the env; language setup is shared. + +### 5.2 New `test-smokes-built.yml` + +```yaml +name: Smoke Tests (Built Version) +on: + schedule: [{ cron: "0 8 * * 1" }] # weekly, Monday + workflow_dispatch: + inputs: + source: { type: choice, options: [build, release], default: build } + version: { type: string, default: "pre-release" } # for source: release + +jobs: + build-artifact: # source: build (default; also the schedule path) + if: inputs.source != 'release' + runs-on: ubuntu-latest + outputs: { sha: "${{ steps.rec.outputs.sha }}" } + steps: + - uses: actions/checkout@v6 + - id: rec + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - run: ./configure.sh + - run: | + pushd package/src + ./quarto-bld prepare-dist \ + --set-version "$(cat ../../version.txt).$(date +%Y%m%d)-test" \ + --log-level info + popd + - run: | # same recipe as make-tarball + pushd package + mv pkg-working quarto-built-test + tar czf built-quarto-linux-amd64.tar.gz quarto-built-test + mv quarto-built-test pkg-working + popd + - uses: actions/upload-artifact@v7 + with: { name: built-quarto-linux-amd64, + path: package/built-quarto-linux-amd64.tar.gz } + + run-smokes-artifact: + if: inputs.source != 'release' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + ref: "${{ needs.build-artifact.outputs.sha }}" # harness == binary commit + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + + run-smokes-release: # dispatch-only, curative/backfill + if: inputs.source == 'release' + uses: ./.github/workflows/test-smokes.yml + with: + ref: "refs/tags/v${{ }}" # resolve pre-release → concrete tag first + quarto-install: release + quarto-version: "${{ inputs.version }}" +``` + +Notes: + +- The version marker embeds `-test` so the built binary can never be + mistaken for a published build; it also keeps `quarto --version` ≠ + `99.9.9`, satisfying the guard. +- Platform: **linux-amd64 first** (Windows zip needs the signing pipeline; + Windows coverage comes via `source: release`, or later an unsigned local + zip build). +- `release` mode needs a small resolve step (input `pre-release`/`release` + → concrete `1.x.y` via `_prerelease.json`/`_download.json` or the GitHub + API) before computing the checkout tag. +- Full run duration: the daily `test-smokes.yml` scheduled run already + does a full un-sharded run per OS, so a weekly built-mode equivalent is + a known, bounded cost; the `run-parallel-tests` bucket matrix remains + available if needed. +- Later (Phase 4): call the same parameterized workflow from + `create-release.yml` as an initially non-blocking gate before + `publish-release`. + +## 6. Phased roadmap + +**Phase 0 — spike (≈1 day).** Locally: `configure.sh` + +`quarto-bld prepare-dist`, copy `pkg-working` **outside the checkout** +(§3 dev-mode trap), hack the seam, run ~20–30 representative smoke-all +docs + a few `.test.ts` smokes. Acceptance: catalog of failure classes; +confirmation the log-file contract holds end-to-end; finalized env +allowlist. + +**Phase 1 — fixtures + harness binary mode.** Land the ten +`quarto-required` relaxations and the `drafts-env` anti-pattern fix +(standalone, dev-mode no-ops). Implement `runQuarto`, the env contract, +`quartoDevCmd()` change, runner plumbing (incl. `unit/` exclusion), +version guard, `requiresDevQuarto`, the ~14 shared-helper migrations, the +2 file moves to `tests/unit/`. Acceptance: full dev-mode suite still +green (default path untouched); a binary-mode subset green locally. + +**Phase 2 — CI.** Parameterize `test-smokes.yml` (additive inputs); +add `test-smokes-built.yml` (weekly + dispatch, build-then-test, full +smoke-all, linux). Acceptance: first green (or triaged) weekly run; +failures classified into product bugs / harness assumptions / dev-only. + +**Phase 3 — broaden.** `source: release` path (Windows coverage, version +backfill), ff-matrix bucket job, guard-rail lint checks. Acceptance: +dispatch run against latest published prerelease succeeds on both OSes. + +**Phase 4 — optional.** Non-blocking gate in `create-release.yml`; +decouple harness from `src/` (via `quarto inspect`) only if testing +binaries from a *different* commit than the harness ever matters. + +## 7. Risks & open items + +- **Subprocess startup cost** (~100–300 ms × hundreds of renders): small + vs render time; measure in Phase 0. +- **`shouldError` + passthrough commands**: `run`/`pandoc`/`typst` + failures exit non-zero without a log ERROR record (§3); their tests + already use `execProcess` and don't rely on log verifiers — document + the constraint in the seam. +- **Behavioral diffs without `QUARTO_DEBUG`**: ERROR messages lose stack + traces; no known verifier depends on them; Phase 0 confirms. +- **`quarto check` dev branch**: with a real version the + `git rev-parse`/dev branch is skipped — `check`-based tests + (`smoke/check`, `env/check`) assert against installed behavior; adapt + expectations (§4.5). +- **Intra- vs inter-process concurrency**: `issues/9133` reproduces an + in-process race; the two-subprocess variant needs a runtime check that + the regression still triggers. +- **`configure.sh` cost in binary-mode CI**: it still downloads the full + dev toolchain (pandoc, dart-sass, …) the binary won't use — wasted + minutes, correctness-neutral; optional later flag in `quarto-bld + configure`. +- **Skip-list drift**: guard-rail greps (§4.5) + `requiresDevQuarto` + review-time convention. --- -## 7. Classification sweep results (2026-07-17) +## 8. Appendix — classification sweep (2026-07-17) -All 133 `tests/smoke/**/*.test.ts` files were read in full and classified -for binary-mode compatibility: **107 compatible / 24 adapt / 2 dev-only**. +All 133 `tests/smoke/**/*.test.ts` read in full: **107 compatible / 24 +adapt / 2 dev-only**. -### 7.1 Compatible directories (no changes needed) +### 8.1 Compatible as-is (107) +Everything funneling through `testQuartoCmd` or its wrappers +(`testRender`, `testSite`, `testProjectRender`, `testManuscriptRender`): `render` (28/31), `crossref`+`site`+`website` (25/26), `project` (8/8), -`inspect` (5/6), `extensions` (5/7), `yaml`, `ojs`, `use`, `verify`, `jats`, -`book`, `shortcodes`, `search`, `scholar`, `manuscript`, `embed`, `authors`, -`build-ts-extension`, `check`, and more — everything funneling through -`testQuartoCmd` or its wrappers (`testRender`, `testSite`, -`testProjectRender`, `testManuscriptRender`). Their `src/` imports are -expectation/path/cleanup helpers only. `TestContext.env` is already passed -as a parameter (not `Deno.env.set`), so it forwards cleanly to a subprocess -— binary mode actually *improves* isolation for env-dependent tests. - -### 7.2 The 24 "adapt" files — four mechanical patterns - -**(a) Direct `quarto()` import from `src/quarto.ts`** (~14 files; greppable -via `from ".*src/quarto.ts"`). Setup-side pre-renders or multi-step bodies: -`render-freeze`, `render-format-extension`, `render-output-file-collision`, -`crossref/syntax`, `extensions/extension-render-{journals,typst-templates}`, -`convert/issue-12318`, `jupyter/{cache,issue-10097,issue-12374}`, -`engine/invalid-engine-in-project`, `self-contained/stdout`, `issues/9133`, -and `smoke-all.test.ts` itself (project pre-render). Fix: one shared -`runQuarto(args, {env, cwd})` helper dispatching to in-process `quarto()` -or a `QUARTO_TEST_BIN` spawn; three of these are trivial rewrites to plain -`testQuartoCmd`. Notes: `invalid-engine-in-project` asserts on a thrown -Error (convert to exit-code + ERROR-log assertion; its `assertRejects` is -currently not awaited, so it silently passes today — a pre-existing bug); -`issues/9133` reproduces an *intra*-process concurrency bug, so the -two-subprocess version needs a runtime check that it still triggers. - -**(b) PATH-quarto subprocess via `quartoDevCmd()`** (`run/*` ×3, -`lua-unit`, `logging`, `create`): a **one-line fix** — `quartoDevCmd()` -(`tests/utils.ts:244`) returns `QUARTO_TEST_BIN` when set. These tests -already spawn a real binary; several (e.g. `stdlib-run-version`, -`lua-unit`) arguably *belong* in binary mode since they verify the shipped -`quarto run` stdlib/embedded deno. - -**(c) Hardcoded `../package/dist/bin/quarto` spawns** -(`filters/editor-support`, `typst-gather` tests 7–12): consolidate onto the -patched `quartoDevCmd()`. - -**(d) Semantic one-offs**: `env/check.test.ts` hardcodes `Version: 99.9.9` -(compute expectation from the binary, or relax the regex); -`inspect/inspect-standalone-rstudio.test.ts` uses the in-process -`_setIsRStudioForTest` hook (in binary mode, set `RSTUDIO=1` in the child -env instead — simpler than today; the companion "not RStudio" test requires -the harness to spawn with a *clean* env). - -Implementation caveats surfaced by the sweep: -- `testQuartoCmd`'s `cwd` option must keep chdir-ing the **harness** - process too (relative-path verifiers and teardowns depend on it), while - also setting the subprocess cwd. -- Binary mode's `--log/--log-format` injection applies only to - `testQuartoCmd`-driven invocations — never to tests that spawn quarto - themselves and own their flags (`logging/log-level-and-formats` exists - precisely to test those flags). - -### 7.3 Genuinely dev-only (2 files) +`inspect` (5/6), `extensions` (5/7), `yaml`, `ojs`, `use`, `verify`, +`jats`, `book`, `shortcodes`, `search`, `scholar`, `manuscript`, `embed`, +`authors`, `build-ts-extension`, `check`, and more. Their `src/` imports +are expectation/path/cleanup helpers only. `TestContext.env` is already a +parameter (not `Deno.env.set`) and forwards cleanly. + +### 8.2 Adapt (24) — four mechanical patterns + +(a) direct `quarto()` import (~14 files; greppable) → `runQuarto` helper / +trivial `testQuartoCmd` rewrites; (b) `quartoDevCmd()` PATH spawns +(`run/*`, `lua-unit`, `logging`, `create`) → one-line env-var switch; +(c) hardcoded `../package/dist/bin/quarto` spawns (`filters/ +editor-support`, `typst-gather` 7–12) → consolidate onto (b); (d) +semantic one-offs (`env/check` 99.9.9 expectation; +`inspect-standalone-rstudio` in-process hook → `RSTUDIO=1` child env). +Details and per-file notes in §4.5. Note: "uses `unitTest()`" is NOT a +dev-only signal — several files use it as a generic wrapper around +subprocess spawns. + +### 8.3 Dev-only (2) `yaml-intelligence/yaml-intelligence.test.ts` and -`yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts` — -`unitTest()` calls exercising `src/core/lib/yaml-intelligence` internals -with no CLI surface. **Recommendation: move them to `tests/unit/`** (the -other two files in `smoke/yaml-intelligence/` are ordinary render tests and -stay). With that move, *zero* smoke tests need `requiresDevQuarto` today; -the flag remains as an escape hatch for future tests. - -**Reorganization verdict:** a folder/name convention (e.g. `smoke-dev/`, -`*.dev.test.ts`) is not worth it — dev-only-ness is too rare. Caution: -"uses `unitTest()`" is NOT a reliable dev-only signal (`run/*`, -`lua-unit`, `typst-gather` use it as a generic wrapper around subprocess -spawns). The reliable, greppable signals are: (a) `import ... from -".*src/quarto.ts"` in a test file, (b) spawns not routed through -`quartoDevCmd()`. Enforce both as lightweight lint/CI rules so new tests -stay binary-compatible by construction. - -### 7.4 Smoke-all documents audit - -The docs themselves are almost entirely binary-clean: no doc executes -quarto from a code cell, no runtime dev-tree paths, no pre/post-render -scripts in fixture `_quarto.yml`s, and **every `printsMessage` assertion -uses INFO/WARN/ERROR — never DEBUG** — matching a built binary's default -log level. The json-stream record shape from `--log --log-format -json-stream` is identical to what `readExecuteOutput` parses, and -`--quiet` does not affect the file handler. - -**One systemic blocker:** ten fixture extensions declare -`quarto-required: '>=99.9.0'`, which passes only against the dev sentinel -version — a real-version binary hard-errors in `validateExtension` -(`src/extension/extension.ts:776-788`). Affected fixtures: the -`dragonstyle/lipsum` copies under `dashboard/`, `lightbox/`, -`format/html/`, `2023/04/24/`; the brand/typst extension fixtures under -`typst/brand-yaml/typography/`, `typst/font-paths/`, -`brand/typography/remote-font-extension/`, `brand/logo/logo-extension*/`; -and `2023/01/06/input-relative/`. Two options: -- **Relax the fixtures to `'>=1.9'`** (preferred — keeps the binary's real - version visible to everything else), unless a doc specifically tests - version gating; -- or export `QUARTO_FORCE_VERSION=99.9.9` into the child env (honored at - `src/core/quarto.ts:35`) — a blunter tool that masks real version - behavior. - -Driver adaptations beyond plain render: route the `editor-support-crossref` -pseudo-format (2 docs) and the 24 `render-project` pre-renders through the -binary; tolerate non-zero child exit (read the log file regardless — the -binary's top-level handler writes the ERROR record `shouldError` needs). -Runtime one-time checks: `engine/class-override` extensions -(`quarto-required '>=1.9.17'` — fails on older release binaries by design), -`QUARTO_EXECUTE_INFO` / `QUARTO_PROJECT_ROOT` env-var docs, and the +`yaml-intelligence-folded-block-strings.test.ts` — in-process +yaml-intelligence internals with no CLI surface → move to `tests/unit/`. + +### 8.4 Smoke-all documents + +Binary-clean except the ten `quarto-required: '>=99.9.0'` fixtures +(§4.6): no doc executes quarto from a code cell, no runtime dev-tree +paths, no pre/post-render scripts, every `printsMessage` uses +INFO/WARN/ERROR (never DEBUG). Driver adaptations: project pre-renders +(24 docs) and `editor-support-crossref` (2 docs) through `runQuarto`; +tolerate non-zero child exit. One-time runtime checks: +`engine/class-override` (`>=1.9.17` — correctly fails on older release +binaries), `QUARTO_EXECUTE_INFO`/`QUARTO_PROJECT_ROOT` env docs, the pandoc-args INFO echo docs (`2025/12/09/13775-*`). From 1c30286bccb8580931617a66957897b4cb249fd7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 12:06:42 +0000 Subject: [PATCH 05/43] Fold adversarial review findings into built-version test plan (v3) Five-lens adversarial review (19 agents, every critical/major finding independently verification-checked); 32 findings stood, none refuted. Design changes: - Version marker must use semver BUILD metadata (+test.YYYYMMDD): the vendored semver throws on the 4-component string the v2 sketch produced, and any '-prerelease' marker fails all >=X.Y ranges (verified by executing deno.land/x/semver@v1.4.0), which would have broken every quarto-required gate including the freshly relaxed fixtures. - Silent-green invariant: a child failing before logger init (deno startup, bundle load, missing share) or via commandFailed paths exits non-zero with an empty log that vacuously passes noErrorsOrWarnings (~23% of the 1424-doc corpus is log-only); runQuarto must synthesize an ERROR record for record-free non-zero exits. - Keep QUARTO_SHARE_PATH exported for the harness process in all modes (getenv throws when unset, killing smoke-all at module load); child isolation via spawn-time strip only. - Env mechanism flipped from clearEnv+allowlist to inherit+strip (Windows system-var surface too large to enumerate safely); QUARTO_DENO_DOM corrected as the real leak vector. - runQuarto gains throwOnFailure (direct/setup call sites keep throw semantics; test.execute uses no-throw), process-tree timeout kill (launchers spawn-and-wait, so killing the child orphans deno and corrupts the two-writer log), piped+drained stdout/stderr with stderr tail in failure reports, and logger-lifecycle branching in test(). - CI: runners input to restrict artifact mode to linux (Windows leg would download a Linux tarball), buckets becomes optional, resolve job for release mode, fork guard, Windows setup-step gates extended to non-dev full runs, release mode scoped to post-Phase-1 tags (older tags lack harness plumbing and local composite actions). - Corrected citations: nightly create-release runs are build-only (publishing comes from dispatch), prepare-dist also regenerates src/ artifacts (separate build/test jobs are a requirement), dev-symlink location, local configure-test-env PATH-quarto nuance. Co-Authored-By: Claude Fable 5 --- dev-docs/smoke-tests-built-version-plan.md | 698 ++++++++++++--------- 1 file changed, 414 insertions(+), 284 deletions(-) diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index 55124fe019..338d4b6e57 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -1,6 +1,7 @@ # Plan: Running Smoke Tests Against a Built Quarto -Status: **detailed design — v2** (grounded; pending adversarial review + Phase 0 spike). +Status: **detailed design — v3** (grounded + adversarially reviewed; +ready for Phase 0 spike). Goal: run the existing smoke test suites — the full smoke-all corpus first, plus the `.test.ts` smokes and the feature-format matrix — against a @@ -16,6 +17,11 @@ Decisions (maintainer, 2026-07-17): published prereleases (that stays as a secondary dispatch input). - No release-pipeline gating until the mode has a track record. +Review history: v2 was attacked by a 5-lens adversarial review (19 agents; +every critical/major finding independently verification-checked). All 32 +standing findings are folded into this v3; the design-changing ones are +marked **[R]** below. + --- ## 1. How testing works today (established facts) @@ -36,8 +42,9 @@ Decisions (maintainer, 2026-07-17): `QUARTO_BIN_PATH=package/dist/bin` (`:47`), `QUARTO_SHARE_PATH=src/resources` (`:60`), `QUARTO_DEBUG=true` (`:61`), `DENO_DIR=package/dist/bin/deno_cache` (`:50-54`). -- `smoke-all.test.ts` globs `docs/smoke-all/**/*.{md,qmd,ipynb}`, parses - `_quarto.tests` with dev-source YAML code, dispatches spec keys to +- `smoke-all.test.ts` globs `docs/smoke-all/**/*.{md,qmd,ipynb}` + (**1424 documents** as of 2026-07-17, many with multiple format specs), + parses `_quarto.tests` with dev-source YAML code, dispatches spec keys to verifiers via an inline `verifyMap`, and renders via `testQuartoCmd("render", [input, "--to", format])` (`:462`); project pre-renders call `quarto(["render", projectPath])` directly (`:434`). @@ -45,30 +52,38 @@ Decisions (maintainer, 2026-07-17): `test-ff-matrix.yml:44-50` calls `test-smokes.yml` with a qmd glob that `run-tests.sh:132-164` routes to `smoke-all.test.ts`. Anything that fixes smoke-all fixes the matrix. -- `create-release.yml` runs nightly: builds per-OS artifacts, uploads them - as workflow artifacts, publishes GitHub Release assets (prerelease by - default) and tags `v` on the exact commit built - (`create-release.yml:88-100,678-703`). The linux tarball recipe is just +- `create-release.yml` builds per-OS artifacts and uploads them as workflow + artifacts. **[R]** The nightly `schedule` runs are build-only (the + tag-creating commit and the whole `publish-release` job are gated on + `inputs.publish-release`, and `inputs` is empty on schedule events); + published + `v`-tagged (pre)releases come from + `workflow_dispatch` runs, where `publish-release` defaults to true + (`create-release.yml:88-100,678-703`). The linux tarball recipe is `./configure.sh` → `quarto-bld prepare-dist --set-version ` → tar of `package/pkg-working` (`create-release.yml:136-159`). ## 2. Strategy -1. **The dev-tree coupling is concentrated in one seam** — the body of - `test.execute()` (in-process `quarto()` call + programmatic logger init). - The verifier layer and the `_quarto.tests` dispatch consume only the - json-stream log file and files on disk, both reproducible by a - subprocess via `--log --log-format json-stream --log-level info`. +1. **The dev-tree coupling is concentrated in the execution path** — the + body of `test.execute()` plus the logger lifecycle around it in + `test()` (two touch points, both in `tests/test.ts`). The verifier + layer and the `_quarto.tests` dispatch consume only the json-stream log + file and files on disk, both reproducible by a subprocess via + `--log --log-format json-stream`. 2. **Version skew is avoided by construction.** The primary workflow builds the artifact from the same SHA the harness checks out. For - published-release testing, check out `refs/tags/v`. The harness - keeps importing `src/` for *parsing and path math* only; *execution* - moves to the binary. + published-release testing, check out `refs/tags/v` — with the + scope limitation in §5.3 **[R]**. The harness keeps importing `src/` + for *parsing and path math* only; *execution* moves to the binary. -## 3. Ground truths verified in code (2026-07-17) +**What this design does NOT test [R]:** signing/notarization, MSI/pkg/deb +installer logic and installer PATH setup, Cloudsmith packages, arm64 +tarballs, and (until Phase 3 release mode) the Windows Rust launcher. A +green "Smoke Tests (Built Version)" badge means the linux-amd64 built +*layout* renders correctly — not that release artifacts are safe +end-to-end. Keep this list next to any badge/gate. -These were verified by dedicated review passes with file:line evidence; -the design in §4–§5 depends on them. +## 3. Ground truths verified in code (2026-07-17) **Log pipeline (CONFIRMED):** @@ -82,23 +97,34 @@ the design in §4–§5 depends on them. - json-stream writes one `JSON.stringify(logRecord)` per line with `msg`/`level`/`levelName` (`log.ts:262-263`) — byte-compatible with `readExecuteOutput`. The file handler **flushes after every record** - (`log.ts:267-273`), so even a hard crash loses nothing already logged. + (`log.ts:267-273`). - On render failure an ERROR record is written before exit 1: errors propagate to `mainRunner`'s catch → `logError(e)` (`main.ts:70-73`) → ERROR record in the file; then `exitWithCleanup(1)` (`main.ts:74-81`). `CommandError` likewise (`src/quarto.ts:202-208`). -- **Exception:** the `pandoc`/`typst`/`run` passthroughs `Deno.exit(code)` - with the child's code and route child stderr directly — a failing - passthrough writes no ERROR record to the log file (`src/quarto.ts:100, - 119, 139`). The tests covering these already use `execProcess` and assert - on stdout/stderr/exit code, so this is a documentation caveat for the - seam, not a blocker. - -**`QUARTO_FORCE_VERSION=99.9.9` is rejected (NUANCED → do not use):** -honored at `src/core/quarto.ts:34-45` but `99.9.9` equals -`kLocalDevelopment`, which (a) satisfies every lower-bound version gate -(extensions `src/extension/extension.ts:776-788`, document -`quarto-required` `src/command/render/render-files.ts:130-144`, engines +- **[R] But "non-zero exit ⇒ ERROR record in the log" is NOT an + invariant.** It fails for (a) any child failure *before* `mainRunner`'s + logger init — deno startup errors, bundle/import-resolution failures + loading the esbuild `quarto.js`, missing modules (`main.ts:26-27` runs + after the module graph loads; the launcher just execs deno, + `common/quarto:205-209`); (b) `quarto add`/`remove` failures via + `commandFailed()`/`signalCommandFailure` → `exitWithCleanup(1)` with + only INFO output (`src/quarto.ts:199-201`, + `src/command/add/cmd.ts:48-51,59-61`); (c) the `pandoc`/`typst`/`run` + passthroughs, which `Deno.exit(code)` and route child stderr directly + (`src/quarto.ts:100,119,139`). Because the harness pre-creates the log + file, an untouched log parses to `[]` and `noErrors`/ + `noErrorsOrWarnings` pass **vacuously** — and ~23% of smoke-all docs + (326/1424) use the default log-only spec with no file verifier. The + seam therefore MUST synthesize an ERROR record for record-free non-zero + exits (§4.1) — otherwise the exact failure class binary mode exists to + catch (broken bundle, missing share) reports green on those docs. + +**`QUARTO_FORCE_VERSION=99.9.9` is rejected (do not use):** honored at +`src/core/quarto.ts:34-45` but `99.9.9` equals `kLocalDevelopment`, which +(a) satisfies every lower-bound version gate (extensions +`src/extension/extension.ts:776-788`, document `quarto-required` +`src/command/render/render-files.ts:130-144`, engines `src/execute/engine.ts:62-80`, freezer `src/core/cache/cache.ts:81`) — masking real "version too old" behavior — and (b) flips `quarto check` into a dev-mode branch that shells `git rev-parse` in `$QUARTO_ROOT` @@ -112,81 +138,92 @@ version truncated to `major.minor.0` (`99.9.9`) that yields exactly `>=99.9.0`. All ten fixtures were authored on dev checkouts (four separate lipsum vendorings 2023; brand/typst fixtures 2025–2026). No test anywhere exercises the version-gate error -path (zero test references to `quarto-required` / "incompatible with this -quarto"); sibling extensions in the same `_extensions/` dirs use -`>=1.3.0`, as does upstream `quarto-ext/lipsum` and Quarto's own bundled -copy (`src/resources/extensions/quarto/lipsum/_extension.yml:4`). **All -ten are safe to relax.** +path; sibling extensions in the same `_extensions/` dirs use `>=1.3.0`, +as does upstream `quarto-ext/lipsum` and Quarto's own bundled copy +(`src/resources/extensions/quarto/lipsum/_extension.yml:4`). **All ten +are safe to relax.** + +**[R] Semver behavior of version markers (verified by executing the +vendored `deno.land/x/semver@v1.4.0`):** `satisfies()` **throws** on +non-semver strings (no try/catch around `new SemVer` in `Range.test`, +unlike node-semver), and **prerelease versions do not satisfy plain +ranges** (`1.10.16-test` fails `>=1.9`; no call site passes +`includePrerelease`). Only **build metadata** (`1.10.15+test.20260717`) +is both valid and range-transparent. This dictates the §5.2 version +marker. **Launcher & env behavior:** -- The installed launcher inherits `QUARTO_SHARE_PATH` and `QUARTO_DEBUG` - if already set (`package/scripts/common/quarto:102-110, 68-70`; - `package/scripts/windows/quarto.cmd:80-81, 47`), recomputes - `QUARTO_ROOT`/`QUARTO_BIN_PATH` unconditionally, and **never sets - `DENO_DIR` outside dev mode** — so a leaked dev `DENO_DIR` reaches the - binary's deno. Strip list follows in §4.2. -- **Dev-mode trap:** the launcher decides dev vs installed by finding a - sibling `src/quarto.ts` relative to its own path - (`common/quarto:22-37`). Therefore `QUARTO_TEST_BIN` pointing at the - in-repo `package/dist/bin/quarto` silently runs **dev mode** (TS - sources, `--check`, dev env defaults) — not the built layout. The dist - must be extracted **outside the git checkout**, and the seam must fail - loudly if ` --version` reports `99.9.9`. -- A genuinely installed binary runs a single esbuild-bundled `quarto.js` - with `--no-check` (`common/quarto:91-121, 205-208`), a real - `share/version` file, inlined Lua filters, and arch-specific - deno/deno_dom (`package/src/common/prepare-dist.ts:128-147, 191-224`). - Binary mode therefore covers bundling/import-resolution errors, missing - share resources, version wiring, and the `--no-check` gap — none of - which dev mode can catch. -- `QUARTO_DEBUG=true` (dev default) only adds stack traces to ERROR `msg` - text (`src/core/log.ts:371-386`) plus dev-only reconfigure/watch paths. - Binary mode runs **without** it — truer to shipped behavior; ERROR - counts are unaffected, only `printsMessage` regexes matching stack text - could differ (none known in smoke-all: audit found all `printsMessage` - levels are INFO/WARN/ERROR with content-based regexes). +- The installed bash launcher inherits `QUARTO_SHARE_PATH` and + `QUARTO_DEBUG` if already set (`package/scripts/common/quarto:102-110, + 68-70`; `quarto.cmd:80-81, 47`), recomputes `QUARTO_ROOT`/ + `QUARTO_BIN_PATH` unconditionally, **never sets `DENO_DIR` outside dev + mode**, and inherits `QUARTO_DENO` and `QUARTO_DENO_DOM` + (`common/quarto:167-173`) — note it is `QUARTO_DENO_DOM` that is + inherited; `DENO_DOM_PLUGIN` itself is unconditionally overwritten by + both launchers **[R]**. +- **Dev-mode trap:** the bash/cmd launchers decide dev vs installed by + finding a sibling `src/quarto.ts` relative to their own path + (`common/quarto:22-37`). `QUARTO_TEST_BIN` pointing at the in-repo + `package/dist/bin/quarto` silently runs **dev mode** (TS sources, + `--check`, dev env defaults). The dist must be extracted **outside the + git checkout**, and the seam must fail loudly if ` --version` + reports `99.9.9`. - **Windows built layout ships two entry points.** `prepare-dist` copies `quarto.cmd` into the dist (`copyQuartoScript`, `package/src/common/configure.ts:143-149`); the `make-installer-win` job additionally builds and signs the **Rust launcher `quarto.exe`** - (`cargo build`, `package/launcher/src/main.rs`; - `create-release.yml:350-429`), which is what the MSI/zip put on PATH — - i.e. what Windows users actually run. `QUARTO_TEST_BIN` should target - `quarto.exe` for published-release testing (trivial spawn, no `.cmd` - handling); a `prepare-dist`-only Windows artifact has `quarto.cmd`, - which spawns **directly** with `Deno.Command` — no `cmd /c` (the repo's - established pattern: `quartoDevCmd()` `tests/utils.ts:244-246`; npm.cmd - in `src/project/serve/serve.ts:434`). Notes: the Rust launcher has **no - dev-mode branch** (always runs the bundled `quarto.js` — the dev-mode - trap below doesn't apply to it), reads `--version` straight from - `share/version`, and inherits `QUARTO_SHARE_PATH` and `QUARTO_DENO` - from the environment (`path_from_env`, `main.rs:21-23,50-52`) — the - strip list applies to it equally. + (`package/launcher/src/main.rs`; `create-release.yml:350-429`) — what + the MSI/zip put on PATH, i.e. what Windows users actually run. + `QUARTO_TEST_BIN` should target `quarto.exe` for published-release + testing; a `prepare-dist`-only Windows artifact has `quarto.cmd`, which + spawns **directly** with `Deno.Command` — no `cmd /c` (established + pattern: `quartoDevCmd()` `tests/utils.ts:244-246`). The Rust launcher + has **no dev-mode branch**, reads `--version` straight from + `share/version`, and inherits `QUARTO_SHARE_PATH`, `QUARTO_DENO`, and + `QUARTO_DENO_DOM` from the environment (`main.rs:21-23,50-60,65-68`). +- A genuinely installed binary runs a single esbuild-bundled `quarto.js` + with `--no-check` (`common/quarto:91-121, 205-208`), a real + `share/version` file, inlined Lua filters, and arch-specific + deno/deno_dom (`package/src/common/prepare-dist.ts:128-147, 191-224`). + Binary mode covers bundling/import-resolution errors, missing share + resources, version wiring, and the `--no-check` gap. +- `QUARTO_DEBUG=true` (dev default) adds stack traces to ERROR `msg` text + (`src/core/log.ts:371-386`) plus dev-only reconfigure/watch paths. + Binary mode spawns the child **without** it. ERROR counts are + unaffected; no smoke-all `printsMessage` depends on stack text. **CI structure:** -- `run-tests.sh` **never** uses PATH `quarto`; it hardcodes the harness - runtime at `package/dist/bin/tools//deno` + - `package/dist/bin/deno_cache` + `src/import_map.json` - (`run-tests.sh:47,57,164`), all produced only by `configure.sh`. - **Consequence: the `quarto-dev` action must run in every CI mode** — - it provisions the harness runtime; the built binary is added *on top* - (PATH override + `QUARTO_TEST_BIN`), not substituted. (Corrects v1 §4.0, - which framed the setup modes as mutually exclusive.) -- The dev symlink lands in `/usr/local/bin` - (`package/src/common/dependencies.ts:87-127`); a later `$GITHUB_PATH` - prepend shadows it for all subsequent steps, so +- `run-tests.sh` hardcodes the harness runtime at + `package/dist/bin/tools//deno` + `package/dist/bin/deno_cache` + + `src/import_map.json` (`run-tests.sh:47,57,164`), all produced only by + `configure.sh`. **Consequence: the `quarto-dev` action must run in + every CI mode** — it provisions the harness runtime; the built binary + is added *on top* (PATH override + `QUARTO_TEST_BIN`), not substituted. + **[R]** Nuance: this "never uses PATH quarto" claim is CI-scoped — + local runs source `configure-test-env.sh` (which calls PATH `quarto + install tinytex/verapdf`) unless `QUARTO_TESTS_NO_CONFIG` is set; the + Phase 0/1 local recipe must set it or order PATH deliberately. +- The dev quarto reaches PATH via a `/usr/local/bin` symlink on + Linux/macOS (`package/src/common/configure.ts:81-133`, + `suggestUserBinPaths`) and via a `GITHUB_PATH` append of + `package/dist/bin` on Windows CI **[R]**; in both cases a later + `$GITHUB_PATH` prepend shadows it for subsequent steps, so `quarto install tinytex/verapdf/chrome-headless-shell` (`test-smokes.yml:236,243,251`) and PATH-based tests automatically use - the binary under test. `merge-extension-tests` is a plain `cp -r` — mode - independent. -- `prepare-dist` writes only to `package/pkg-working` (disjoint from - `package/dist`; `package/src/common/config.ts:65-89`) and removes only - `package/dist/config` (unused by `run-tests.sh`), so build and test can - share a checkout; separate jobs remain preferable for artifact reuse and - a future OS matrix. `configure.sh` + `prepare-dist` need network (public - URLs) but **no secrets** and no signing for the linux tarball. + the binary under test. `merge-extension-tests` is a plain `cp -r` — + mode independent. +- **[R]** `prepare-dist` populates `package/pkg-working` (disjoint from + `package/dist`; `package/src/common/config.ts:65-89`) **but also + regenerates artifacts inside `src/`** (quarto-preview JS via + `build.ts --force`; schema/yaml-intelligence assets under + `src/resources` via `buildAssets()`), so a build job and a test run + sharing one checkout can race on the harness's own + `QUARTO_SHARE_PATH=src/resources`. Separate build/test jobs (the + chosen design) are the mitigation, not merely a preference. + `configure.sh` + `prepare-dist` need network (public URLs) but **no + secrets** for the linux tarball. ## 4. Design — harness "binary mode" @@ -194,78 +231,119 @@ Activated by `QUARTO_TEST_BIN=` (on Windows prefer `quarto.exe` — what releases ship, §3). Unset ⇒ current behavior, byte-for-byte unchanged. -### 4.1 The execution seam (`tests/test.ts` + a shared helper) - -New helper (e.g. `tests/quarto-cmd.ts`): +### 4.1 The execution seam (`tests/quarto-cmd.ts` + `tests/test.ts`) ```ts -export function binaryMode(): string | undefined => - Deno.env.get("QUARTO_TEST_BIN") || undefined; +export function binaryMode(): string | undefined; -// Single dispatch point used by test.execute() AND all direct call sites. export async function runQuarto( args: string[], - options?: { env?: Record; cwd?: string; - logFile?: string; timeoutMs?: number }, + options?: { + env?: Record; + cwd?: string; + logFile?: string; // json-stream target (binary mode) + timeoutMs?: number; + throwOnFailure?: boolean; // [R] default TRUE for direct call sites + logLevel?: string; // [R] honor per-test log intent + }, ): Promise ``` - **Dev branch** (no `QUARTO_TEST_BIN`): call in-process `quarto(args, undefined, options?.env)` exactly as today. -- **Binary branch**: `new Deno.Command(bin, { args: [...args, "--log", - logFile, "--log-format", "json-stream", "--log-level", "info"], cwd, - env: , clearEnv: true })`. - - Reuse the temp log path already created at `test.ts:243`; in binary - mode skip `initializeLogger` for capture but keep the harness able to - append its own ERROR records (spawn failure, timeout) to the same file - so existing reporting works. - - **Do not throw on non-zero exit** — the binary already wrote its ERROR - record (§3); mirrors today's catch-and-log at `test.ts:262-266`. - - Timeout: `setTimeout` → `child.kill("SIGKILL" /* windows: kill() */)`, - then append a timeout ERROR record. (Improvement over dev mode, where - a timed-out render keeps running.) - - `--log*` flags are appended **only** here — never for tests that spawn - quarto themselves and own their flags - (`logging/log-level-and-formats.test.ts` tests exactly those flags). +- **Binary branch**: spawn the binary with + `--log --log-format json-stream --log-level ` + appended — but **only** for seam-driven invocations; never for tests + that spawn quarto themselves and own their flags + (`logging/log-level-and-formats.test.ts`). **[R]** If a test overlays + `QUARTO_LOG_LEVEL` via `context.env` or passes `logConfig`, the seam + uses that level/format for the flags instead of silently overriding + (flags beat env in `logOptions`, so passing the env var through is not + enough). +- **[R] Failure semantics — the silent-green invariant.** The seam never + relies on "child exited non-zero ⇒ ERROR record exists" (§3). After the + child exits: parse the log; if exit ≠ 0 **and** the parsed records + contain no ERROR-level entry, append a synthetic ERROR record (exit + code + stderr tail) to the log before verification. With that + guarantee: + - `test.execute()` uses `throwOnFailure: false` (mirrors today's + catch-and-log, `test.ts:262-266`); verifiers see the synthetic record. + - **Direct call sites keep throw semantics** (`throwOnFailure: true`, + the default): the smoke-all module-level project pre-render and the + `context.setup` pre-renders (`render-freeze`, extension installs, …) + run *outside* the test try/catch today and fail loudly on error — + a never-throwing helper would convert those into false greens against + stale outputs. +- **[R] stdout/stderr**: pipe both and drain concurrently (undrained + pipes deadlock at 64 KiB on verbose renders — the child runs without + `--quiet`, so its StdErr handler emits all progress). Keep the stderr + tail for the synthetic ERROR record and for failure reports. Do not + pass `--quiet` (shipped console behavior stays exercised and available + for triage). +- **[R] Timeout kills the process tree, not the direct child.** + `QUARTO_TEST_BIN` is a launcher; the bash script and `quarto.exe` both + spawn-and-wait on deno (`common/quarto:205-209` does not `exec`), so + `child.kill()` orphans the actual renderer — which also still holds + the log file at its own offset (mode `"w"`, no `O_APPEND`), corrupting + any harness-appended record. Implementation: POSIX — spawn in its own + process group and signal the group (or, better long-term, change the + installed launcher's non-dev branch to `exec "${QUARTO_DENO}" …`, a + one-line shipped improvement); Windows — `taskkill /PID /T /F`. + Append the timeout ERROR record only after the tree is confirmed dead. +- **[R] Logger lifecycle in `test()`**: in binary mode, `test()` skips + `initializeLogger`/`cleanupLogger`/`flushLoggers` entirely (the child + owns the log file), and the execute-catch appends a synthetic ERROR + record to the file instead of calling `logError` (which, with no + initialized logger, would go to the console handler and — worse — + `cleanupLogger()` would permanently destroy the default handlers for + subsequent tests in the same file). - `testQuartoCmd`'s `cwd` context keeps chdir-ing the harness process - (relative-path verifiers and teardowns depend on it) *and* passes cwd to - the subprocess. -- Startup guard, once per run: if `QUARTO_TEST_BIN` is set, execute - ` --version`; **fail hard** if it reports `99.9.9` (dev-mode trap, - §3) or doesn't match `QUARTO_TEST_EXPECTED_VERSION` when provided. - Print the version banner. + (relative-path verifiers and teardowns depend on it) *and* passes cwd + to the subprocess. +- Startup guard, once per run: execute ` --version`; **fail hard** + on `99.9.9` (dev-mode trap) or mismatch with + `QUARTO_TEST_EXPECTED_VERSION` when provided; print the banner. ### 4.2 Env contract for the spawned binary -With `clearEnv: true`, construct the child env as: - -1. **Base pass-through from ambient env** (needed for engines/toolchains): - `PATH`, `HOME`/`USERPROFILE`, `TMPDIR`/`TEMP`/`TMP`, `LANG`/`LC_*`, - `VIRTUAL_ENV`, `PYTHONPATH`, `R_LIBS*`, `RENV_*`, `JULIA_*`, - `GH_TOKEN`, CI markers (`CI`, `GITHUB_ACTIONS`), proxy vars, and the - toolchain overrides `QUARTO_R`, `QUARTO_PYTHON`, `QUARTO_TYPST`, - `QUARTO_ESBUILD`, `QUARTO_DART_SASS`, `QUARTO_CHROMIUM`, - `QUARTO_TEXLIVE_BINPATH`, `QUARTO_TINYTEX_REPOSITORY`, - `QUARTO_KNITR_RSCRIPT_ARGS` (audit: legitimately CI-environment vars, - not dev-tree vars — stripping them breaks every engine test). -2. **Never pass** (dev-tree identity; the strip list): +**[R] Mechanism: inherit ambient env + strip, not `clearEnv` + +allowlist.** The v2 `clearEnv` design required enumerating every var the +toolchains need; the Windows system-var surface alone (`SystemRoot`, +`ComSpec`, `PATHEXT`, `APPDATA`, `LOCALAPPDATA`, `ProgramData`, +`PROGRAMFILES`, `windir`, …) is large, failure modes are obscure +(CPython misbehaves without `SystemRoot`; TinyTeX resolves from +`APPDATA` — `tinyTexInstallDir()`; `quartoDataDir()` is +`LOCALAPPDATA`-based for chrome-headless-shell/verapdf), and a +linux-only Phase 0 cannot validate it. The dangerous set is the small, +known one. Contract: + +1. **Inherit** the ambient environment (toolchain vars `QUARTO_R`, + `QUARTO_PYTHON`, `QUARTO_TYPST`, `QUARTO_ESBUILD`, + `QUARTO_DART_SASS`, `QUARTO_CHROMIUM`, `QUARTO_TEXLIVE_BINPATH`, + `QUARTO_TINYTEX_REPOSITORY`, `QUARTO_KNITR_RSCRIPT_ARGS` are ambient + CI env and flow through, as do PATH/HOME/locale/proxy/venv/renv/Julia + and all platform system vars). +2. **Strip** (dev-tree identity + stale-leak guards): `QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, `QUARTO_DEBUG`, `DENO_DIR`, - `QUARTO_DENO`, `DENO_DOM_PLUGIN` (both launchers inherit these if - set — bash `common/quarto:167-179`, Rust `main.rs:50-60`), - `QUARTO_ROOT`, `QUARTO_SRC_PATH`, `QUARTO_FORCE_VERSION`, - `QUARTO_VERSION_REQUIREMENT`, `QUARTO_PROJECT_DIR` (stale-leak guard), - `RSTUDIO` (must be affirmatively absent for the "not RStudio" test). -3. **Overlay `context.env` last** (per-test intent wins): e.g. - `QUARTO_PROFILE`, `QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES/_OUTPUT_FILES`, - `QUARTO_PDF_STANDARD`, `QUARTO_LOG_LEVEL`, `RSTUDIO=1`, `LUA_PATH`. - -The exact allowlist is finalized in Phase 0 when real failures show what's -missing; the shape (allowlist + strip + overlay) is fixed. + `QUARTO_DENO`, `QUARTO_DENO_DOM` **[R]**, `QUARTO_ROOT`, + `QUARTO_SRC_PATH`, `QUARTO_FORCE_VERSION`, + `QUARTO_VERSION_REQUIREMENT`, `QUARTO_PROJECT_DIR`, `QUARTO_PROFILE` + (unless test-provided), `QUARTO_LOG`/`_LEVEL`/`_FORMAT` (seam owns + logging via flags), `RSTUDIO` (must be affirmatively absent for the + "not RStudio" test). +3. **Overlay `context.env` last** (per-test intent wins): + `QUARTO_PROFILE`, `QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES/ + _OUTPUT_FILES`, `QUARTO_PDF_STANDARD`, `RSTUDIO=1`, `LUA_PATH`, etc. + +**[R]** The same strip must apply to the `quartoDevCmd()`/`execProcess` +spawns once they target `QUARTO_TEST_BIN` (§4.5b/c) — the one-line +path switch alone leaves them inheriting `DENO_DIR` etc. Route them +through a shared `spawnQuartoEnv()` helper. ### 4.3 smoke-all driver changes (`tests/smoke/smoke-all.test.ts`) -- Route the project pre-render (`:434`) and the `editor-support-crossref` - pseudo-format (2 docs) through `runQuarto`. +- Route the project pre-render (`:434`; `throwOnFailure: true`) and the + `editor-support-crossref` pseudo-format (2 docs) through `runQuarto`. - Everything else (discovery, `_quarto.tests` parsing, `verifyMap` dispatch, cleanup, `run:` skip logic) is harness-side and unchanged. - The YAML-intelligence bootstrap stays (harness-process-only). @@ -274,98 +352,118 @@ missing; the shape (allowlist + strip + overlay) is fixed. When `QUARTO_TEST_BIN` is set (or `--bin ` given): -- Still resolve the repo Deno + import map — the harness always needs them. -- **Do not export** `QUARTO_SHARE_PATH` / `QUARTO_DEBUG` (defense in depth - on top of `clearEnv`; also keeps the harness's own process honest). -- Default no-args test selection must **exclude `unit/`** (dev-only by - definition) — e.g. pass the `smoke/` tree explicitly instead of letting - `deno test` discover everything. -- Print ` --version` banner; refuse `99.9.9` (§4.1 guard). +- Still resolve the repo Deno + import map — the harness always needs + them. **[R] Keep exporting `QUARTO_SHARE_PATH` (and the dev env + generally) for the harness process in ALL modes** — the harness itself + requires it (`smoke-all.test.ts`'s module-level + `initYamlIntelligenceResourcesFromFilesystem()` → + `quartoConfig.sharePath()` → `getenv()` **throws** when unset, + `src/core/env.ts:9-16`). Child isolation is achieved solely by the + §4.2 strip, applied at spawn time. +- **[R] Default test selection**: `run-tests.sh` cannot take a directory + argument today (the arg-validation loop rejects non-`.ts`/doc paths + and exits 1). In binary mode, set `TESTS_TO_RUN` internally to the + smoke tree, and classify `tests/integration/*.test.ts` explicitly + (include or document as dev-only) rather than losing them silently. + `tests/unit/` is excluded (dev-only by definition). +- Print the ` --version` banner; refuse `99.9.9` (§4.1 guard). +- Local recipe: set `QUARTO_TESTS_NO_CONFIG=true` (or order PATH + deliberately) so `configure-test-env.sh` doesn't invoke a PATH + `quarto` you didn't intend (§3 **[R]**). ### 4.5 Test adaptations (from the 133-file classification, §8) - **Shared-helper migrations** (~14 files importing `src/quarto.ts`): - replace direct `quarto()` calls with `runQuarto` (setup pre-renders: - `render-freeze`, `render-format-extension`, - `render-output-file-collision`, `extension-render-{journals, - typst-templates}`; bodies: `crossref/syntax`, `convert/issue-12318`, - `jupyter/cache`, `self-contained/stdout`, `issues/9133`). Trivial - `testQuartoCmd` rewrites where possible (`jupyter/issue-10097`, - `issue-12374`). `engine/invalid-engine-in-project` converts from - `assertRejects` (currently un-awaited — a latent bug) to exit-code + - ERROR-log assertion. -- **`quartoDevCmd()` one-liner** (`tests/utils.ts:244`): return + replace direct `quarto()` calls with `runQuarto` (setup pre-renders + keep throw semantics — §4.1 **[R]**). Trivial `testQuartoCmd` rewrites + where possible (`jupyter/issue-10097`, `issue-12374`). + `engine/invalid-engine-in-project` converts from `assertRejects` + (currently un-awaited — a latent bug) to exit-code + ERROR-log + assertion. +- **`quartoDevCmd()` switch** (`tests/utils.ts:244`): return `QUARTO_TEST_BIN` when set — migrates `run/*`, `lua-unit`, `logging`, - `create` wholesale; consolidate the hardcoded - `../package/dist/bin/quarto` spawns (`filters/editor-support`, - `typst-gather`) onto it. + `create`; consolidate the hardcoded `../package/dist/bin/quarto` + spawns (`filters/editor-support`, `typst-gather`) onto it. **[R]** + Pair the switch with the shared env-strip helper (§4.2) — the path + change alone is insufficient. - **Semantic one-offs**: `env/check.test.ts` — compute expected version from the binary instead of hardcoding `99.9.9`; `inspect-standalone-rstudio` — `RSTUDIO=1` via `context.env` in binary - mode (the in-process `_setIsRStudioForTest` hook stays for dev mode). + mode. - **Anti-pattern fix**: `website/drafts-env.test.ts` converts - `Deno.env.set("QUARTO_PROFILE", ...)` to `context.env` (already flagged - by `.claude/rules/testing/test-anti-patterns.md`; would become - cross-subprocess leakage in binary mode). -- **Move, don't flag**: the 2 genuinely dev-only files + `Deno.env.set("QUARTO_PROFILE", ...)` to `context.env`. +- **Move, don't flag**: the 2 dev-only files (`yaml-intelligence/yaml-intelligence.test.ts`, - `yaml-intelligence-folded-block-strings.test.ts`) move to `tests/unit/`. - `TestContext.requiresDevQuarto` is still added as an escape hatch - (sets Deno `ignore` in binary mode) but starts with zero users. -- **Guard rails** (lightweight lint/CI checks): forbid - `import ... from ".*src/quarto.ts"` under `tests/smoke/`; forbid - quarto spawns not routed through `quartoDevCmd()`/`runQuarto`. + `yaml-intelligence-folded-block-strings.test.ts`) move to + `tests/unit/`. `TestContext.requiresDevQuarto` is added as an escape + hatch (sets Deno `ignore` in binary mode), starting with zero users. +- **Guard rails** (lint/CI checks): forbid `import ... from + ".*src/quarto.ts"` under `tests/smoke/`; forbid quarto spawns not + routed through `quartoDevCmd()`/`runQuarto`. ### 4.6 Fixture fixes - Relax the ten `quarto-required: '>=99.9.0'` fixture extensions to - `'>=1.9'` (evidence in §3; matches their own sibling extensions). -- These are safe, standalone commits that can land **before** any harness - work (they are no-ops for dev mode since `99.9.9 >= 1.9`). + `'>=1.9'` (evidence in §3; matches their sibling extensions). Safe, + standalone, dev-mode no-ops (`99.9.9 >= 1.9`) — can land first. ## 5. CI design ### 5.1 Parameterize `test-smokes.yml` New `workflow_call` inputs: `quarto-install` (`dev`|`release`|`artifact`, -default `dev`), `quarto-version` (for `release`), `quarto-artifact-name` -(for `artifact`), and `ref` (checkout ref, so callers can pin harness = -binary commit). - -Step changes — all **additive**, inserted after the existing `quarto-dev` -step (`test-smokes.yml:230`), which now runs **unconditionally in every -mode** (it provisions the harness Deno runtime, §3): +default `dev`), `quarto-version`, `quarto-artifact-name`, `ref` +(checkout ref), and **[R]** `runners` (JSON list for the OS matrix, +default `'["ubuntu-latest","windows-latest"]'`). **[R]** `buckets` +changes from `required: true` to `required: false, default: ""` (both +existing callers always pass it — behavior-neutral). + +Additional adjustments **[R]**: + +- The seven Windows setup steps gated + `runner.os != 'Windows' || github.event_name == 'schedule'` + (node/playwright/multiplex) must also fire for full non-dev runs + (`inputs.quarto-install != 'dev'`), or a dispatch-triggered full + Windows release-mode run executes in an environment no full Windows + run has ever used. +- All new steps carry `shell: bash` explicitly (they run on the Windows + leg too). + +Step changes — inserted after the existing `quarto-dev` step +(`test-smokes.yml:230`), which runs **unconditionally in every mode** +(harness runtime, §3): ```yaml -- uses: ./.github/workflows/actions/quarto-dev # ALWAYS (harness runtime) +- uses: ./.github/workflows/actions/quarto-dev # ALWAYS -- name: Set up release quarto # release mode +- name: Set up release quarto if: inputs.quarto-install == 'release' uses: quarto-dev/quarto-actions/setup@v2 with: { version: "${{ inputs.quarto-version }}" } -- name: Download built quarto artifact # artifact mode +- name: Download built quarto artifact if: inputs.quarto-install == 'artifact' - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 # repo standard [R] with: { name: "${{ inputs.quarto-artifact-name }}" } -- name: Install built quarto outside the checkout # artifact mode +- name: Install built quarto outside the checkout if: inputs.quarto-install == 'artifact' - run: | # extract OUTSIDE repo (§3 dev-mode trap) + shell: bash + run: | # outside repo (§3 dev-mode trap) mkdir -p "$RUNNER_TEMP/quarto-under-test" tar -xzf built-quarto-*.tar.gz -C "$RUNNER_TEMP/quarto-under-test" --strip-components=1 echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" -- name: Pin and verify test target # both non-dev modes +- name: Pin and verify test target if: inputs.quarto-install != 'dev' + shell: bash run: | v="$(quarto --version)" [ "$v" != "99.9.9" ] || { echo "dev sentinel detected"; exit 1; } echo "QUARTO_TEST_BIN=$(command -v quarto)" >> "$GITHUB_ENV" ``` -Everything downstream is untouched: `quarto install tinytex/verapdf/…` -pick up the PATH override automatically; `run-tests.sh` reads -`QUARTO_TEST_BIN` from the env; language setup is shared. +`quarto install tinytex/verapdf/…` pick up the PATH override +automatically; language setup is shared. ### 5.2 New `test-smokes-built.yml` @@ -380,7 +478,9 @@ on: jobs: build-artifact: # source: build (default; also the schedule path) - if: inputs.source != 'release' + if: > + (github.event_name != 'schedule' || github.repository == 'quarto-dev/quarto-cli') + && (github.event.inputs.source != 'release') # fork guard [R] runs-on: ubuntu-latest outputs: { sha: "${{ steps.rec.outputs.sha }}" } steps: @@ -388,13 +488,13 @@ jobs: - id: rec run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - run: ./configure.sh + - run: | # [R] BUILD METADATA marker — + pushd package/src # semver-valid, range-transparent, + ./quarto-bld prepare-dist \ # still != 99.9.9. NEVER use a + --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" \ + --log-level info # '-prerelease' suffix or a 4th + popd # component (both fail all gates) - run: | - pushd package/src - ./quarto-bld prepare-dist \ - --set-version "$(cat ../../version.txt).$(date +%Y%m%d)-test" \ - --log-level info - popd - - run: | # same recipe as make-tarball pushd package mv pkg-working quarto-built-test tar czf built-quarto-linux-amd64.tar.gz quarto-built-test @@ -405,102 +505,130 @@ jobs: path: package/built-quarto-linux-amd64.tar.gz } run-smokes-artifact: - if: inputs.source != 'release' + if: github.event.inputs.source != 'release' needs: [build-artifact] uses: ./.github/workflows/test-smokes.yml with: - ref: "${{ needs.build-artifact.outputs.sha }}" # harness == binary commit + ref: "${{ needs.build-artifact.outputs.sha }}" # harness == binary commit quarto-install: artifact quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' # linux only [R] + buckets: "" # full run [R] + + resolve-release: # [R] resolve step needs its + if: github.event.inputs.source == 'release' # own job (uses:-jobs have no steps) + runs-on: ubuntu-latest + outputs: { version: "${{ steps.r.outputs.version }}" } + steps: + - id: r + run: | # input 'pre-release'/'release' -> concrete 1.x.y via _prerelease.json/_download.json + ... - run-smokes-release: # dispatch-only, curative/backfill - if: inputs.source == 'release' + run-smokes-release: + if: github.event.inputs.source == 'release' + needs: [resolve-release] uses: ./.github/workflows/test-smokes.yml with: - ref: "refs/tags/v${{ }}" # resolve pre-release → concrete tag first + ref: "refs/tags/v${{ needs.resolve-release.outputs.version }}" quarto-install: release - quarto-version: "${{ inputs.version }}" + quarto-version: "${{ needs.resolve-release.outputs.version }}" + buckets: "" ``` -Notes: - -- The version marker embeds `-test` so the built binary can never be - mistaken for a published build; it also keeps `quarto --version` ≠ - `99.9.9`, satisfying the guard. -- Platform: **linux-amd64 first** (Windows zip needs the signing pipeline; - Windows coverage comes via `source: release`, or later an unsigned local - zip build). -- `release` mode needs a small resolve step (input `pre-release`/`release` - → concrete `1.x.y` via `_prerelease.json`/`_download.json` or the GitHub - API) before computing the checkout tag. -- Full run duration: the daily `test-smokes.yml` scheduled run already - does a full un-sharded run per OS, so a weekly built-mode equivalent is - a known, bounded cost; the `run-parallel-tests` bucket matrix remains - available if needed. -- Later (Phase 4): call the same parameterized workflow from - `create-release.yml` as an initially non-blocking gate before - `publish-release`. +**[R] Version-marker guard**: Phase 0 asserts (with the vendored semver) +`satisfies("", ">=1.3.0") === true` before adopting any marker; +the CI "Pin and verify" step gains the same assertion. + +### 5.3 Release mode scope **[R]** + +Release mode checks out the tag while the *workflow file and local +composite actions* resolve from the checked-out tree. Consequences: + +- The harness at the tag must already contain the binary-mode plumbing — + **no existing tag has it**. Release mode therefore only works for + releases cut **after Phase 1 merges**. "Backfill any historical + version" is out of scope (would require the Phase-4 harness/binary + decoupling: main-branch harness testing an older binary). +- Local-action interface drift is real (`.github/actions/cache-typst` is + referenced unconditionally and only exists since 2026-05; older tags + fail with "action not found"). +- Add a preflight step that fails with a clear message when the + checked-out ref lacks `QUARTO_TEST_BIN` support in `tests/run-tests.sh`. + +### 5.4 Later (Phase 4+) + +Call the parameterized workflow from `create-release.yml` as an +initially non-blocking gate before `publish-release`; revisit the §2 +non-goals list before presenting it as a release gate. ## 6. Phased roadmap -**Phase 0 — spike (≈1 day).** Locally: `configure.sh` + -`quarto-bld prepare-dist`, copy `pkg-working` **outside the checkout** -(§3 dev-mode trap), hack the seam, run ~20–30 representative smoke-all -docs + a few `.test.ts` smokes. Acceptance: catalog of failure classes; -confirmation the log-file contract holds end-to-end; finalized env -allowlist. +**Phase 0 — spike (≈1–2 days).** Locally: `configure.sh` + +`quarto-bld prepare-dist`, copy `pkg-working` **outside the checkout**, +`QUARTO_TESTS_NO_CONFIG=true`, hack the seam, run ~20–30 representative +smoke-all docs + a few `.test.ts` smokes. Acceptance: failure-class +catalog; log-file contract confirmed end-to-end **including the +synthetic-ERROR path** (kill a child mid-render; corrupt the dist and +confirm red, not green); semver marker assertion passes; **measured +per-spawn overhead** (product cold-start, not just process launch — the +corpus is 1424 docs ⇒ 1500+ spawns; if overhead is high, the +`run-parallel-tests` bucket matrix becomes the Phase 2 default rather +than a contingency). **Phase 1 — fixtures + harness binary mode.** Land the ten -`quarto-required` relaxations and the `drafts-env` anti-pattern fix -(standalone, dev-mode no-ops). Implement `runQuarto`, the env contract, -`quartoDevCmd()` change, runner plumbing (incl. `unit/` exclusion), -version guard, `requiresDevQuarto`, the ~14 shared-helper migrations, the -2 file moves to `tests/unit/`. Acceptance: full dev-mode suite still -green (default path untouched); a binary-mode subset green locally. - -**Phase 2 — CI.** Parameterize `test-smokes.yml` (additive inputs); -add `test-smokes-built.yml` (weekly + dispatch, build-then-test, full -smoke-all, linux). Acceptance: first green (or triaged) weekly run; -failures classified into product bugs / harness assumptions / dev-only. - -**Phase 3 — broaden.** `source: release` path (Windows coverage, version -backfill), ff-matrix bucket job, guard-rail lint checks. Acceptance: -dispatch run against latest published prerelease succeeds on both OSes. +`quarto-required` relaxations and the `drafts-env` fix (standalone, +dev-mode no-ops). Implement `runQuarto` (+ `throwOnFailure`, synthetic +ERROR, tree-kill, stderr capture), the env strip helper, `quartoDevCmd()` +switch, runner plumbing (internal `TESTS_TO_RUN`, `integration/` +classification, version guard), logger-lifecycle branching in `test()`, +`requiresDevQuarto`, the ~14 migrations, the 2 file moves. Acceptance: +full dev-mode suite green (default path untouched); binary-mode subset +green locally; the corrupt-dist scenario fails red. + +**Phase 2 — CI.** Parameterize `test-smokes.yml` (§5.1 incl. `runners`, +`buckets` default, Windows-gate fix); add `test-smokes-built.yml` +(weekly + dispatch, build-then-test, full smoke-all, linux). Acceptance: +first green (or fully triaged) weekly run; failures classified into +product bugs / harness assumptions / dev-only. + +**Phase 3 — broaden.** `source: release` path for releases cut after +Phase 1 (Windows coverage via `quarto.exe`; §5.3 scope), ff-matrix +bucket job, guard-rail lint checks. Acceptance: dispatch run against the +first post-Phase-1 prerelease succeeds on both OSes. **Phase 4 — optional.** Non-blocking gate in `create-release.yml`; -decouple harness from `src/` (via `quarto inspect`) only if testing -binaries from a *different* commit than the harness ever matters. +harness/binary decoupling (via `quarto inspect`) if testing binaries +from a different commit than the harness (incl. true backfill) becomes +worth the drift risk. ## 7. Risks & open items -- **Subprocess startup cost** (~100–300 ms × hundreds of renders): small - vs render time; measure in Phase 0. -- **`shouldError` + passthrough commands**: `run`/`pandoc`/`typst` - failures exit non-zero without a log ERROR record (§3); their tests - already use `execProcess` and don't rely on log verifiers — document - the constraint in the seam. -- **Behavioral diffs without `QUARTO_DEBUG`**: ERROR messages lose stack - traces; no known verifier depends on them; Phase 0 confirms. -- **`quarto check` dev branch**: with a real version the - `git rev-parse`/dev branch is skipped — `check`-based tests - (`smoke/check`, `env/check`) assert against installed behavior; adapt - expectations (§4.5). +- **Per-spawn cost at corpus scale**: 1500+ spawns × product cold-start + (bundle load, schema init) — measured in Phase 0; bucket-matrix + sharding is the fallback. +- **Binary-only flakiness classes**: jupyter daemon behavior across + separate quarto processes, per-process knitr/renv startup, memory/CPU + contention under deno-test parallelism — watch in Phase 2 triage. +- **`shouldError` + passthrough/add/remove commands**: §3's record-free + non-zero exits are handled by the synthetic-ERROR invariant; the + passthrough tests keep their own `execProcess` assertions. +- **`quarto check` dev branch**: with a real version the dev branch is + skipped — `check`-based tests assert against installed behavior (§4.5). - **Intra- vs inter-process concurrency**: `issues/9133` reproduces an in-process race; the two-subprocess variant needs a runtime check that the regression still triggers. -- **`configure.sh` cost in binary-mode CI**: it still downloads the full - dev toolchain (pandoc, dart-sass, …) the binary won't use — wasted - minutes, correctness-neutral; optional later flag in `quarto-bld - configure`. -- **Skip-list drift**: guard-rail greps (§4.5) + `requiresDevQuarto` - review-time convention. +- **`configure.sh` cost in binary-mode CI**: still downloads the dev + toolchain the binary won't use — wasted minutes, correctness-neutral. +- **Skip-list drift**: guard-rail greps + `requiresDevQuarto` convention. +- **Scope honesty**: keep the §2 "does not test" list current. --- ## 8. Appendix — classification sweep (2026-07-17) All 133 `tests/smoke/**/*.test.ts` read in full: **107 compatible / 24 -adapt / 2 dev-only**. +adapt / 2 dev-only**. (`tests/integration/*.test.ts` — 3 files — were +outside this sweep and get classified in Phase 1, §4.4.) ### 8.1 Compatible as-is (107) @@ -517,12 +645,12 @@ parameter (not `Deno.env.set`) and forwards cleanly. (a) direct `quarto()` import (~14 files; greppable) → `runQuarto` helper / trivial `testQuartoCmd` rewrites; (b) `quartoDevCmd()` PATH spawns -(`run/*`, `lua-unit`, `logging`, `create`) → one-line env-var switch; -(c) hardcoded `../package/dist/bin/quarto` spawns (`filters/ -editor-support`, `typst-gather` 7–12) → consolidate onto (b); (d) -semantic one-offs (`env/check` 99.9.9 expectation; -`inspect-standalone-rstudio` in-process hook → `RSTUDIO=1` child env). -Details and per-file notes in §4.5. Note: "uses `unitTest()`" is NOT a +(`run/*`, `lua-unit`, `logging`, `create`) → env-var switch **plus the +shared env-strip helper (§4.2)**; (c) hardcoded +`../package/dist/bin/quarto` spawns (`filters/editor-support`, +`typst-gather` 7–12) → consolidate onto (b); (d) semantic one-offs +(`env/check` 99.9.9 expectation; `inspect-standalone-rstudio` in-process +hook → `RSTUDIO=1` child env). Note: "uses `unitTest()`" is NOT a dev-only signal — several files use it as a generic wrapper around subprocess spawns. @@ -537,9 +665,11 @@ yaml-intelligence internals with no CLI surface → move to `tests/unit/`. Binary-clean except the ten `quarto-required: '>=99.9.0'` fixtures (§4.6): no doc executes quarto from a code cell, no runtime dev-tree paths, no pre/post-render scripts, every `printsMessage` uses -INFO/WARN/ERROR (never DEBUG). Driver adaptations: project pre-renders -(24 docs) and `editor-support-crossref` (2 docs) through `runQuarto`; -tolerate non-zero child exit. One-time runtime checks: -`engine/class-override` (`>=1.9.17` — correctly fails on older release -binaries), `QUARTO_EXECUTE_INFO`/`QUARTO_PROJECT_ROOT` env docs, the -pandoc-args INFO echo docs (`2025/12/09/13775-*`). +INFO/WARN/ERROR (never DEBUG). Corpus: 1424 docs; ~326 use the default +log-only spec (the silent-green exposure §3 [R] closes); 1032 of the +1098 spec-bearing docs also have file-content verifiers that fail loudly +when nothing renders. Driver adaptations: project pre-renders (24 docs) +and `editor-support-crossref` (2 docs) through `runQuarto`. One-time +runtime checks: `engine/class-override` (`>=1.9.17`), +`QUARTO_EXECUTE_INFO`/`QUARTO_PROJECT_ROOT` env docs, the pandoc-args +INFO echo docs (`2025/12/09/13775-*`). From a3aef4ea5ffe572e9c9731b7622600c21d406099 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 12:19:32 +0000 Subject: [PATCH 06/43] Relax dev-only quarto-required fixtures and fix drafts-env env mutation The ten fixture extensions declaring quarto-required '>=99.9.0' got that value from create-extension scaffolding run on a dev build (the template computes major.minor.0 from the running version, which is the 99.9.9 dev sentinel). No test exercises version gating, and sibling fixture extensions use realistic constraints, so relax them to '>=1.9' to make the fixtures loadable by a real-version built quarto. No behavior change under dev (99.9.9 satisfies >=1.9). Also convert drafts-env.test.ts from a process-global Deno.env.set (flagged in .claude/rules/testing/test-anti-patterns.md as a parallel test race) to the sanctioned per-test TestContext.env channel. Part of the built-version smoke testing plan (dev-docs/smoke-tests-built-version-plan.md). Co-Authored-By: Claude Fable 5 --- .../01/06/input-relative/test/_extensions/test/_extension.yml | 2 +- .../2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml | 2 +- .../_extensions/my-org/my-brand/_extension.yml | 2 +- .../logo/logo-extension/_extensions/my-brand/_extension.yml | 2 +- .../remote-font-extension/_extensions/my-brand/_extension.yml | 2 +- .../dashboard/_extensions/dragonstyle/lipsum/_extension.yml | 2 +- .../format/html/_extensions/dragonstyle/lipsum/_extension.yml | 2 +- .../lightbox/_extensions/dragonstyle/lipsum/_extension.yml | 2 +- .../_extensions/typst-brand-typography-example/_extension.yml | 2 +- .../_extensions/font-provider/_extension.yml | 2 +- tests/smoke/website/drafts-env.test.ts | 3 +-- 11 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml index eb86e8b6bf..1a6f1d67ea 100644 --- a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml +++ b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml @@ -1,7 +1,7 @@ title: Test author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: project: project: diff --git a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d2..c4db023f92 100644 --- a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml index 2d5882388a..6657128508 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml index 2d5882388a..6657128508 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml index 859925eab7..c95827710d 100644 --- a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My Brand author: Quarto version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d2..c4db023f92 100644 --- a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d2..c4db023f92 100644 --- a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d2..c4db023f92 100644 --- a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml index 106a96726c..4bc546fbf5 100644 --- a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml +++ b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml @@ -1,7 +1,7 @@ title: Typst-brand-typography-example author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml index f11d93da35..bbc62e89a3 100644 --- a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml +++ b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml @@ -1,7 +1,7 @@ title: Font Provider Extension author: Test version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: formats: typst: diff --git a/tests/smoke/website/drafts-env.test.ts b/tests/smoke/website/drafts-env.test.ts index 7e7584d4d8..bb1f80f54a 100644 --- a/tests/smoke/website/drafts-env.test.ts +++ b/tests/smoke/website/drafts-env.test.ts @@ -16,13 +16,12 @@ const renderDir = docs("websites/drafts/drafts-env"); const dir = join(Deno.cwd(), renderDir); const outDir = join(dir, "_site"); -Deno.env.set("QUARTO_PROFILE", "drafts"); - testQuartoCmd( "render", [renderDir], [noErrorsOrWarnings, ...[doesntHaveContentLinksToDrafts, doesntHaveEnvelopeLinksToDrafts, draftPostIsEmpty, searchDoesntHaveDraft, siteMapDoesntHaveDraft].map((ver) => { return ver(outDir)})], { + env: { QUARTO_PROFILE: "drafts" }, teardown: async () => { if (existsSync(outDir)) { await Deno.remove(outDir, { recursive: true }); From 8612a6f892838d8557af08367f701caf1433fcad Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 12:27:45 +0000 Subject: [PATCH 07/43] Add binary mode (QUARTO_TEST_BIN) to the smoke test harness Introduces tests/quarto-cmd.ts as the single dispatch point for invoking the quarto under test. Without QUARTO_TEST_BIN nothing changes: quarto() runs in-process exactly as before. With it, quarto is spawned as a subprocess with --log/--log-format json-stream so the existing log-record verifiers work unchanged against a built distribution. Seam design (see dev-docs/smoke-tests-built-version-plan.md): - runQuarto() dev branch preserves the historical in-process call and timeout semantics byte-for-byte; the binary branch spawns with inherit+strip env (dev-tree identity vars like QUARTO_SHARE_PATH, DENO_DIR, QUARTO_DEBUG never reach the child; per-test env overlays last), pipes and drains stdout/stderr, and kills the whole process tree on timeout (the launcher spawn-and-waits on deno, so killing only the direct child would orphan the renderer). - Silent-green guard: a child can exit non-zero without writing any ERROR record (failures before logger init, quarto add/remove commandFailed paths). runQuarto appends a synthetic ERROR record (exit code + stderr tail) whenever a non-zero exit leaves the log record-free, so log-only verifiers like the default noErrorsOrWarnings cannot pass vacuously. - throwOnFailure defaults to true so direct call sites (module-level project pre-render in smoke-all, context.setup pre-renders) keep fail-loudly semantics; testQuartoCmd passes false and lets verifiers consume the log records. - test() skips initializeLogger/cleanupLogger/flushLoggers in binary mode (the child owns the log file; cleanupLogger would tear down the default handlers for subsequent tests) and appends harness-side failures to the log file directly. - TestContext.requiresDevQuarto marks in-process tests to ignore in binary mode (currently unused; escape hatch). - quartoDevCmd() returns QUARTO_TEST_BIN when set, migrating the subprocess-based tests (run/, lua-unit, logging, create) wholesale. - run-tests.sh/.ps1 verify the binary before running (fail on the 99.9.9 dev sentinel, which means the launcher resolved to dev mode) and default to smoke/ in binary mode (unit/ and integration/ are dev-only). Co-Authored-By: Claude Fable 5 --- tests/quarto-cmd.ts | 312 ++++++++++++++++++++++++++++++++++ tests/run-tests.ps1 | 29 ++++ tests/run-tests.sh | 27 +++ tests/smoke/smoke-all.test.ts | 6 +- tests/test.ts | 73 +++++--- tests/utils.ts | 7 + 6 files changed, 430 insertions(+), 24 deletions(-) create mode 100644 tests/quarto-cmd.ts diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts new file mode 100644 index 0000000000..253155a523 --- /dev/null +++ b/tests/quarto-cmd.ts @@ -0,0 +1,312 @@ +/* +* quarto-cmd.ts +* +* Single dispatch point for invoking the quarto under test. +* +* By default (dev mode) quarto runs in-process by calling the `quarto()` +* entry point imported from ../src/quarto.ts, exactly as the harness always +* has. When QUARTO_TEST_BIN points at an installed quarto (a built +* distribution extracted OUTSIDE this checkout — the launcher enters dev +* mode when it finds a sibling src/quarto.ts), quarto is spawned as a +* subprocess instead, with `--log --log-format json-stream` so the +* existing log-record verifiers keep working unchanged. +* +* See dev-docs/smoke-tests-built-version-plan.md for the full design. +* +* Copyright (C) 2020-2026 Posit Software, PBC +* +*/ +import { quarto } from "../src/quarto.ts"; +import { isWindows } from "../src/deno_ral/platform.ts"; + +// Env vars that identify the dev tree (or stale per-render state) and must +// never leak into the spawned binary: the installed launchers inherit +// QUARTO_SHARE_PATH / QUARTO_DEBUG / QUARTO_DENO / QUARTO_DENO_DOM when +// already set, and never reset DENO_DIR outside dev mode. Logging vars are +// stripped because the seam owns logging via explicit CLI flags. Everything +// else in the ambient environment (PATH, HOME, toolchain locations like +// QUARTO_PYTHON/QUARTO_R, platform system vars) is inherited; per-test env +// is overlaid last so tests can deliberately set any of these. +const kStripEnvVars = [ + "QUARTO_SHARE_PATH", + "QUARTO_BIN_PATH", + "QUARTO_DEBUG", + "DENO_DIR", + "QUARTO_DENO", + "QUARTO_DENO_DOM", + "QUARTO_ROOT", + "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION", + "QUARTO_VERSION_REQUIREMENT", + "QUARTO_PROJECT_DIR", + "QUARTO_PROFILE", + "QUARTO_LOG", + "QUARTO_LOG_LEVEL", + "QUARTO_LOG_FORMAT", + "RSTUDIO", +]; + +const kDevVersionSentinel = "99.9.9"; + +// json-stream ERROR record shape expected by readExecuteOutput/verify.ts +// (level 40 == std/log LogLevels.ERROR) +const kErrorLevel = 40; + +export function binaryMode(): string | undefined { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + return bin && bin.length > 0 ? bin : undefined; +} + +export function buildBinaryEnv( + overlay?: Record, +): Record { + const env = Deno.env.toObject(); + for (const name of kStripEnvVars) { + delete env[name]; + } + return { ...env, ...(overlay ?? {}) }; +} + +// Appends a synthetic ERROR record to a json-stream log file. Only call +// after the child process has exited (single-writer at that point). +export function appendLogError(logFile: string, msg: string) { + const record = JSON.stringify({ + msg, + level: kErrorLevel, + levelName: "ERROR", + }); + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // file may not exist yet + } + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + record + "\n"); +} + +function hasErrorRecord(logFile: string): boolean { + let content = ""; + try { + content = Deno.readTextFileSync(logFile); + } catch { + return false; + } + for (const line of content.split("\n")) { + if (!line) continue; + try { + const record = JSON.parse(line); + if ( + typeof record?.levelName === "string" && + record.levelName.toLowerCase() === "error" + ) { + return true; + } + } catch { + // tolerate partial/corrupt lines + } + } + return false; +} + +// QUARTO_TEST_BIN must point at an installed layout, not the dev tree: the +// installed launcher enters dev mode (runs the TS sources) whenever a +// sibling src/quarto.ts exists, and reports the 99.9.9 dev sentinel. Fail +// loudly rather than silently testing the wrong thing. +let checkedBinary: string | undefined; +export function assertTestBinary(bin: string) { + if (checkedBinary === bin) { + return; + } + const result = new Deno.Command(bin, { + args: ["--version"], + stdout: "piped", + stderr: "piped", + }).outputSync(); + const version = new TextDecoder().decode(result.stdout).trim(); + if (result.code !== 0) { + const stderr = new TextDecoder().decode(result.stderr).trim(); + throw new Error( + `QUARTO_TEST_BIN (${bin}) failed to report a version (exit ${result.code}):\n${stderr}`, + ); + } + if (version === kDevVersionSentinel) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports the dev version sentinel ${kDevVersionSentinel}. ` + + `It is resolving to a dev-mode quarto (the launcher runs the TS sources when a ` + + `sibling src/quarto.ts exists). Point QUARTO_TEST_BIN at a built distribution ` + + `extracted outside the git checkout.`, + ); + } + const expected = Deno.env.get("QUARTO_TEST_EXPECTED_VERSION"); + if (expected && version !== expected) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports version ${version}, expected ${expected} ` + + `(QUARTO_TEST_EXPECTED_VERSION).`, + ); + } + console.log(`[binary mode] testing quarto ${version} at ${bin}`); + checkedBinary = bin; +} + +// QUARTO_TEST_BIN is a launcher that spawns deno and waits (it does not +// exec), so killing the direct child would orphan the actual renderer — +// which also still holds the --log file at its own offset. Kill the whole +// tree, deepest first. (Best effort: children spawned between collection +// and kill can escape.) +async function killProcessTree(pid: number) { + if (isWindows) { + await new Deno.Command("taskkill", { + args: ["/PID", String(pid), "/T", "/F"], + stdout: "null", + stderr: "null", + }).output(); + return; + } + const pids: number[] = []; + const stack = [pid]; + while (stack.length > 0) { + const current = stack.pop()!; + pids.push(current); + try { + const result = new Deno.Command("ps", { + args: ["-o", "pid=", "--ppid", String(current)], + stdout: "piped", + stderr: "null", + }).outputSync(); + const children = new TextDecoder() + .decode(result.stdout) + .split("\n") + .map((line) => parseInt(line.trim(), 10)) + .filter((child) => !isNaN(child)); + stack.push(...children); + } catch { + // ps unavailable; fall back to killing what we have + } + } + for (const target of pids.reverse()) { + try { + Deno.kill(target, "SIGKILL"); + } catch { + // already exited + } + } +} + +export interface RunQuartoOptions { + // per-test environment overlay (TestContext.env) + env?: Record; + // working directory for the spawned binary (binary mode only; dev mode + // call sites manage cwd via Deno.chdir as they always have) + cwd?: string; + // json-stream log target; when set (binary mode), --log/--log-format/ + // --log-level flags are appended so verifiers can read the records + logFile?: string; + logLevel?: string; + logFormat?: string; + timeoutMs?: number; + // Binary mode only: throw when the child exits non-zero or times out. + // Defaults to TRUE so direct call sites (module-level project + // pre-renders, context.setup pre-renders) keep today's fail-loudly + // semantics. testQuartoCmd passes false and relies on the log records + // (including the synthetic one below) reaching the verifiers. + // In dev mode failures always propagate as exceptions, exactly as today. + throwOnFailure?: boolean; +} + +export interface RunQuartoResult { + code: number; + timedOut: boolean; + stderrTail?: string; +} + +export async function runQuarto( + args: string[], + options: RunQuartoOptions = {}, +): Promise { + const bin = binaryMode(); + const timeoutMs = options.timeoutMs ?? 600000; + + if (!bin) { + // dev mode: in-process call, preserving existing semantics exactly + // (a timeout rejects but does not kill the in-process render) + const timeout = new Promise((_resolve, reject) => { + setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); + }); + await Promise.race([quarto(args, undefined, options.env), timeout]); + return { code: 0, timedOut: false }; + } + + assertTestBinary(bin); + const throwOnFailure = options.throwOnFailure ?? true; + + const spawnArgs = [...args]; + if (options.logFile) { + spawnArgs.push( + "--log", + options.logFile, + "--log-format", + options.logFormat ?? "json-stream", + // per-test log intent must land in the flags: explicit flags beat + // QUARTO_LOG_LEVEL env in quarto's logOptions, so passing the env + // var through would be silently ignored + "--log-level", + options.logLevel ?? options.env?.QUARTO_LOG_LEVEL ?? "info", + ); + } + + const child = new Deno.Command(bin, { + args: spawnArgs, + cwd: options.cwd ?? Deno.cwd(), + env: buildBinaryEnv(options.env), + clearEnv: true, + stdout: "piped", + stderr: "piped", + }).spawn(); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + killProcessTree(child.pid); + }, timeoutMs); + + // output() drains both streams (undrained pipes deadlock at the 64KiB + // buffer on verbose renders) and resolves once the process exits — + // including after a kill + const output = await child.output(); + clearTimeout(timer); + + const stderrText = new TextDecoder().decode(output.stderr); + const stderrTail = stderrText.split("\n").slice(-25).join("\n").trim(); + const commandLine = `quarto ${args.join(" ")}`; + + if (options.logFile) { + if (timedOut) { + appendLogError( + options.logFile, + `${commandLine} timed out after ${timeoutMs}ms and was killed`, + ); + } else if (output.code !== 0 && !hasErrorRecord(options.logFile)) { + // A child can exit non-zero without any ERROR record: failures + // before logger init (deno startup, bundle load, missing share) and + // commandFailed paths (quarto add/remove). Without this synthetic + // record, log-only verifiers (noErrorsOrWarnings — the default + // smoke-all spec) would pass vacuously against an empty log. + appendLogError( + options.logFile, + `${commandLine} exited with code ${output.code} without logging an error\n` + + `stderr (tail):\n${stderrTail}`, + ); + } + } + + if ((output.code !== 0 || timedOut) && throwOnFailure) { + throw new Error( + timedOut + ? `${commandLine} timed out after ${timeoutMs}ms` + : `${commandLine} exited with code ${output.code}\nstderr (tail):\n${stderrTail}`, + ); + } + + return { code: output.code, timedOut, stderrTail }; +} diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index f0f3ef77f7..58b6bef88d 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -66,6 +66,27 @@ If ($null -eq $Env:QUARTO_DENO_DIR) { $Env:DENO_DIR = $Env:QUARTO_DENO_DIR } +# BINARY MODE: when QUARTO_TEST_BIN is set, tests run against a built quarto +# (an installed distribution extracted OUTSIDE this checkout). The harness +# itself still runs from the dev tree — the dev env vars above stay — and +# tests/quarto-cmd.ts dispatches quarto invocations to the binary, stripping +# the dev env from the child process. See +# dev-docs/smoke-tests-built-version-plan.md. +If ($null -ne $Env:QUARTO_TEST_BIN) { + If (-not (Test-Path $Env:QUARTO_TEST_BIN)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) does not exist" + Exit 1 + } + $QUARTO_TEST_BIN_VERSION = & $Env:QUARTO_TEST_BIN --version + If ($QUARTO_TEST_BIN_VERSION -eq "99.9.9") { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + Write-Host -ForegroundColor red "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." + Write-Host -ForegroundColor red "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + Exit 1 + } + Write-Host "> BINARY MODE: testing built quarto $QUARTO_TEST_BIN_VERSION at $($Env:QUARTO_TEST_BIN)" +} + # Preparing running Deno with default arguments $QUARTO_IMPORT_MAP_ARG="--importmap=$(Join-Path $QUARTO_SRC_DIR "import_map.json")" @@ -161,6 +182,14 @@ If ($customArgs[0] -notlike "*smoke-all.test.ts") { $TESTS_TO_RUN=$customArgs } +# Binary-mode default selection: smoke tests only. tests/unit/ exercises +# quarto internals in-process (dev-only by definition) and +# tests/integration/ requires the dev playwright setup. +If ($null -ne $Env:QUARTO_TEST_BIN -and $TESTS_TO_RUN.count -eq 0 -and $customArgs.count -eq 0) { + $TESTS_TO_RUN = @("smoke/") + Write-Host "> BINARY MODE: defaulting to smoke/ tests (unit/ and integration/ are dev-only)" +} + # ---- Running tests with Deno ------- $DENO_ARGS = @() diff --git a/tests/run-tests.sh b/tests/run-tests.sh index a2c134fd39..3744c741da 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -62,6 +62,26 @@ export QUARTO_DEBUG=true QUARTO_DENO_OPTIONS="--config test-conf.json --v8-flags=--enable-experimental-regexp-engine,--max-old-space-size=8192,--max-heap-size=8192 --unstable-kv --unstable-ffi --no-lock --allow-all" +# BINARY MODE: when QUARTO_TEST_BIN is set, tests run against a built quarto +# (an installed distribution extracted OUTSIDE this checkout). The harness +# itself still runs from the dev tree — the dev env exports above stay — and +# tests/quarto-cmd.ts dispatches quarto invocations to the binary, stripping +# the dev env from the child process. See +# dev-docs/smoke-tests-built-version-plan.md. +if [[ -n "$QUARTO_TEST_BIN" ]]; then + if [[ ! -x "$QUARTO_TEST_BIN" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) does not exist or is not executable" + exit 1 + fi + QUARTO_TEST_BIN_VERSION="$("$QUARTO_TEST_BIN" --version 2>/dev/null)" + if [[ "$QUARTO_TEST_BIN_VERSION" == "99.9.9" ]]; then + echo "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + echo "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." + echo "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + exit 1 + fi + echo "> BINARY MODE: testing built quarto ${QUARTO_TEST_BIN_VERSION} at ${QUARTO_TEST_BIN}" +fi if [[ -z $GITHUB_ACTION ]] && [[ -z $QUARTO_TESTS_NO_CONFIG ]] then @@ -161,6 +181,13 @@ else TESTS_TO_RUN=("${SMOKE_ALL_TEST_FILE}" "--" "${SMOKE_ALL_FILES[@]}") fi fi + # Binary-mode default selection: smoke tests only. tests/unit/ exercises + # quarto internals in-process (dev-only by definition) and + # tests/integration/ requires the dev playwright setup. + if [[ -n "$QUARTO_TEST_BIN" && "${#TESTS_TO_RUN[@]}" -eq 0 && -z "$*" ]]; then + TESTS_TO_RUN=("smoke/") + echo "> BINARY MODE: defaulting to smoke/ tests (unit/ and integration/ are dev-only)" + fi # TESTS_TO_RUN is an array and quoted here on purpose: a bucket can be a # literal, unexpanded ** glob pattern (e.g. from the ff-matrix CI bucket), # and smoke-all.test.ts expands it itself via expandGlobSync. Expanding it diff --git a/tests/smoke/smoke-all.test.ts b/tests/smoke/smoke-all.test.ts index 1172d2f13e..df4791a496 100644 --- a/tests/smoke/smoke-all.test.ts +++ b/tests/smoke/smoke-all.test.ts @@ -57,7 +57,7 @@ import { findProjectDir, findProjectOutputDir, outputForInput } from "../utils.t import { jupyterNotebookToMarkdown } from "../../src/command/convert/jupyter.ts"; import { basename, dirname, join, relative } from "../../src/deno_ral/path.ts"; import { WalkEntry } from "../../src/deno_ral/fs.ts"; -import { quarto } from "../../src/quarto.ts"; +import { runQuarto } from "../quarto-cmd.ts"; import { safeExistsSync, safeRemoveSync } from "../../src/core/path.ts"; import { runningInCI } from "../../src/core/ci-info.ts"; @@ -431,7 +431,9 @@ for (const { path: fileName } of files) { projectPath && !renderedProjects.has(projectPath) ) { - await quarto(["render", projectPath]); + // fail-loudly pre-render (throwOnFailure defaults to true); + // dispatches to the built binary when QUARTO_TEST_BIN is set + await runQuarto(["render", projectPath]); renderedProjects.add(projectPath); } diff --git a/tests/test.ts b/tests/test.ts index 729568e41f..08975ddeb2 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -10,7 +10,7 @@ import { warning } from "../src/deno_ral/log.ts"; import { initDenoDom } from "../src/core/deno-dom.ts"; import { cleanupLogger, initializeLogger, flushLoggers, logError, LogLevel, LogFormat } from "../src/core/log.ts"; -import { quarto } from "../src/quarto.ts"; +import { appendLogError, binaryMode, runQuarto } from "./quarto-cmd.ts"; import { join } from "../src/deno_ral/path.ts"; import * as colors from "fmt/colors"; import { runningInCI } from "../src/core/ci-info.ts"; @@ -36,8 +36,9 @@ export interface TestDescriptor { // Sets up the test context: TestContext; - // Executes the test - execute: () => Promise; + // Executes the test. In binary mode (QUARTO_TEST_BIN) the harness passes + // the json-stream log file path so the spawned quarto can write it. + execute: (logFile?: string) => Promise; // Used to verify the outcome of the test verify: Verify[]; @@ -90,6 +91,12 @@ export interface TestContext { // Defaults to 600000 (10 minutes). Lower it to assert a performance budget // (e.g. a render that must not regress into a hang). timeout?: number; + + // Marks a test that exercises quarto internals in-process and therefore + // cannot run against an external built binary. Such tests are ignored + // when QUARTO_TEST_BIN is set. Use sparingly — most tests should go + // through testQuartoCmd/runQuarto and work in both modes. + requiresDevQuarto?: boolean; } // Allow to merge test contexts in Tests helpers @@ -127,6 +134,9 @@ export function mergeTestContexts(baseContext: TestContext, additionalContext?: }, // override ignore if provided ignore: additionalContext.ignore ?? baseContext.ignore, + // override requiresDevQuarto if provided + requiresDevQuarto: additionalContext.requiresDevQuarto ?? + baseContext.requiresDevQuarto, // merge env with additional context taking precedence env: { ...baseContext.env, ...additionalContext.env }, // override timeout if provided @@ -147,19 +157,17 @@ export function testQuartoCmd( } test({ name, - execute: async () => { - const timeoutMs = context?.timeout ?? 600000; - const timeout = new Promise((_resolve, reject) => { - setTimeout( - reject, - timeoutMs, - `timed out after ${timeoutMs}ms`, - ); + execute: async (logFile?: string) => { + await runQuarto([cmd, ...args], { + env: context?.env, + logFile, + logLevel: logConfig?.level, + logFormat: logConfig?.format, + timeoutMs: context?.timeout, + // failures must reach the verifiers as log records, not exceptions + // (mirrors the historical catch-and-log behavior in test()) + throwOnFailure: false, }); - await Promise.race([ - quarto([cmd, ...args], undefined, context?.env), - timeout, - ]); }, verify, context: context || {}, @@ -213,7 +221,9 @@ export function test(test: TestDescriptor) { const sanitizeResources = test.context.sanitize?.resources; const sanitizeOps = test.context.sanitize?.ops; const sanitizeExit = test.context.sanitize?.exit; - const ignore = test.context.ignore; + // dev-only tests are ignored when targeting an external built binary + const ignore = test.context.ignore || + (binaryMode() !== undefined && test.context.requiresDevQuarto === true); const userSession = !runningInCI(); const args: Deno.TestDefinition = { @@ -231,9 +241,15 @@ export function test(test: TestDescriptor) { await test.context.setup(); } + // In binary mode the spawned quarto owns the log file; the harness + // must not initialize (or later destroy) its own logger for + // capture — cleanupLogger() would permanently tear down the + // default handlers for subsequent tests in this process. + const binMode = binaryMode() !== undefined; + let cleanedup = false; const cleanupLogOnce = async () => { - if (!cleanedup) { + if (!cleanedup && !binMode) { await cleanupLogger(); cleanedup = true; } @@ -241,8 +257,9 @@ export function test(test: TestDescriptor) { // Capture the output const log = Deno.makeTempFileSync({ suffix: ".json" }); - const handlers = await initializeLogger({ - log: test.logConfig?.log || log, + const logTarget = test.logConfig?.log || log; + const handlers = binMode ? undefined : await initializeLogger({ + log: logTarget, level: test.logConfig?.level || "INFO", format: test.logConfig?.format || "json-stream", quiet: true, @@ -260,15 +277,27 @@ export function test(test: TestDescriptor) { try { try { - await test.execute(); + await test.execute(logTarget); } catch (e) { - logError(e); + if (binMode) { + // no harness logger in binary mode — append the failure to + // the log file directly so verifiers (and the failure + // report) still see it + const message = e instanceof Error + ? `${e.message}\n${e.stack ?? ""}` + : String(e); + appendLogError(logTarget, message); + } else { + logError(e); + } } // Cleanup the output logging await cleanupLogOnce(); - flushLoggers(handlers); + if (handlers) { + flushLoggers(handlers); + } // Read the output const testOutput = logOutput(log); diff --git a/tests/utils.ts b/tests/utils.ts index eabd8344be..813a371a27 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -240,8 +240,15 @@ export function fileLoader(...path: string[]) { }; } +// Resolves the quarto executable for tests that spawn a real subprocess. +// Honors QUARTO_TEST_BIN (binary mode) so these tests target the built +// quarto under test; otherwise the dev quarto from PATH. // On Windows, `quarto.cmd` needs to be explicit in `execProcess()` export function quartoDevCmd(): string { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + if (bin && bin.length > 0) { + return bin; + } return isWindows ? "quarto.cmd" : "quarto"; } From 72bc84362ea3fab62b7cebe9df0f7f8773a5727c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 12:34:06 +0000 Subject: [PATCH 08/43] Migrate direct quarto() call sites to the runQuarto dispatch helper Replaces every 'import { quarto } from src/quarto.ts' in smoke tests with runQuarto from tests/quarto-cmd.ts so these tests work in both dev mode (unchanged in-process behavior) and binary mode (QUARTO_TEST_BIN subprocess): - Setup pre-renders keep fail-loudly semantics (throwOnFailure defaults to true): render-freeze, render-format-extension, render-output-file-collision, extension-render-journals, extension-render-typst-templates, self-contained/stdout, issues/9133 (parallel renders; no shared log file since two concurrent children would interleave one json-stream). - Hand-rolled execute() bodies now accept the harness log file and pass throwOnFailure:false, mirroring testQuartoCmd: crossref/syntax, convert/issue-12318, jupyter/cache. - jupyter/issue-10097 and issue-12374 rewritten as plain testQuartoCmd calls (log-only verifiers). - engine/invalid-engine-in-project: the previous assertRejects was never awaited, so the test passed regardless of outcome. Rewritten as testQuartoCmd + printsMessage on the specific engine error, which actually asserts the failure and works in both modes. Co-Authored-By: Claude Fable 5 --- tests/smoke/convert/issue-12318.test.ts | 14 +++++-- tests/smoke/crossref/syntax.test.ts | 15 ++++++-- .../engine/invalid-engine-in-project.test.ts | 38 ++++++++++--------- .../extension-render-journals.test.ts | 4 +- .../extension-render-typst-templates.test.ts | 4 +- tests/smoke/issues/9133/9133.test.ts | 22 +++++------ tests/smoke/jupyter/cache.test.ts | 38 +++++++++++-------- tests/smoke/jupyter/issue-10097.test.ts | 33 ++++++++-------- tests/smoke/jupyter/issue-12374.test.ts | 30 +++++++-------- .../render/render-format-extension.test.ts | 4 +- tests/smoke/render/render-freeze.test.ts | 4 +- .../render-output-file-collision.test.ts | 4 +- tests/smoke/self-contained/stdout.test.ts | 6 ++- ...-intelligence-folded-block-strings.test.ts | 0 .../yaml-intelligence.test.ts | 0 15 files changed, 119 insertions(+), 97 deletions(-) rename tests/{smoke => unit}/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts (100%) rename tests/{smoke => unit}/yaml-intelligence/yaml-intelligence.test.ts (100%) diff --git a/tests/smoke/convert/issue-12318.test.ts b/tests/smoke/convert/issue-12318.test.ts index 13d95b9e06..812bd90905 100644 --- a/tests/smoke/convert/issue-12318.test.ts +++ b/tests/smoke/convert/issue-12318.test.ts @@ -11,7 +11,7 @@ import { test, } from "../../test.ts"; import { assert } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; (() => { const input = "docs/convert/issue-12318"; @@ -29,9 +29,15 @@ import { quarto } from "../../../src/quarto.ts"; }, // Executes the test - execute: async () => { - await quarto(["convert", "docs/convert/issue-12318.qmd"]); - await quarto(["convert", "docs/convert/issue-12318.ipynb", "--output", "issue-12318-2.qmd"]); + execute: async (logFile?: string) => { + await runQuarto(["convert", "docs/convert/issue-12318.qmd"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["convert", "docs/convert/issue-12318.ipynb", "--output", "issue-12318-2.qmd"], { + logFile, + throwOnFailure: false, + }); const txt = Deno.readTextFileSync("issue-12318-2.qmd"); assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); }, diff --git a/tests/smoke/crossref/syntax.test.ts b/tests/smoke/crossref/syntax.test.ts index 8d4f913d71..968d6dc0f0 100644 --- a/tests/smoke/crossref/syntax.test.ts +++ b/tests/smoke/crossref/syntax.test.ts @@ -15,7 +15,7 @@ import { Verify, } from "../../test.ts"; import { assert, fail } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; const syntaxQmd = crossref("syntax.qmd", "html"); testRender(syntaxQmd.input, "html", false, [ @@ -69,9 +69,16 @@ const context: TestContext = { const testDesc: TestDescriptor = { // FIXME: why is this test flaky now? Ask @dragonstyle name: "test html produced by different figure syntax", context, - execute: async () => { - await quarto(["render", imgQmd.input]); - await quarto(["render", divQmd.input]); + execute: async (logFile?: string) => { + // failures reach the harness as log records, mirroring testQuartoCmd + await runQuarto(["render", imgQmd.input], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", divQmd.input], { + logFile, + throwOnFailure: false, + }); }, verify: [verify], type: "smoke", diff --git a/tests/smoke/engine/invalid-engine-in-project.test.ts b/tests/smoke/engine/invalid-engine-in-project.test.ts index 4b7081e94f..e4c810c820 100644 --- a/tests/smoke/engine/invalid-engine-in-project.test.ts +++ b/tests/smoke/engine/invalid-engine-in-project.test.ts @@ -1,19 +1,21 @@ -import { assertRejects } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; +import { testQuartoCmd } from "../../test.ts"; +import { printsMessage } from "../../verify.ts"; -test( - { - name: "invalid engines option errors", - execute: async () => { - assertRejects( - async () => {await quarto(["render", "docs/engine/invalid-project/notebook.qmd"])}, - Error, - "'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine", - ) - }, - type: "smoke", - context: {}, - verify: [], - } -) \ No newline at end of file +// The previous version of this test wrapped an (un-awaited) assertRejects +// around an in-process quarto() call, so it passed vacuously. Asserting on +// the ERROR log record works in both dev and binary (QUARTO_TEST_BIN) modes: +// in binary mode the message reaches the log via the child's error logging +// (or the synthetic record carrying the stderr tail). +testQuartoCmd( + "render", + ["docs/engine/invalid-project/notebook.qmd"], + [ + printsMessage({ + level: "ERROR", + regex: + /'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine/, + }), + ], + {}, + "invalid engines option errors", +); diff --git a/tests/smoke/extensions/extension-render-journals.test.ts b/tests/smoke/extensions/extension-render-journals.test.ts index 0327048fb6..3ed045c42b 100644 --- a/tests/smoke/extensions/extension-render-journals.test.ts +++ b/tests/smoke/extensions/extension-render-journals.test.ts @@ -5,7 +5,7 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; @@ -44,7 +44,7 @@ for (const journalRepo of journalRepos) { console.log(`using quarto-journals/${journalRepo.repo}`); const wd = Deno.cwd(); Deno.chdir(workingDir); - await quarto([ + await runQuarto([ "use", "template", `quarto-journals/${journalRepo.repo}`, diff --git a/tests/smoke/extensions/extension-render-typst-templates.test.ts b/tests/smoke/extensions/extension-render-typst-templates.test.ts index 5acdbe9148..3511e59022 100644 --- a/tests/smoke/extensions/extension-render-typst-templates.test.ts +++ b/tests/smoke/extensions/extension-render-typst-templates.test.ts @@ -5,7 +5,7 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; @@ -42,7 +42,7 @@ for (const name of typstTemplates) { console.log(`using template: ${source}`); const wd = Deno.cwd(); Deno.chdir(workingDir); - await quarto([ + await runQuarto([ "use", "template", source, diff --git a/tests/smoke/issues/9133/9133.test.ts b/tests/smoke/issues/9133/9133.test.ts index d0b21db04a..82b5b6c06a 100644 --- a/tests/smoke/issues/9133/9133.test.ts +++ b/tests/smoke/issues/9133/9133.test.ts @@ -1,23 +1,23 @@ -import { quarto } from "../../../../src/quarto.ts"; +import { runQuarto } from "../../../quarto-cmd.ts"; import { test } from "../../../test.ts"; if (Deno.build.os !== "windows") { test({ - name: "https://github.com/quarto-dev/quarto-cli/issues/9133", + name: "https://github.com/quarto-dev/quarto-cli/issues/9133", context: { setup: async () => { Deno.mkdirSync("smoke/issues/9133/oh'\"no", { recursive: true }); Deno.copyFileSync("smoke/issues/9133/jl", "smoke/issues/9133/oh'\"no/jl.qmd"); Deno.copyFileSync("smoke/issues/9133/py", "smoke/issues/9133/oh'\"no/py.qmd"); - const timeout = new Promise((_resolve, reject) => { - setTimeout(reject, 600000, "timed out after 10 minutes"); - }); - await Promise.race([ - Promise.all([ - quarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), - quarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), - ]), - timeout, + // concurrent renders reproduce the original intra-process race; + // in binary mode each render is a separate process, so the race + // may not reproduce there. runQuarto supplies the 10-minute + // timeout the explicit Promise.race used to provide, and failures + // throw (no logFile: two concurrent children would interleave a + // shared log file). + await Promise.all([ + runQuarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), + runQuarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), ]); } }, diff --git a/tests/smoke/jupyter/cache.test.ts b/tests/smoke/jupyter/cache.test.ts index 6a687778aa..932279c824 100644 --- a/tests/smoke/jupyter/cache.test.ts +++ b/tests/smoke/jupyter/cache.test.ts @@ -4,7 +4,7 @@ * Copyright (C) 2023 Posit Software, PBC */ import { dirname, join } from "path"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { docs } from "../../utils.ts"; import { folderExists, printsMessage } from "../../verify.ts"; @@ -16,14 +16,18 @@ const cacheFolder = join(dirname(testInput.input), ".jupyter_cache") test({ name: "Jupyter cache is working", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); + // repeated executions to trigger jupyter cache; failures reach the + // verifiers as log records, mirroring testQuartoCmd + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { @@ -54,14 +58,18 @@ const cacheFolder2 = join(dirname(testInput2.input), ".cache/jupyter-cache") test({ name: "Jupyter cache folder can be change", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); + // repeated executions to trigger jupyter cache; failures reach the + // verifiers as log records, mirroring testQuartoCmd + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { diff --git a/tests/smoke/jupyter/issue-10097.test.ts b/tests/smoke/jupyter/issue-10097.test.ts index 19586200e2..743e237a76 100644 --- a/tests/smoke/jupyter/issue-10097.test.ts +++ b/tests/smoke/jupyter/issue-10097.test.ts @@ -1,26 +1,25 @@ /* * parameter-label-duplication.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/10097 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:parameter:label-duplication", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/10097 - await quarto(["render", - "docs/jupyter/parameters/issue-10097.qmd", - "--execute-param", 'datapath:"weird"', - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/10097 +testQuartoCmd( + "render", + [ + "docs/jupyter/parameters/issue-10097.qmd", + "--execute-param", + 'datapath:"weird"', + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:parameter:label-duplication", +); diff --git a/tests/smoke/jupyter/issue-12374.test.ts b/tests/smoke/jupyter/issue-12374.test.ts index 56c5ce67dc..3007adb349 100644 --- a/tests/smoke/jupyter/issue-12374.test.ts +++ b/tests/smoke/jupyter/issue-12374.test.ts @@ -1,25 +1,23 @@ /* * issue-12374.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/12374 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:issue-12374.test.ts", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/12374 - await quarto(["render", - "docs/jupyter/issue-12374.ipynb", - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/12374 +testQuartoCmd( + "render", + [ + "docs/jupyter/issue-12374.ipynb", + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:issue-12374.test.ts", +); diff --git a/tests/smoke/render/render-format-extension.test.ts b/tests/smoke/render/render-format-extension.test.ts index f82374cf32..e9743fb08a 100644 --- a/tests/smoke/render/render-format-extension.test.ts +++ b/tests/smoke/render/render-format-extension.test.ts @@ -13,7 +13,7 @@ import { safeRemoveSync } from "../../../src/core/path.ts"; import { docs } from "../../utils.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { testRender } from "./render.ts"; @@ -32,7 +32,7 @@ const updateExtensions = async () => { Deno.chdir(docs("extensions/format/academic")); for (const repo of ["acs", "elsevier"]) { - await quarto([ + await runQuarto([ "update", "extension", `quarto-journals/${repo}`, diff --git a/tests/smoke/render/render-freeze.test.ts b/tests/smoke/render/render-freeze.test.ts index 7291e24d79..6e4d5affe4 100644 --- a/tests/smoke/render/render-freeze.test.ts +++ b/tests/smoke/render/render-freeze.test.ts @@ -10,7 +10,7 @@ import { assert } from "testing/asserts"; import { Metadata } from "../../../src/config/types.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ExecuteOutput, Verify } from "../../test.ts"; import { outputCreated } from "../../verify.ts"; import { testRender } from "./render.ts"; @@ -107,7 +107,7 @@ function testFileContext( markdown, ); - await quarto(["render", path]); + await runQuarto(["render", path]); }, teardown: async () => { // Clean up the test file diff --git a/tests/smoke/render/render-output-file-collision.test.ts b/tests/smoke/render/render-output-file-collision.test.ts index f360ed0d54..0b802f536f 100644 --- a/tests/smoke/render/render-output-file-collision.test.ts +++ b/tests/smoke/render/render-output-file-collision.test.ts @@ -14,7 +14,7 @@ */ import { existsSync, safeRemoveSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { docs } from "../../utils.ts"; import { testQuartoCmd } from "../../test.ts"; import { @@ -103,7 +103,7 @@ testQuartoCmd( { setup: async () => { await cleanup(defaultDir, [...defaultOutputs, ".quarto"])(); - await quarto(["render", defaultDir]); + await runQuarto(["render", defaultDir]); }, teardown: cleanup(defaultDir, [...defaultOutputs, ".quarto"]), }, diff --git a/tests/smoke/self-contained/stdout.test.ts b/tests/smoke/self-contained/stdout.test.ts index f03a8e6c73..5254c0a397 100644 --- a/tests/smoke/self-contained/stdout.test.ts +++ b/tests/smoke/self-contained/stdout.test.ts @@ -1,11 +1,13 @@ -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; test({ name: "https://github.com/quarto-dev/quarto-cli/issues/11068", context: { setup: async() => { - await quarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); + // only asserts the render to stdout succeeds (failure throws); in + // binary mode the child's stdout is drained and discarded + await runQuarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); } }, execute: async () => {}, diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence.test.ts From b59fadfdb327026e48314c7cf676092d8658918d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 12:34:50 +0000 Subject: [PATCH 09/43] Add built-version smoke testing CI: parameterize test-smokes.yml, add weekly workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test-smokes.yml gains workflow_call inputs (quarto-install: dev|release|artifact, quarto-version, quarto-artifact-name, ref, runners; buckets becomes optional) while keeping every default-path behavior identical — the quarto-dev action still runs unconditionally in all modes because run-tests.sh hardcodes the harness Deno runtime it provisions. Non-dev modes add: release install via quarto-actions/setup, artifact download + extraction OUTSIDE the checkout (the installed launcher enters dev mode when a sibling src/quarto.ts exists), and a pin-and-verify step that fails on the 99.9.9 dev sentinel, validates the version marker shape (build metadata only — prerelease suffixes fail every semver range in the vendored library), checks the checkout has binary-mode support, and exports QUARTO_TEST_BIN. Windows playwright/node setup gates also fire for full non-dev runs. test-smokes-built.yml (weekly Monday + workflow_dispatch) implements the preventive build-then-test mode: build the linux-amd64 dist from the current commit with the same configure.sh + quarto-bld prepare-dist recipe as create-release.yml make-tarball, versioned with semver build metadata (+test.YYYYMMDD), then run the full smoke suite against it at the same SHA on ubuntu only. A dispatch-only source=release path resolves pre-release/release/explicit versions from quarto.org and tests the published binary at its matching v tag (scoped to releases that contain binary-mode support; preflighted via the contents API). Validated with actionlint 1.7.7 (zero findings) and yaml parse; not yet exercised on CI. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-smokes-built.yml | 157 ++++++++++++++++++++++++ .github/workflows/test-smokes.yml | 99 +++++++++++++-- 2 files changed, 247 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test-smokes-built.yml diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml new file mode 100644 index 0000000000..8bd31ceaa2 --- /dev/null +++ b/.github/workflows/test-smokes-built.yml @@ -0,0 +1,157 @@ +# Runs the smoke test suite against a BUILT quarto instead of the dev source tree. +# Two modes: +# - build (default; also the weekly schedule path): build a linux-amd64 dist from +# this checkout with `quarto-bld prepare-dist`, then run the smokes against it +# (harness and binary come from the same commit). +# - release (workflow_dispatch only): install a published (pre-)release with +# quarto-actions/setup and run the smokes at the matching refs/tags/v. +# Only works for releases whose harness already has binary-mode support +# (tests/quarto-cmd.ts) - see dev-docs/smoke-tests-built-version-plan.md §5.3. +name: Smoke Tests (Built Version) +on: + schedule: + - cron: "0 8 * * 1" # weekly, Monday + workflow_dispatch: + inputs: + source: + description: "Test a quarto built from this ref (build) or a published (pre-)release (release)" + required: false + type: choice + options: + - build + - release + default: build + version: + description: "For source=release: 'pre-release' (latest prerelease), 'release' (latest stable), or a literal version like 1.9.10" + required: false + type: string + default: "pre-release" + +# required by the called test-smokes.yml (julia cache cleanup) +permissions: + actions: write + contents: read + +jobs: + # source: build (default; also the schedule path) + build-artifact: + name: Build quarto dist (linux-amd64) + if: >- + (github.event_name != 'schedule' || github.repository == 'quarto-dev/quarto-cli') + && github.event.inputs.source != 'release' + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.rec.outputs.sha }} + steps: + - name: Checkout Repo + uses: actions/checkout@v6 + + - name: Record commit under test + id: rec + shell: bash + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Configure + shell: bash + run: ./configure.sh + + - name: Prepare Distribution + shell: bash + # BUILD METADATA version marker (+test.): semver-valid and + # range-transparent for every 'quarto-required' gate, while still + # distinguishable from the dev sentinel 99.9.9. NEVER use a '-suffix' + # (prereleases fail plain ranges) or a 4th dot component (not semver). + run: | + pushd package/src + ./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" --log-level info + popd + + - name: Make Tarball + shell: bash + # mirrors create-release.yml's make-tarball recipe, with a fixed name + run: | + pushd package + mv pkg-working quarto-built-test + tar --owner=root --group=root -czf built-quarto-linux-amd64.tar.gz quarto-built-test + mv quarto-built-test pkg-working + popd + + - name: Upload Artifact + uses: actions/upload-artifact@v7 + with: + name: built-quarto-linux-amd64 + path: ./package/built-quarto-linux-amd64.tar.gz + + run-smokes-artifact: + name: Smoke tests against built artifact + if: github.event.inputs.source != 'release' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + # harness commit == binary commit: no version skew by construction + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + buckets: "" # full run + + # source: release (workflow_dispatch only) + resolve-release: + name: Resolve release version + if: github.event.inputs.source == 'release' + runs-on: ubuntu-latest + outputs: + version: ${{ steps.r.outputs.version }} + steps: + - name: Resolve version + id: r + shell: bash + run: | + input="${{ github.event.inputs.version }}" + case "$input" in + pre-release | prerelease | "") + version="$(curl -fsSL https://quarto.org/docs/download/_prerelease.json | jq -r '.version')" + ;; + release) + version="$(curl -fsSL https://quarto.org/docs/download/_download.json | jq -r '.version')" + ;; + *) + version="$input" + ;; + esac + if [ -z "$version" ] || [ "$version" = "null" ]; then + echo "::error::could not resolve a quarto version from input '$input'" + exit 1 + fi + echo "Resolved version: $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Preflight - tag must have binary-mode harness support + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.r.outputs.version }} + # releases cut before the QUARTO_TEST_BIN harness support merged cannot be + # tested in this mode (the harness at the tag lacks tests/quarto-cmd.ts); + # test-smokes.yml's "Pin and verify" step re-checks this in the checkout. + run: | + status="$(curl -s -o /dev/null -w '%{http_code}' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github.raw+json" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/contents/tests/quarto-cmd.ts?ref=refs/tags/v${VERSION}")" + if [ "$status" != "200" ]; then + echo "::error::tests/quarto-cmd.ts not found at refs/tags/v${VERSION} (HTTP ${status}): either the tag does not exist or the release predates binary-mode harness support - it cannot be tested in release mode" + exit 1 + fi + echo "refs/tags/v${VERSION} has binary-mode harness support" + + run-smokes-release: + name: Smoke tests against published release + if: github.event.inputs.source == 'release' + needs: [resolve-release] + uses: ./.github/workflows/test-smokes.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + buckets: "" # full run diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 652807eddd..d3069071a2 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -5,9 +5,10 @@ on: workflow_call: inputs: buckets: - description: "JSON string for buckets of tests to run in loop. Array of grouped tests." - required: true + description: "JSON string for buckets of tests to run in loop. Array of grouped tests. Empty means a full run." + required: false type: string + default: "" time-test: description: "Should we run tests to produce test file" required: false @@ -18,6 +19,31 @@ on: required: false type: string default: "" + quarto-install: + description: "Which quarto to test: 'dev' (source tree, default), 'release' (published release via quarto-actions/setup), or 'artifact' (built tarball from a workflow artifact)" + required: false + type: string + default: "dev" + quarto-version: + description: "Version to install when quarto-install is 'release' (passed to quarto-dev/quarto-actions/setup)" + required: false + type: string + default: "" + quarto-artifact-name: + description: "Workflow artifact name containing the built quarto tarball when quarto-install is 'artifact'" + required: false + type: string + default: "" + ref: + description: "Git ref to check out (empty means the default checkout behavior)" + required: false + type: string + default: "" + runners: + description: "JSON list of runner labels for the OS matrix" + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' workflow_dispatch: inputs: buckets: @@ -50,7 +76,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + # inputs is empty on workflow_dispatch/schedule triggers; fall back to the historical default + os: ${{ fromJSON(inputs.runners || '["ubuntu-latest", "windows-latest"]') }} time-test: - ${{ inputs.time-test }} exclude: @@ -63,6 +90,9 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v6 + with: + # empty (the default) keeps the standard checkout behavior + ref: ${{ inputs.ref }} - name: Fix temp dir to use runner one (windows) if: runner.os == 'Windows' @@ -86,7 +116,7 @@ jobs: node-version: 22 - name: Cache multiplex server node_modules - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} uses: actions/cache@v5 with: path: tests/integration/playwright/multiplex-server/node_modules @@ -95,13 +125,13 @@ jobs: ${{ runner.os }}-multiplex-server- - name: Install node dependencies - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} run: npm install --loglevel=error --no-audit working-directory: ./tests/integration/playwright shell: bash - name: Get Playwright version - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} run: | VERSION=$(node -p "require('@playwright/test/package.json').version") echo "PLAYWRIGHT_VERSION=$VERSION" >> "$GITHUB_ENV" @@ -109,7 +139,7 @@ jobs: shell: bash - name: Cache Playwright browsers - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} uses: actions/cache@v5 with: path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} @@ -118,13 +148,13 @@ jobs: ${{ runner.os }}-playwright- - name: Install Playwright system dependencies - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} timeout-minutes: 15 run: npx playwright install-deps working-directory: ./tests/integration/playwright - name: Install Playwright Browsers - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} timeout-minutes: 15 run: npx playwright install working-directory: ./tests/integration/playwright @@ -227,8 +257,59 @@ jobs: run: | uv sync --frozen + # ALWAYS runs, in every quarto-install mode: it provisions the Deno runtime, + # deno_cache and dev tree that the test harness itself needs. In non-dev modes + # the quarto under test is layered on top via a PATH prepend + QUARTO_TEST_BIN. - uses: ./.github/workflows/actions/quarto-dev + - name: Set up release quarto + if: inputs.quarto-install == 'release' + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: ${{ inputs.quarto-version }} + + - name: Download built quarto artifact + if: inputs.quarto-install == 'artifact' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + + - name: Install built quarto outside the checkout + if: inputs.quarto-install == 'artifact' + shell: bash + # extraction must happen OUTSIDE the git checkout: the launcher detects a + # sibling src/quarto.ts and would silently run dev mode (dev-mode trap) + run: | + mkdir -p "$RUNNER_TEMP/quarto-under-test" + tar -xzf "$RUNNER_TEMP"/quarto-artifact/built-quarto-*.tar.gz -C "$RUNNER_TEMP/quarto-under-test" --strip-components=1 + echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" + + - name: Pin and verify test target + if: inputs.quarto-install != 'dev' && inputs.quarto-install != '' + shell: bash + run: | + v="$(quarto --version)" + echo "quarto under test: $v ($(command -v quarto))" + if [ "$v" = "99.9.9" ]; then + echo "::error::quarto on PATH reports the dev sentinel version 99.9.9 - the built/released quarto was not picked up (dev-mode trap)" + exit 1 + fi + # version-marker guard: version must be plain semver, optionally with build + # metadata (+...). A '-prerelease' suffix or a 4th component would fail + # every 'quarto-required' range check in the corpus (semver range gates). + if ! echo "$v" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(\+[0-9A-Za-z.-]+)?$'; then + echo "::error::quarto version '$v' is not range-transparent semver (only build metadata '+...' is allowed; never a '-suffix' or a 4th component)" + exit 1 + fi + # preflight: the checked-out harness must support binary mode; older tags + # (before tests/quarto-cmd.ts existed) cannot be tested in this mode + if [ ! -f tests/quarto-cmd.ts ]; then + echo "::error::this checkout's test harness lacks binary-mode support (tests/quarto-cmd.ts is missing); release/artifact mode only works for refs that include the QUARTO_TEST_BIN harness support" + exit 1 + fi + echo "QUARTO_TEST_BIN=$(command -v quarto)" >> "$GITHUB_ENV" + - name: Install Tinytex env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From a213fa97904996b09ceaa1b9aab76f690b131932 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 12:35:36 +0000 Subject: [PATCH 10/43] Apply binary-mode env hygiene to subprocess-spawning tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests that spawn quarto themselves (via execProcess or Deno.Command) previously inherited the harness's full environment — including QUARTO_SHARE_PATH, QUARTO_DEBUG and DENO_DIR exported by run-tests.sh, which the installed launcher inherits when set, so in binary mode the spawned binary would silently use dev-tree resources. Adds quartoSpawnEnvOptions() to tests/quarto-cmd.ts: in binary mode it returns the ambient env with the dev-tree strip list applied (per-test overlay merged last) plus clearEnv, and in dev mode today's inherit-and-merge behavior exactly. execProcess forwards clearEnv to Deno.Command untouched, so no src/ change is needed. Applied at every quarto spawn site in run/, lua-unit, logging, create, filters/ editor-support and typst-gather; the hardcoded 'quarto' and package/dist/bin paths in stdlib-run-version, editor-support and typst-gather now resolve through quartoDevCmd() (which honors QUARTO_TEST_BIN). Semantic one-offs: env/check.test.ts asserts a real semver version in binary mode instead of the 99.9.9 dev sentinel; inspect-standalone-rstudio passes RSTUDIO=1 via per-test env in binary mode (the in-process _setIsRStudioForTest hook cannot cross the process boundary and remains the dev-mode mechanism). The two in-process yaml-intelligence tests were moved to tests/unit/ in the previous commit; with that, zero smoke tests need the requiresDevQuarto escape hatch today. Co-Authored-By: Claude Fable 5 --- tests/quarto-cmd.ts | 18 ++++++++++++++++++ tests/smoke/create/create.test.ts | 4 ++++ tests/smoke/env/check.test.ts | 9 ++++++++- tests/smoke/filters/editor-support.test.ts | 11 ++++------- .../inspect/inspect-standalone-rstudio.test.ts | 17 +++++++++++++---- .../logging/log-level-and-formats.test.ts | 4 +++- tests/smoke/lua-unit/lua-unit.test.ts | 3 ++- tests/smoke/run/command-passthrough.test.ts | 6 ++++-- tests/smoke/run/run-script.test.ts | 8 +++++--- tests/smoke/run/stdlib-run-version.test.ts | 5 ++++- tests/smoke/typst-gather/typst-gather.test.ts | 16 +++++++--------- 11 files changed, 72 insertions(+), 29 deletions(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index 253155a523..50f03df538 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -67,6 +67,24 @@ export function buildBinaryEnv( return { ...env, ...(overlay ?? {}) }; } +// Env/clearEnv spawn options for tests that launch the quarto under test +// themselves (via execProcess or Deno.Command with quartoDevCmd()) instead +// of going through runQuarto. In binary mode: full ambient env with the +// dev-tree vars stripped (kStripEnvVars) and the per-test overlay merged +// last, plus clearEnv so the sanitized env is authoritative — otherwise the +// installed launcher would inherit QUARTO_SHARE_PATH/QUARTO_DEBUG/DENO_DIR +// from the harness (run-tests.sh exports them) and silently use dev-tree +// resources. In dev mode: today's behavior exactly — the overlay (if any) +// merges into the inherited ambient env. +export function quartoSpawnEnvOptions( + overlay?: Record, +): { env?: Record; clearEnv?: boolean } { + if (binaryMode()) { + return { env: buildBinaryEnv(overlay), clearEnv: true }; + } + return overlay !== undefined ? { env: overlay } : {}; +} + // Appends a synthetic ERROR record to a json-stream log file. Only call // after the child process has exited (single-writer at that point). export function appendLogError(logFile: string, msg: string) { diff --git a/tests/smoke/create/create.test.ts b/tests/smoke/create/create.test.ts index dc685b9d49..cdec16994b 100644 --- a/tests/smoke/create/create.test.ts +++ b/tests/smoke/create/create.test.ts @@ -11,6 +11,7 @@ import { walkSync } from "../../../src/deno_ral/fs.ts"; import { CreateResult } from "../../../src/command/create/cmd-types.ts"; import { assert } from "testing/asserts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; const kCreateTypes: Record = { "project": ["website", "default", "book", "website:blog"], @@ -54,6 +55,7 @@ for (const type of Object.keys(kCreateTypes)) { args: cmd.slice(1), stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }, stdIn); assert(process.success, process.stderr); if (process.stdout) { @@ -101,6 +103,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(buildProcess.success, buildProcess.stderr); } @@ -115,6 +118,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(process.success, process.stderr); } diff --git a/tests/smoke/env/check.test.ts b/tests/smoke/env/check.test.ts index 9125b71d9c..b48c4ddbea 100644 --- a/tests/smoke/env/check.test.ts +++ b/tests/smoke/env/check.test.ts @@ -5,13 +5,20 @@ * */ import { testQuartoCmd } from "../../test.ts"; +import { binaryMode } from "../../quarto-cmd.ts"; import { noErrorsOrWarnings, printsMessage } from "../../verify.ts"; +// Dev mode reports the 99.9.9 sentinel version; a built binary reports its +// real version, so only require a semver-shaped version line there. +const versionRegex = binaryMode() + ? /Version: \d+\.\d+\.\d+/ + : /Version: 99\.9\.9/; + testQuartoCmd( "check", [], [ noErrorsOrWarnings, - printsMessage({level: "INFO", regex: /Version: 99\.9\.9/}), + printsMessage({level: "INFO", regex: versionRegex}), ], ); diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 77729c882e..417c11865c 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -4,21 +4,18 @@ * Copyright (C) 2023 Posit Software, PBC */ -import { docs } from "../../utils.ts"; +import { docs, quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { assertEquals } from "testing/asserts"; -import { isWindows } from "../../../src/deno_ral/platform.ts"; async function runEditorSupportCrossref(doc: string) { - const cmdLine: string = isWindows ? - "../package/dist/bin/quarto.cmd" : - "../package/dist/bin/quarto"; - - const cmd = new Deno.Command(cmdLine, { + const cmd = new Deno.Command(quartoDevCmd(), { args: ["editor-support", "crossref"], stdin: "piped", stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); const child = cmd.spawn(); const writer = child.stdin.getWriter(); diff --git a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts index 2f26608179..b012ecfba2 100644 --- a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts +++ b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts @@ -7,6 +7,7 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { _setIsRStudioForTest } from "../../../src/core/platform.ts"; +import { binaryMode } from "../../quarto-cmd.ts"; import { ExecuteOutput, testQuartoCmd, @@ -14,8 +15,11 @@ import { import { assert, assertEquals } from "testing/asserts"; // Test: standalone file inspect with RStudio override should NOT emit project. -// Uses _setIsRStudioForTest to avoid Deno.env.set() race conditions in -// parallel tests (see #14218, PR #12621). +// Dev mode uses _setIsRStudioForTest to avoid Deno.env.set() race conditions +// in parallel tests (see #14218, PR #12621). In binary mode the in-process +// hook cannot reach the spawned quarto, so RSTUDIO=1 is passed via the +// test's env instead (isRStudio() checks the env var; buildBinaryEnv strips +// ambient RSTUDIO so the companion "not RStudio" test below stays clean). (() => { const input = "docs/inspect/standalone-hello.qmd"; const output = "docs/inspect/standalone-hello.json"; @@ -34,11 +38,16 @@ import { assert, assertEquals } from "testing/asserts"; } ], { + env: binaryMode() ? { RSTUDIO: "1" } : undefined, setup: async () => { - _setIsRStudioForTest(true); + if (!binaryMode()) { + _setIsRStudioForTest(true); + } }, teardown: async () => { - _setIsRStudioForTest(undefined); + if (!binaryMode()) { + _setIsRStudioForTest(undefined); + } if (existsSync(output)) { Deno.removeSync(output); } diff --git a/tests/smoke/logging/log-level-and-formats.test.ts b/tests/smoke/logging/log-level-and-formats.test.ts index 9c2bbf703d..bbd01d2859 100644 --- a/tests/smoke/logging/log-level-and-formats.test.ts +++ b/tests/smoke/logging/log-level-and-formats.test.ts @@ -10,6 +10,7 @@ import { execProcess } from "../../../src/core/process.ts"; import { md5HashSync } from "../../../src/core/hash.ts"; import { safeRemoveIfExists } from "../../../src/core/path.ts"; import { quartoDevCmd, outputForInput } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { assert } from "testing/asserts"; import { LogFormat } from "../../../src/core/log.ts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; @@ -90,7 +91,8 @@ function testLogDirectly(options: { cmd: quartoDevCmd(), args: args, stdout: "piped", - stderr: "piped" + stderr: "piped", + ...quartoSpawnEnvOptions(), }); // Get stdout/stderr with fallback to empty string diff --git a/tests/smoke/lua-unit/lua-unit.test.ts b/tests/smoke/lua-unit/lua-unit.test.ts index 1553052f60..77dcf429fa 100644 --- a/tests/smoke/lua-unit/lua-unit.test.ts +++ b/tests/smoke/lua-unit/lua-unit.test.ts @@ -21,6 +21,7 @@ import { fromFileUrl, join } from "../../../src/deno_ral/path.ts"; import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; // Explicit list, relative to tests/unit-lua/. Keep alphabetized. @@ -50,7 +51,7 @@ for (const relPath of LUA_TESTS) { { cmd: quartoDevCmd(), args: ["run", luaScript], - env: { LUA_PATH }, + ...quartoSpawnEnvOptions({ LUA_PATH }), }, undefined, undefined, diff --git a/tests/smoke/run/command-passthrough.test.ts b/tests/smoke/run/command-passthrough.test.ts index 6db94e984e..462580d896 100644 --- a/tests/smoke/run/command-passthrough.test.ts +++ b/tests/smoke/run/command-passthrough.test.ts @@ -1,16 +1,18 @@ import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; const testPassthroughCmd = (name: string, command: string, args: string[]) => { unitTest(name, async () => { const result = await execProcess({ - cmd: quartoDevCmd(), + cmd: quartoDevCmd(), args: [ command, ...args, - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }); diff --git a/tests/smoke/run/run-script.test.ts b/tests/smoke/run/run-script.test.ts index ee6fb0f16f..308a465ffc 100644 --- a/tests/smoke/run/run-script.test.ts +++ b/tests/smoke/run/run-script.test.ts @@ -3,6 +3,7 @@ import { ensureDirSync } from "../../../src/deno_ral/fs.ts"; import { assert, assertEquals } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; import { EOL } from "fs/eol"; import { lines } from "../../../src/core/text.ts"; @@ -20,9 +21,9 @@ const ensureStreams = (name: string, script: string, stdout: string, stderr: str basename(script), ], // disable logging here to allow for checking the output - env: { + ...quartoSpawnEnvOptions({ "QUARTO_LOG_LEVEL": "CRITICAL", - } + }), }, undefined, undefined, @@ -55,7 +56,8 @@ const testRunCmd = (name: string, script: string) => { args: [ "run", basename(script), - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }, diff --git a/tests/smoke/run/stdlib-run-version.test.ts b/tests/smoke/run/stdlib-run-version.test.ts index bbe7dadf72..eef54b0564 100644 --- a/tests/smoke/run/stdlib-run-version.test.ts +++ b/tests/smoke/run/stdlib-run-version.test.ts @@ -8,15 +8,18 @@ import { execProcess } from "../../../src/core/process.ts"; import { assert } from "testing/asserts"; import { unitTest } from "../../test.ts"; +import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { isWindows } from "../../../src/deno_ral/platform.ts"; unitTest("stdlib-run-version", async () => { const result = await execProcess({ - cmd: "quarto", + cmd: quartoDevCmd(), args: [ "run", "docs/run/test-stdlib.ts", ], + ...quartoSpawnEnvOptions(), }); console.log({result}) assert(result.success); diff --git a/tests/smoke/typst-gather/typst-gather.test.ts b/tests/smoke/typst-gather/typst-gather.test.ts index 7920a63158..2def4d8683 100644 --- a/tests/smoke/typst-gather/typst-gather.test.ts +++ b/tests/smoke/typst-gather/typst-gather.test.ts @@ -3,6 +3,8 @@ import { assert } from "testing/asserts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { execProcess } from "../../../src/core/process.ts"; +import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; // Test 1: Auto-detection from _extension.yml const verifyPackagesCreated: Verify = { @@ -268,20 +270,16 @@ async function runQuarto( cwd: string, env?: Record, ): Promise<{ success: boolean; stdout: string; stderr: string }> { - const quartoCmd = Deno.build.os === "windows" ? "quarto.cmd" : "quarto"; - const quartoPath = join( - Deno.cwd(), - "..", - "package/dist/bin", - quartoCmd, - ); const result = await execProcess({ - cmd: quartoPath, + cmd: quartoDevCmd(), args, cwd, stdout: "piped", stderr: "piped", - env: env ? { ...Deno.env.toObject(), ...env } : undefined, + // dev mode: overlay merges into the inherited ambient env (same child + // env as the previous explicit { ...Deno.env.toObject(), ...env }); + // binary mode: sanitized ambient env + overlay with clearEnv + ...quartoSpawnEnvOptions(env), }); return { success: result.success, From 5520f69066ac857f14e6d9f628ed28a5c7197382 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 12:54:39 +0000 Subject: [PATCH 11/43] Fix binary-mode issues found by code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correctness fixes from an 8-angle review of the binary-mode diff: - Shared-logfile truncation: the built quarto opens --log in truncate mode, so a test running two runQuarto calls against one log file (crossref/syntax, convert/issue-12318, jupyter/cache) would have the second invocation erase the first's records, including the synthetic ERROR guard. Each child now writes its own temp log which runQuarto merges into the caller's log file after exit; the record-free check now inspects only that invocation's output. - drafts-env: restore the module-load QUARTO_PROFILE set alongside context.env. src/project/project-profile.ts caches the base profile from the env on the first render in the process, so a per-render override is ignored whenever another test rendered first — the process-global set (before any test runs) is what makes the drafts profile apply in dev mode; context.env is what the spawned binary sees in binary mode. Documented in llm-docs/testing-patterns.md. - Timeout kill portability: process-tree walk now uses pgrep -P (Linux + macOS/BSD) instead of the procps-only ps --ppid, which on macOS silently killed only the launcher and orphaned the renderer. - Dev-branch timer leak: runQuarto's in-process timeout timer is now cleared after the race; migrated direct call sites previously had no timer and must not gain a dangling 10-minute one. - Windows path resolution in CI pin-and-verify: 'command -v quarto' under git-bash yields an extensionless /c/... path that Deno.Command cannot spawn; resolve quarto.exe/quarto.cmd explicitly and convert with cygpath -w. Also export QUARTO_TEST_EXPECTED_VERSION when the caller pinned a concrete version so every spawn is verified against it (previously an unwired guard). - editor-support and typst-gather were pinned to the locally built package/dist/bin/quarto before the migration; quartoDevCmd() would have silently retargeted them to PATH quarto in local dev runs. New quartoDevBinCmd() preserves the dev-tree pinning (QUARTO_BIN_PATH) while honoring QUARTO_TEST_BIN. - Reuse: dev sentinel now imported from src/core/quarto.ts kLocalDevelopment instead of a private literal copy. - Docs: testing-patterns.md env-var section updated to reflect the drafts-env resolution and the TestContext.env channel; testing rules core-file tables list tests/quarto-cmd.ts. Co-Authored-By: Claude Fable 5 --- .claude/rules/testing/overview.md | 1 + .claude/rules/testing/typescript-tests.md | 2 +- .github/workflows/test-smokes.yml | 19 ++++- llm-docs/testing-patterns.md | 23 ++--- tests/quarto-cmd.ts | 83 ++++++++++++++----- tests/smoke/filters/editor-support.test.ts | 8 +- tests/smoke/typst-gather/typst-gather.test.ts | 8 +- tests/smoke/website/drafts-env.test.ts | 9 ++ 8 files changed, 116 insertions(+), 37 deletions(-) diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index bec82f921b..e194b19874 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -59,6 +59,7 @@ Managed via: | File | Purpose | |------|---------| | `test.ts` | Test infrastructure (`testQuartoCmd`, `unitTest`) | +| `quarto-cmd.ts` | Quarto invocation dispatch (`runQuarto`; in-process dev vs `QUARTO_TEST_BIN` binary mode) | | `verify.ts` | Verification functions | | `utils.ts` | Path utilities (`docs()`, `outputForInput()`) | | `README.md` | Comprehensive documentation | diff --git a/.claude/rules/testing/typescript-tests.md b/.claude/rules/testing/typescript-tests.md index d34043a4aa..d3f2509d2b 100644 --- a/.claude/rules/testing/typescript-tests.md +++ b/.claude/rules/testing/typescript-tests.md @@ -22,7 +22,7 @@ TypeScript-based tests using Deno. Smoke tests render documents; unit tests veri ## Core Infrastructure -Core test files (`test.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. +Core test files (`test.ts`, `quarto-cmd.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. ### Search for an existing verifier before writing one diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index d3069071a2..b8582f248f 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -308,7 +308,24 @@ jobs: echo "::error::this checkout's test harness lacks binary-mode support (tests/quarto-cmd.ts is missing); release/artifact mode only works for refs that include the QUARTO_TEST_BIN harness support" exit 1 fi - echo "QUARTO_TEST_BIN=$(command -v quarto)" >> "$GITHUB_ENV" + # resolve the executable path in a form Deno.Command can spawn: + # on Windows, `command -v quarto` yields an extensionless bash-style + # path (/c/...) that neither Test-Path nor Deno.Command can use — + # prefer the real .exe/.cmd and convert to a native Windows path + if [ "$RUNNER_OS" = "Windows" ]; then + bin="$(command -v quarto.exe || command -v quarto.cmd || command -v quarto)" + bin="$(cygpath -w "$bin")" + else + bin="$(command -v quarto)" + fi + echo "QUARTO_TEST_BIN=$bin" >> "$GITHUB_ENV" + # when the caller pinned a concrete version, have the harness verify + # every spawn targets exactly that version + if [ -n "${{ inputs.quarto-version }}" ] \ + && [ "${{ inputs.quarto-version }}" != "pre-release" ] \ + && [ "${{ inputs.quarto-version }}" != "release" ]; then + echo "QUARTO_TEST_EXPECTED_VERSION=${{ inputs.quarto-version }}" >> "$GITHUB_ENV" + fi - name: Install Tinytex env: diff --git a/llm-docs/testing-patterns.md b/llm-docs/testing-patterns.md index fd36ab1ff6..5562dcb646 100644 --- a/llm-docs/testing-patterns.md +++ b/llm-docs/testing-patterns.md @@ -382,16 +382,19 @@ Rscript -e "renv::install(); renv::snapshot()" | `./run-tests.sh` (default) | **Race condition** | Files run in parallel, share `Deno.env` | | `./run-parallel-tests.sh` | **None** | Separate OS processes | -**Existing bad pattern** - `tests/smoke/website/drafts-env.test.ts`: - -```typescript -// BAD: Sets env var, never restores it -// Only "works" because no other test reads QUARTO_PROFILE -Deno.env.set("QUARTO_PROFILE", "drafts"); -testQuartoCmd("render", [renderDir], [...]); -``` - -**Alternatives:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. +**Preferred channel:** pass per-test env via `TestContext.env` — it reaches +the in-process `quarto()` call in dev mode and the spawned binary in binary +mode (`QUARTO_TEST_BIN`), without mutating process-global state. + +**Known justified exception** - `tests/smoke/website/drafts-env.test.ts` +still sets `QUARTO_PROFILE` at module load *in addition to* `context.env`: +`src/project/project-profile.ts` caches the base profile from the env on the +first render in the process (`baseQuartoProfile`), so in dev (in-process) +mode a per-render env override is ignored whenever another test rendered +first. The module-load set runs before any test and keeps the cache correct; +the `context.env` copy is what the spawned binary sees in binary mode. + +**Alternatives for new tests:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. ## Testing File Exclusion diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index 50f03df538..f2c073d04a 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -17,7 +17,9 @@ * */ import { quarto } from "../src/quarto.ts"; +import { kLocalDevelopment } from "../src/core/quarto.ts"; import { isWindows } from "../src/deno_ral/platform.ts"; +import { join } from "../src/deno_ral/path.ts"; // Env vars that identify the dev tree (or stale per-render state) and must // never leak into the spawned binary: the installed launchers inherit @@ -46,8 +48,6 @@ const kStripEnvVars = [ "RSTUDIO", ]; -const kDevVersionSentinel = "99.9.9"; - // json-stream ERROR record shape expected by readExecuteOutput/verify.ts // (level 40 == std/log LogLevels.ERROR) const kErrorLevel = 40; @@ -57,6 +57,20 @@ export function binaryMode(): string | undefined { return bin && bin.length > 0 ? bin : undefined; } +// Path to the quarto executable for tests that historically pinned the +// locally-built dev CLI (package/dist/bin/quarto) rather than PATH quarto — +// keeps them testing the dev tree in local runs where PATH may hold a stale +// system quarto. QUARTO_BIN_PATH is exported by run-tests.sh/.ps1. +export function quartoDevBinCmd(): string { + const bin = binaryMode(); + if (bin) { + return bin; + } + const binPath = Deno.env.get("QUARTO_BIN_PATH") ?? + join("..", "package", "dist", "bin"); + return join(binPath, isWindows ? "quarto.cmd" : "quarto"); +} + export function buildBinaryEnv( overlay?: Record, ): Record { @@ -103,13 +117,7 @@ export function appendLogError(logFile: string, msg: string) { Deno.writeTextFileSync(logFile, existing + sep + record + "\n"); } -function hasErrorRecord(logFile: string): boolean { - let content = ""; - try { - content = Deno.readTextFileSync(logFile); - } catch { - return false; - } +function hasErrorRecordText(content: string): boolean { for (const line of content.split("\n")) { if (!line) continue; try { @@ -148,9 +156,9 @@ export function assertTestBinary(bin: string) { `QUARTO_TEST_BIN (${bin}) failed to report a version (exit ${result.code}):\n${stderr}`, ); } - if (version === kDevVersionSentinel) { + if (version === kLocalDevelopment) { throw new Error( - `QUARTO_TEST_BIN (${bin}) reports the dev version sentinel ${kDevVersionSentinel}. ` + + `QUARTO_TEST_BIN (${bin}) reports the dev version sentinel ${kLocalDevelopment}. ` + `It is resolving to a dev-mode quarto (the launcher runs the TS sources when a ` + `sibling src/quarto.ts exists). Point QUARTO_TEST_BIN at a built distribution ` + `extracted outside the git checkout.`, @@ -187,8 +195,10 @@ async function killProcessTree(pid: number) { const current = stack.pop()!; pids.push(current); try { - const result = new Deno.Command("ps", { - args: ["-o", "pid=", "--ppid", String(current)], + // pgrep -P is portable across Linux and macOS/BSD (unlike the + // procps-only `ps --ppid` long option) + const result = new Deno.Command("pgrep", { + args: ["-P", String(current)], stdout: "piped", stderr: "null", }).outputSync(); @@ -199,7 +209,7 @@ async function killProcessTree(pid: number) { .filter((child) => !isNaN(child)); stack.push(...children); } catch { - // ps unavailable; fall back to killing what we have + // pgrep unavailable; fall back to killing what we have } } for (const target of pids.reverse()) { @@ -248,21 +258,34 @@ export async function runQuarto( if (!bin) { // dev mode: in-process call, preserving existing semantics exactly // (a timeout rejects but does not kill the in-process render) + let timer: ReturnType | undefined; const timeout = new Promise((_resolve, reject) => { - setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); + timer = setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); }); - await Promise.race([quarto(args, undefined, options.env), timeout]); + try { + await Promise.race([quarto(args, undefined, options.env), timeout]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } return { code: 0, timedOut: false }; } assertTestBinary(bin); const throwOnFailure = options.throwOnFailure ?? true; + // The binary's LogFileHandler opens --log in truncate mode, so two + // invocations sharing one log file would erase the first invocation's + // records (including any synthetic ERROR). Give each child its own temp + // log and merge it into the caller's log file after exit. const spawnArgs = [...args]; + let childLog: string | undefined; if (options.logFile) { + childLog = Deno.makeTempFileSync({ suffix: ".json" }); spawnArgs.push( "--log", - options.logFile, + childLog, "--log-format", options.logFormat ?? "json-stream", // per-test log intent must land in the flags: explicit flags beat @@ -298,13 +321,35 @@ export async function runQuarto( const stderrTail = stderrText.split("\n").slice(-25).join("\n").trim(); const commandLine = `quarto ${args.join(" ")}`; - if (options.logFile) { + if (options.logFile && childLog) { + // merge this invocation's records into the caller's log file + let childContent = ""; + try { + childContent = Deno.readTextFileSync(childLog); + } catch { + // child never wrote the log (e.g. failed before logger init) + } + try { + Deno.removeSync(childLog); + } catch { + // best effort + } + if (childContent.length > 0) { + let existing = ""; + try { + existing = Deno.readTextFileSync(options.logFile); + } catch { + // log file may not exist yet + } + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(options.logFile, existing + sep + childContent); + } if (timedOut) { appendLogError( options.logFile, `${commandLine} timed out after ${timeoutMs}ms and was killed`, ); - } else if (output.code !== 0 && !hasErrorRecord(options.logFile)) { + } else if (output.code !== 0 && !hasErrorRecordText(childContent)) { // A child can exit non-zero without any ERROR record: failures // before logger init (deno startup, bundle load, missing share) and // commandFailed paths (quarto add/remove). Without this synthetic diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 417c11865c..720bc3deb5 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -4,13 +4,15 @@ * Copyright (C) 2023 Posit Software, PBC */ -import { docs, quartoDevCmd } from "../../utils.ts"; -import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; +import { docs } from "../../utils.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { assertEquals } from "testing/asserts"; async function runEditorSupportCrossref(doc: string) { - const cmd = new Deno.Command(quartoDevCmd(), { + // pinned to the locally-built dev CLI (not PATH quarto) as before the + // binary-mode migration; resolves to QUARTO_TEST_BIN in binary mode + const cmd = new Deno.Command(quartoDevBinCmd(), { args: ["editor-support", "crossref"], stdin: "piped", stdout: "piped", diff --git a/tests/smoke/typst-gather/typst-gather.test.ts b/tests/smoke/typst-gather/typst-gather.test.ts index 2def4d8683..e048cb9341 100644 --- a/tests/smoke/typst-gather/typst-gather.test.ts +++ b/tests/smoke/typst-gather/typst-gather.test.ts @@ -3,8 +3,8 @@ import { assert } from "testing/asserts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { execProcess } from "../../../src/core/process.ts"; -import { quartoDevCmd } from "../../utils.ts"; -import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; + +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; // Test 1: Auto-detection from _extension.yml const verifyPackagesCreated: Verify = { @@ -271,7 +271,9 @@ async function runQuarto( env?: Record, ): Promise<{ success: boolean; stdout: string; stderr: string }> { const result = await execProcess({ - cmd: quartoDevCmd(), + // pinned to the locally-built dev CLI (not PATH quarto) as before the + // binary-mode migration; resolves to QUARTO_TEST_BIN in binary mode + cmd: quartoDevBinCmd(), args, cwd, stdout: "piped", diff --git a/tests/smoke/website/drafts-env.test.ts b/tests/smoke/website/drafts-env.test.ts index bb1f80f54a..73347ba1bd 100644 --- a/tests/smoke/website/drafts-env.test.ts +++ b/tests/smoke/website/drafts-env.test.ts @@ -16,6 +16,15 @@ const renderDir = docs("websites/drafts/drafts-env"); const dir = join(Deno.cwd(), renderDir); const outDir = join(dir, "_site"); +// The process-global set is required in dev (in-process) mode: +// src/project/project-profile.ts caches the base profile from the env on the +// FIRST render in the process (`baseQuartoProfile`), so a per-render env +// override is ignored whenever another test rendered first. Setting it at +// module load (before any test runs) preserves the pre-existing behavior. +// The context.env below is what reaches the spawned binary in binary mode +// (a fresh process per render, so the cache concern doesn't apply there). +Deno.env.set("QUARTO_PROFILE", "drafts"); + testQuartoCmd( "render", [renderDir], From 1dd50eb1b4a1c974670a42ee02ea3e14eb3cf4c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:02:24 +0000 Subject: [PATCH 12/43] Allow partial runs of the built-version smoke workflow via a buckets input An empty buckets input (the default, and what schedule runs use) keeps the full smoke run; a dispatch can pass a JSON list of bucket globs (e.g. the feature-format-matrix qmd glob) for a cheaper validation run before committing to the full 1400-document suite. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-smokes-built.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 8bd31ceaa2..82a515232e 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -21,6 +21,11 @@ on: - build - release default: build + buckets: + description: "Optional JSON list of test buckets/globs for a partial run (e.g. the ff-matrix bucket); empty = full smoke run" + required: false + type: string + default: "" version: description: "For source=release: 'pre-release' (latest prerelease), 'release' (latest stable), or a literal version like 1.9.10" required: false @@ -93,7 +98,7 @@ jobs: quarto-install: artifact quarto-artifact-name: built-quarto-linux-amd64 runners: '["ubuntu-latest"]' - buckets: "" # full run + buckets: ${{ github.event.inputs.buckets }} # empty = full run # source: release (workflow_dispatch only) resolve-release: @@ -154,4 +159,4 @@ jobs: ref: refs/tags/v${{ needs.resolve-release.outputs.version }} quarto-install: release quarto-version: ${{ needs.resolve-release.outputs.version }} - buckets: "" # full run + buckets: ${{ github.event.inputs.buckets }} # empty = full run From 3f55c884fbfb2f1144bb9bd487cb1bda759ab09f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:08:14 +0000 Subject: [PATCH 13/43] Extract shared build-dist-tarball composite action The configure.sh + prepare-dist + tarball recipe existed three times: create-release.yml make-tarball, make-arm64-tarball, and the new test-smokes-built.yml build job. Extract it into .github/actions/build-dist-tarball (inputs: version, arch, tarball-name, artifact-name) and use it from all three call sites. Behavior-preserving for create-release.yml: same commands, same --owner/--group root ownership, same tarball top-level directory (quarto-), same artifact names; the only mechanical change is tar -czf instead of the equivalent tar cvf + gzip two-step. The prevent-rerun and version_commit checkout steps stay in the workflow. This also guarantees the weekly built-version smoke run exercises exactly the release build recipe by construction. Co-Authored-By: Claude Fable 5 --- .github/actions/build-dist-tarball/action.yml | 53 +++++++++++++++++ .github/workflows/create-release.yml | 58 ++++--------------- .github/workflows/test-smokes-built.yml | 32 +++------- 3 files changed, 74 insertions(+), 69 deletions(-) create mode 100644 .github/actions/build-dist-tarball/action.yml diff --git a/.github/actions/build-dist-tarball/action.yml b/.github/actions/build-dist-tarball/action.yml new file mode 100644 index 0000000000..9b4810c8ca --- /dev/null +++ b/.github/actions/build-dist-tarball/action.yml @@ -0,0 +1,53 @@ +name: "Build Quarto dist tarball" +description: | + Builds a Quarto distribution tarball from the current checkout: + configure.sh, quarto-bld prepare-dist --set-version, then a tar.gz of + package/pkg-working (top-level directory quarto-) uploaded as a + workflow artifact. Shared by create-release.yml (make-tarball / + make-arm64-tarball) and test-smokes-built.yml. +inputs: + version: + description: "Version to stamp into the distribution (--set-version)" + required: true + arch: + description: "Target architecture: amd64 or arm64" + required: false + default: "amd64" + tarball-name: + description: "File name of the produced .tar.gz" + required: true + artifact-name: + description: "Workflow artifact name to upload the tarball as" + required: true +runs: + using: "composite" + steps: + - name: Configure + shell: bash + run: ./configure.sh + + - name: Prepare Distribution + shell: bash + run: | + pushd package/src/ + if [ "${{ inputs.arch }}" = "arm64" ]; then + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --arch aarch64 --log-level info + else + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --log-level info + fi + popd + + - name: Make Tarball + shell: bash + run: | + pushd package/ + mv pkg-working quarto-${{ inputs.version }} + tar --owner=root --group=root -czf "${{ inputs.tarball-name }}" quarto-${{ inputs.version }} + mv quarto-${{ inputs.version }} pkg-working + popd + + - name: Upload Artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact-name }} + path: ./package/${{ inputs.tarball-name }} diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 04ebc42003..fb800e66e0 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -133,30 +133,13 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-amd64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-amd64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-amd64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: amd64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-amd64.tar.gz + artifact-name: Deb Zip make-arm64-tarball: runs-on: ubuntu-latest @@ -170,30 +153,13 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --arch aarch64 --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-arm64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-arm64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Arm64 Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-arm64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: arm64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-arm64.tar.gz + artifact-name: Deb Arm64 Zip make-tarball-rhel: runs-on: ubuntu-latest diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 82a515232e..fdcf758d94 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -56,36 +56,22 @@ jobs: shell: bash run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Configure - shell: bash - run: ./configure.sh - - - name: Prepare Distribution + - name: Compute version marker + id: version shell: bash # BUILD METADATA version marker (+test.): semver-valid and # range-transparent for every 'quarto-required' gate, while still # distinguishable from the dev sentinel 99.9.9. NEVER use a '-suffix' # (prereleases fail plain ranges) or a 4th dot component (not semver). - run: | - pushd package/src - ./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" --log-level info - popd - - - name: Make Tarball - shell: bash - # mirrors create-release.yml's make-tarball recipe, with a fixed name - run: | - pushd package - mv pkg-working quarto-built-test - tar --owner=root --group=root -czf built-quarto-linux-amd64.tar.gz quarto-built-test - mv quarto-built-test pkg-working - popd + run: echo "version=$(cat version.txt)+test.$(date +%Y%m%d)" >> "$GITHUB_OUTPUT" - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: built-quarto-linux-amd64 - path: ./package/built-quarto-linux-amd64.tar.gz + version: ${{ steps.version.outputs.version }} + arch: amd64 + tarball-name: built-quarto-linux-amd64.tar.gz + artifact-name: built-quarto-linux-amd64 run-smokes-artifact: name: Smoke tests against built artifact From 8ca870f0b84af2b458c90998655747d683e429cc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:13:05 +0000 Subject: [PATCH 14/43] Add nightly mode: test the signed artifacts of a no-publish create-release run The signing steps in make-installer-win run unconditionally, so every create-release run - including the nightly no-publish schedule - already produces a signed Windows Zip (the real quarto.exe) and the linux Deb Zip tarball as workflow artifacts. source=nightly reuses them instead of building anything: resolve a create-release run (input run-id, or latest successful), check out its exact commit for the harness, download its artifacts cross-run (test-smokes.yml gains a quarto-artifact-run-id input backed by download-artifact's run-id/github-token support), and run the smokes on both OSes. This is the preventive Windows coverage path: signed shipped binaries, zero extra build or signing infrastructure, harness at the same commit by construction. The artifact install step now handles both layouts (linux tarball with a quarto-/ top dir, Windows zip with bin/share at the root, extracted via bsdtar as test-zip-win does). Only works once the commit create-release built from contains the binary-mode harness (post-merge); the pin-and-verify preflight fails with a clear message otherwise. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-smokes-built.yml | 89 +++++++++++++++++++++++-- .github/workflows/test-smokes.yml | 37 +++++++++- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index fdcf758d94..a93629c203 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -1,12 +1,17 @@ # Runs the smoke test suite against a BUILT quarto instead of the dev source tree. -# Two modes: +# Three modes: # - build (default; also the weekly schedule path): build a linux-amd64 dist from # this checkout with `quarto-bld prepare-dist`, then run the smokes against it # (harness and binary come from the same commit). +# - nightly (workflow_dispatch only): reuse the SIGNED artifacts of a completed +# create-release run (its nightly schedule builds without publishing) - the +# linux 'Deb Zip' tarball AND the signed 'Windows Zip' (the real quarto.exe) - +# and run the smokes at that run's commit. Preventive Windows coverage without +# any extra build or signing infrastructure. # - release (workflow_dispatch only): install a published (pre-)release with # quarto-actions/setup and run the smokes at the matching refs/tags/v. -# Only works for releases whose harness already has binary-mode support -# (tests/quarto-cmd.ts) - see dev-docs/smoke-tests-built-version-plan.md §5.3. +# nightly/release only work when the target commit's harness has binary-mode +# support (tests/quarto-cmd.ts) - see dev-docs/smoke-tests-built-version-plan.md §5.3. name: Smoke Tests (Built Version) on: schedule: @@ -14,11 +19,12 @@ on: workflow_dispatch: inputs: source: - description: "Test a quarto built from this ref (build) or a published (pre-)release (release)" + description: "Test a quarto built from this ref (build), the artifacts of a create-release run (nightly), or a published (pre-)release (release)" required: false type: choice options: - build + - nightly - release default: build buckets: @@ -31,6 +37,11 @@ on: required: false type: string default: "pre-release" + run-id: + description: "For source=nightly: create-release run id to take artifacts from (empty = latest successful run)" + required: false + type: string + default: "" # required by the called test-smokes.yml (julia cache cleanup) permissions: @@ -44,6 +55,7 @@ jobs: if: >- (github.event_name != 'schedule' || github.repository == 'quarto-dev/quarto-cli') && github.event.inputs.source != 'release' + && github.event.inputs.source != 'nightly' runs-on: ubuntu-latest outputs: sha: ${{ steps.rec.outputs.sha }} @@ -75,7 +87,7 @@ jobs: run-smokes-artifact: name: Smoke tests against built artifact - if: github.event.inputs.source != 'release' + if: github.event.inputs.source != 'release' && github.event.inputs.source != 'nightly' needs: [build-artifact] uses: ./.github/workflows/test-smokes.yml with: @@ -146,3 +158,70 @@ jobs: quarto-install: release quarto-version: ${{ needs.resolve-release.outputs.version }} buckets: ${{ github.event.inputs.buckets }} # empty = full run + + # source: nightly (workflow_dispatch only) - reuse the signed artifacts of a + # completed no-publish create-release run (linux tarball + signed Windows zip) + resolve-nightly: + name: Resolve create-release run + if: github.event.inputs.source == 'nightly' + runs-on: ubuntu-latest + outputs: + run-id: ${{ steps.r.outputs.run-id }} + sha: ${{ steps.r.outputs.sha }} + steps: + - name: Resolve run id and commit + id: r + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + run_id="${{ github.event.inputs.run-id }}" + if [ -z "$run_id" ]; then + run_id="$(curl -fsSL \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/create-release.yml/runs?status=success&per_page=1" \ + | jq -r '.workflow_runs[0].id // empty')" + fi + if [ -z "$run_id" ]; then + echo "::error::no successful create-release run found to take artifacts from" + exit 1 + fi + sha="$(curl -fsSL \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" \ + | jq -r '.head_sha // empty')" + if [ -z "$sha" ]; then + echo "::error::could not resolve head_sha for run ${run_id}" + exit 1 + fi + echo "Using create-release run ${run_id} at commit ${sha}" + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + run-smokes-nightly-linux: + name: Smoke tests against nightly build (linux) + if: github.event.inputs.source == 'nightly' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + # harness commit == the commit create-release built from + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-windows: + name: Smoke tests against nightly build (windows) + if: github.event.inputs.source == 'nightly' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # the SIGNED Windows layout (quarto.exe) produced by make-installer-win + quarto-artifact-name: Windows Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["windows-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index b8582f248f..5a50ae9b0c 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -30,7 +30,12 @@ on: type: string default: "" quarto-artifact-name: - description: "Workflow artifact name containing the built quarto tarball when quarto-install is 'artifact'" + description: "Workflow artifact name containing the built quarto tarball/zip when quarto-install is 'artifact'" + required: false + type: string + default: "" + quarto-artifact-run-id: + description: "Workflow run id to download the artifact from (empty = the current run); lets a caller test artifacts built by another workflow run, e.g. a no-publish create-release build" required: false type: string default: "" @@ -269,12 +274,21 @@ jobs: version: ${{ inputs.quarto-version }} - name: Download built quarto artifact - if: inputs.quarto-install == 'artifact' + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id == '' uses: actions/download-artifact@v8 with: name: ${{ inputs.quarto-artifact-name }} path: ${{ runner.temp }}/quarto-artifact + - name: Download built quarto artifact (from another run) + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id != '' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + run-id: ${{ inputs.quarto-artifact-run-id }} + github-token: ${{ github.token }} + - name: Install built quarto outside the checkout if: inputs.quarto-install == 'artifact' shell: bash @@ -282,7 +296,24 @@ jobs: # sibling src/quarto.ts and would silently run dev mode (dev-mode trap) run: | mkdir -p "$RUNNER_TEMP/quarto-under-test" - tar -xzf "$RUNNER_TEMP"/quarto-artifact/built-quarto-*.tar.gz -C "$RUNNER_TEMP/quarto-under-test" --strip-components=1 + shopt -s nullglob + artifacts=("$RUNNER_TEMP"/quarto-artifact/*.tar.gz "$RUNNER_TEMP"/quarto-artifact/*.zip) + if [ "${#artifacts[@]}" -ne 1 ]; then + echo "::error::expected exactly one .tar.gz or .zip in the quarto artifact, found: ${artifacts[*]:-none}" + exit 1 + fi + case "${artifacts[0]}" in + *.tar.gz) + # linux tarball layout: quarto-/{bin,share} + tar -xzf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" --strip-components=1 + ;; + *.zip) + # Windows Zip layout: {bin,share} at the archive root; bsdtar on + # windows runners extracts zip archives (same as test-zip-win in + # create-release.yml) + tar -xf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" + ;; + esac echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" - name: Pin and verify test target From 4839cec4a9acc72668332f150e7cc29229381a9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:13:26 +0000 Subject: [PATCH 15/43] Document nightly mode and shared build action in the plan Co-Authored-By: Claude Fable 5 --- dev-docs/smoke-tests-built-version-plan.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index 338d4b6e57..6d27aa257e 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -539,6 +539,26 @@ jobs: `satisfies("", ">=1.3.0") === true` before adopting any marker; the CI "Pin and verify" step gains the same assertion. +### 5.2b `source: nightly` — signed artifacts from a no-publish create-release run + +Studied post-review (maintainer suggestion): the signing steps in +`make-installer-win` are **unconditional** — not gated on +`publish-release` — so every create-release run, including the nightly +no-publish schedule, already uploads a **signed** `Windows Zip` (the real +`quarto.exe`) and the linux `Deb Zip` as workflow artifacts. The +implemented `source: nightly` dispatch mode reuses them: resolve a +create-release run (explicit `run-id` input or latest successful), check +out its `head_sha` for the harness (same-commit, no skew), download the +artifacts cross-run (`test-smokes.yml` input `quarto-artifact-run-id` → +`download-artifact` `run-id`/`github-token`), and run the smokes on both +OSes. This is the **preventive Windows coverage** path — signed shipped +binaries, zero extra build/signing infrastructure. The build recipe +itself is also shared now: `.github/actions/build-dist-tarball` is used +by both `create-release.yml` (amd64 + arm64 tarballs) and the `build` +mode, so the weekly build exercises the release recipe by construction. +Constraint: works only for create-release runs whose commit contains the +binary-mode harness (post-merge); the preflight fails clearly otherwise. + ### 5.3 Release mode scope **[R]** Release mode checks out the tag while the *workflow file and local From 6adf54b04f8507e11d26b40a04a58a23df5da6e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:19:48 +0000 Subject: [PATCH 16/43] Add isBinaryMode()/quartoTestBin() predicates for readable mode checks binaryMode() returned the binary path, which made boolean call sites read awkwardly (binaryMode() !== undefined). Split the API: quartoTestBin() returns the path for spawn sites; isBinaryMode() is the boolean predicate for mode checks in test.ts and the two mode-aware tests. Co-Authored-By: Claude Fable 5 --- tests/quarto-cmd.ts | 15 +++++++++++---- tests/smoke/env/check.test.ts | 4 ++-- .../inspect/inspect-standalone-rstudio.test.ts | 8 ++++---- tests/test.ts | 6 +++--- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index f2c073d04a..9e9c005212 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -52,17 +52,24 @@ const kStripEnvVars = [ // (level 40 == std/log LogLevels.ERROR) const kErrorLevel = 40; -export function binaryMode(): string | undefined { +// Path of the built quarto under test, when binary mode is active. +export function quartoTestBin(): string | undefined { const bin = Deno.env.get("QUARTO_TEST_BIN"); return bin && bin.length > 0 ? bin : undefined; } +// True when tests target an external built quarto (QUARTO_TEST_BIN) instead +// of the in-process dev sources. +export function isBinaryMode(): boolean { + return quartoTestBin() !== undefined; +} + // Path to the quarto executable for tests that historically pinned the // locally-built dev CLI (package/dist/bin/quarto) rather than PATH quarto — // keeps them testing the dev tree in local runs where PATH may hold a stale // system quarto. QUARTO_BIN_PATH is exported by run-tests.sh/.ps1. export function quartoDevBinCmd(): string { - const bin = binaryMode(); + const bin = quartoTestBin(); if (bin) { return bin; } @@ -93,7 +100,7 @@ export function buildBinaryEnv( export function quartoSpawnEnvOptions( overlay?: Record, ): { env?: Record; clearEnv?: boolean } { - if (binaryMode()) { + if (isBinaryMode()) { return { env: buildBinaryEnv(overlay), clearEnv: true }; } return overlay !== undefined ? { env: overlay } : {}; @@ -252,7 +259,7 @@ export async function runQuarto( args: string[], options: RunQuartoOptions = {}, ): Promise { - const bin = binaryMode(); + const bin = quartoTestBin(); const timeoutMs = options.timeoutMs ?? 600000; if (!bin) { diff --git a/tests/smoke/env/check.test.ts b/tests/smoke/env/check.test.ts index b48c4ddbea..d0b41d0989 100644 --- a/tests/smoke/env/check.test.ts +++ b/tests/smoke/env/check.test.ts @@ -5,12 +5,12 @@ * */ import { testQuartoCmd } from "../../test.ts"; -import { binaryMode } from "../../quarto-cmd.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { noErrorsOrWarnings, printsMessage } from "../../verify.ts"; // Dev mode reports the 99.9.9 sentinel version; a built binary reports its // real version, so only require a semver-shaped version line there. -const versionRegex = binaryMode() +const versionRegex = isBinaryMode() ? /Version: \d+\.\d+\.\d+/ : /Version: 99\.9\.9/; diff --git a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts index b012ecfba2..d0ae5e8041 100644 --- a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts +++ b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts @@ -7,7 +7,7 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { _setIsRStudioForTest } from "../../../src/core/platform.ts"; -import { binaryMode } from "../../quarto-cmd.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { ExecuteOutput, testQuartoCmd, @@ -38,14 +38,14 @@ import { assert, assertEquals } from "testing/asserts"; } ], { - env: binaryMode() ? { RSTUDIO: "1" } : undefined, + env: isBinaryMode() ? { RSTUDIO: "1" } : undefined, setup: async () => { - if (!binaryMode()) { + if (!isBinaryMode()) { _setIsRStudioForTest(true); } }, teardown: async () => { - if (!binaryMode()) { + if (!isBinaryMode()) { _setIsRStudioForTest(undefined); } if (existsSync(output)) { diff --git a/tests/test.ts b/tests/test.ts index 08975ddeb2..ef2ddbaf09 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -10,7 +10,7 @@ import { warning } from "../src/deno_ral/log.ts"; import { initDenoDom } from "../src/core/deno-dom.ts"; import { cleanupLogger, initializeLogger, flushLoggers, logError, LogLevel, LogFormat } from "../src/core/log.ts"; -import { appendLogError, binaryMode, runQuarto } from "./quarto-cmd.ts"; +import { appendLogError, isBinaryMode, runQuarto } from "./quarto-cmd.ts"; import { join } from "../src/deno_ral/path.ts"; import * as colors from "fmt/colors"; import { runningInCI } from "../src/core/ci-info.ts"; @@ -223,7 +223,7 @@ export function test(test: TestDescriptor) { const sanitizeExit = test.context.sanitize?.exit; // dev-only tests are ignored when targeting an external built binary const ignore = test.context.ignore || - (binaryMode() !== undefined && test.context.requiresDevQuarto === true); + (isBinaryMode() && test.context.requiresDevQuarto); const userSession = !runningInCI(); const args: Deno.TestDefinition = { @@ -245,7 +245,7 @@ export function test(test: TestDescriptor) { // must not initialize (or later destroy) its own logger for // capture — cleanupLogger() would permanently tear down the // default handlers for subsequent tests in this process. - const binMode = binaryMode() !== undefined; + const binMode = isBinaryMode(); let cleanedup = false; const cleanupLogOnce = async () => { From 42ec794de7ea31e095cac4813806ba0b33267129 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:25:32 +0000 Subject: [PATCH 17/43] Doc audit for binary-mode testing (part 1) First tranche of the documentation audit for the QUARTO_TEST_BIN binary mode: debugging-flaky-tests gains a binary-mode isolation tip (a flake vanishing in binary mode implicates in-process harness state), the prerelease checklist gains an optional post-publish validation dispatch of Smoke Tests (Built Version), llm-docs/testing-patterns.md gains staleness frontmatter and a Dev Mode vs Binary Mode section with the authoring rules, and the testing rules overview points at binary mode. Remaining audit edits (tests/README.md et al.) follow. Co-Authored-By: Claude Fable 5 --- .claude/rules/testing/overview.md | 2 + .claude/rules/testing/test-anti-patterns.md | 4 ++ .claude/rules/testing/typescript-tests.md | 8 ++++ .../checklist-make-a-new-quarto-prerelease.md | 1 + dev-docs/debugging-flaky-tests.md | 9 ++++ llm-docs/testing-patterns.md | 44 ++++++++++++++++++- tests/README.md | 39 +++++++++++++++- 7 files changed, 105 insertions(+), 2 deletions(-) diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index e194b19874..43731e1467 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -28,6 +28,8 @@ QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh test.ts # Linux/macOS $env:QUARTO_TESTS_NO_CONFIG=$true; .\run-tests.ps1 # Windows ``` +**Binary mode:** set `QUARTO_TEST_BIN` to a built quarto (extracted outside the checkout — the runner refuses the 99.9.9 dev sentinel) to spawn it instead of running the dev sources in-process; defaults to `smoke/` only (`unit/` and `integration/` are dev-only). See `tests/README.md` → "Binary mode". + ## Test Types | Type | Location | File Pattern | Details | diff --git a/.claude/rules/testing/test-anti-patterns.md b/.claude/rules/testing/test-anti-patterns.md index ab18275a71..a0ccef6a24 100644 --- a/.claude/rules/testing/test-anti-patterns.md +++ b/.claude/rules/testing/test-anti-patterns.md @@ -6,6 +6,10 @@ paths: # Test Anti-Patterns +## Don't: Import `src/quarto.ts` in Tests + +A direct `import { quarto } from "src/quarto.ts"` bypasses binary mode (`QUARTO_TEST_BIN`) and always tests the dev sources. Invoke quarto through `testQuartoCmd()`/`runQuarto()` (`tests/quarto-cmd.ts` is the single dispatch point), and resolve subprocess spawns with `quartoDevCmd()` (`tests/utils.ts`). + ## Don't: Modify Environment Variables `Deno.env.set()` modifies process-global state. Deno runs test files in parallel by default, so other tests can see modified values. diff --git a/.claude/rules/testing/typescript-tests.md b/.claude/rules/testing/typescript-tests.md index d3f2509d2b..901914931a 100644 --- a/.claude/rules/testing/typescript-tests.md +++ b/.claude/rules/testing/typescript-tests.md @@ -24,6 +24,14 @@ TypeScript-based tests using Deno. Smoke tests render documents; unit tests veri Core test files (`test.ts`, `quarto-cmd.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. +### Binary mode compatibility + +Tests must keep working when `QUARTO_TEST_BIN` targets a built quarto (binary mode) instead of the in-process dev sources: + +- Never `import { quarto } from "../../src/quarto.ts"` in tests — invoke quarto via `testQuartoCmd()`/`runQuarto()`. +- Subprocess spawns of quarto: resolve the executable with `quartoDevCmd()` (`tests/utils.ts`, honors `QUARTO_TEST_BIN`) or `quartoDevBinCmd()` (`tests/quarto-cmd.ts`, pins the local dev build), and pass `quartoSpawnEnvOptions()` as spawn env options. +- `TestContext.requiresDevQuarto: true` ignores a test in binary mode — rare escape hatch for tests exercising quarto internals in-process. + ### Search for an existing verifier before writing one `verify.ts` already covers many output shapes — including parsed-content diff --git a/dev-docs/checklist-make-a-new-quarto-prerelease.md b/dev-docs/checklist-make-a-new-quarto-prerelease.md index f2b27a0d2f..9256942fa1 100644 --- a/dev-docs/checklist-make-a-new-quarto-prerelease.md +++ b/dev-docs/checklist-make-a-new-quarto-prerelease.md @@ -11,5 +11,6 @@ - New release prerelease should be on Github at - A new tag should be on main for the new prerelease version - `version.txt` on main should have been updated by the workflow to the pre-release version just released: https://github.com/quarto-dev/quarto-cli/blob/main/version.txt +- (optional) validate the published prerelease: Actions -> "Smoke Tests (Built Version)" -> "Run Workflow" with source `release` (version defaults to `pre-release`) to run the smoke suite against the installed release binaries instead of the dev tree Note: Cloudsmith publishing is skipped for prereleases (only runs for stable releases). diff --git a/dev-docs/debugging-flaky-tests.md b/dev-docs/debugging-flaky-tests.md index 67e30dc33e..70816a80d2 100644 --- a/dev-docs/debugging-flaky-tests.md +++ b/dev-docs/debugging-flaky-tests.md @@ -184,8 +184,17 @@ quarto install tinytex for test in test1.ts test2.ts test3.ts; do ./run-tests.sh $test || break done + +# Run against a built quarto instead of the in-process dev sources +# (binary mode; see dev-docs/smoke-tests-built-version-plan.md) +QUARTO_TEST_BIN=/path/to/installed/quarto ./run-tests.sh path/to/test.ts ``` +By default the harness invokes quarto in-process; with `QUARTO_TEST_BIN` set, +each invocation is spawned as a subprocess of a built distribution. A flake +that disappears in binary mode points at in-process state pollution in the +dev harness rather than a product bug. + ### Package/Dependency Comparison ```bash diff --git a/llm-docs/testing-patterns.md b/llm-docs/testing-patterns.md index 5562dcb646..8aafadddf2 100644 --- a/llm-docs/testing-patterns.md +++ b/llm-docs/testing-patterns.md @@ -1,3 +1,13 @@ +--- +main_commit: a3f6218b7 +analyzed_date: 2026-07-17 +key_files: + - tests/test.ts + - tests/quarto-cmd.ts + - tests/verify.ts + - tests/utils.ts +--- + # Quarto Test Patterns This document describes the standard patterns for writing smoke tests in the Quarto CLI test suite. @@ -7,9 +17,41 @@ This document describes the standard patterns for writing smoke tests in the Qua Quarto uses Deno for testing with custom verification helpers located in: - `tests/test.ts` - Core test runner (`testQuartoCmd`) +- `tests/quarto-cmd.ts` - Quarto invocation dispatch (`runQuarto`: in-process dev quarto vs built binary) - `tests/verify.ts` - Verification helpers (`fileExists`, `pathDoNotExists`, etc.) - `tests/utils.ts` - Utility functions (`docs()`, `outputForInput()`, etc.) +### Dev Mode vs Binary Mode + +`testQuartoCmd` does not call quarto directly — it goes through `runQuarto()` +in `tests/quarto-cmd.ts`, the single dispatch point for invoking the quarto +under test: + +- **Dev mode (default):** quarto runs in-process via the `quarto()` entry + point imported from `src/quarto.ts`, as the harness always has. +- **Binary mode:** when `QUARTO_TEST_BIN` points at an installed quarto (a + built distribution extracted *outside* this checkout), quarto is spawned as + a subprocess with `--log --log-format json-stream`, so the log-record + verifiers work unchanged. Dev-tree env vars (`QUARTO_SHARE_PATH`, + `QUARTO_DEBUG`, `DENO_DIR`, ...) are stripped from the child. + `run-tests.sh`/`.ps1` refuse a binary reporting the `99.9.9` dev sentinel + and default the selection to `smoke/` only (`unit/` and `integration/` are + dev-only). Exercised by `.github/workflows/test-smokes-built.yml`; full + design in `dev-docs/smoke-tests-built-version-plan.md`. + +Consequences for writing smoke tests: + +- Do **not** import `src/quarto.ts` (or call `quarto()`) directly from + `tests/smoke/` — route invocations through `testQuartoCmd`/`runQuarto` so + the test works in both modes. +- Tests that spawn a quarto subprocess themselves should resolve the + executable via `quartoDevCmd()` (`tests/utils.ts`, honors + `QUARTO_TEST_BIN`) and pass `quartoSpawnEnvOptions()` from + `tests/quarto-cmd.ts` as spawn env options. +- A test that genuinely exercises quarto internals in-process can set + `TestContext.requiresDevQuarto: true`; it is ignored in binary mode. Use + sparingly — most such code belongs in `tests/unit/` instead. + ## Common Test Patterns ### Simple Render Tests @@ -110,7 +152,7 @@ testQuartoCmd("render", [projectDir], [noErrors /*, ... */], { **Key points:** - The budget is machine-dependent (post-fix render time must sit well under it, pre-fix hang well over it), so it is defense-in-depth. Pair it with a deterministic unit test on the actual fix mechanism as the primary guard. -- A timed-out render subprocess is not killed by the harness, so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. +- In dev (in-process) mode a timed-out render is not killed by the harness (the timeout only rejects), so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. In binary mode (`QUARTO_TEST_BIN`) the spawned process tree *is* killed on timeout, but the kill is best-effort — keep the same defensive teardown. ### Extension Template Tests diff --git a/tests/README.md b/tests/README.md index ef401eddaa..62fd9043ba 100644 --- a/tests/README.md +++ b/tests/README.md @@ -12,7 +12,7 @@ Tests are run in our CI workflow on GHA at each commit, and for each PR. ## How the tests are created and organized ? -Tests are running through `Deno.test()` framework, adapted for our Quarto project and all written in Typescript. Infrastructure are in `tests.ts`, `tests.deps.ts` `verify.ts` and `utils.ts` which contains the helper functions that can be used. +Tests are running through `Deno.test()` framework, adapted for our Quarto project and all written in Typescript. Infrastructure are in `test.ts`, `test-deps.ts`, `quarto-cmd.ts`, `verify.ts` and `utils.ts` which contains the helper functions that can be used. - `unit/` and `integration/`, `smoke/`contain some `.ts` script representing each tests. - `docs/` is a special folder containing of the necessary files and projects used for the tests. @@ -430,6 +430,41 @@ Don't do ./run-tests.sh smoke/extensions/extension-render-doc.test.ts smoke/smoke-all.test.ts -- ./docs/smoke-all/2023/01/04/issue-3847.qmd ``` +### Binary mode (`QUARTO_TEST_BIN`) + +By default, tests run quarto **in-process** from the dev sources: `runQuarto()` in `tests/quarto-cmd.ts` calls the `quarto()` entry point imported from `src/quarto.ts`. When the `QUARTO_TEST_BIN` environment variable points at an installed quarto, `runQuarto()` instead spawns that binary as a subprocess, with `--log --log-format json-stream` so the log-based verifiers keep working unchanged. This is used to run the smoke tests against a built distribution (see `dev-docs/smoke-tests-built-version-plan.md` for the design, and the `test-smokes-built.yml` CI workflow below). + +To run in binary mode locally: + +```bash +# 1. Build a distribution (after ./configure.sh) +cd package/src +./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" +cd ../.. + +# 2. Copy the built dist OUTSIDE the git checkout. An in-repo quarto +# (e.g. package/dist/bin/quarto) resolves to dev mode — the launcher runs +# the TS sources when it finds a sibling src/quarto.ts — and must NOT be +# used: run-tests.[sh|ps1] refuses a binary reporting the 99.9.9 dev +# version sentinel. +cp -r package/pkg-working ~/quarto-under-test + +# 3. Run the tests against it +cd tests +QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh +``` + +In binary mode: + +- With no test arguments, `run-tests.[sh|ps1]` defaults to `smoke/` only: `unit/` exercises quarto internals in-process and `integration/` requires the dev playwright setup, so both are dev-only. +- The test environment is configured as usual; set `QUARTO_TESTS_NO_CONFIG` to skip that step as in dev mode. +- Tests with `requiresDevQuarto: true` in their `TestContext` are ignored (rare escape hatch for tests that must exercise quarto internals in-process). + +Authoring rules that keep tests working in both modes: + +- Never `import { quarto } from "../src/quarto.ts"` in tests — invoke quarto through `testQuartoCmd()` (`tests/test.ts`) or `runQuarto()` (`tests/quarto-cmd.ts`). +- Tests that spawn quarto as a subprocess themselves should resolve the executable with `quartoDevCmd()` (`tests/utils.ts`, honors `QUARTO_TEST_BIN`) — or `quartoDevBinCmd()` (`tests/quarto-cmd.ts`) when the test must pin the locally-built dev CLI — and pass `quartoSpawnEnvOptions()` as spawn env options so the dev-tree env vars don't leak into the built quarto. + ## Debugging within tests `.vscode/launch.json` has a `Run Quarto test` configuration that can be used to debug when running tests. One need to modify the `program` and `args` fields to match the test to run. @@ -519,3 +554,5 @@ Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tes - `test-smokes.yml` is the main CI workflow which configure the environment, and run the tests on Ubuntu and Windows. - If it was triggerred by `workflow_call`, then it will run each test in using `run-tests.[sh|ps1]` in a for-loop. - Scheduled tests are still run daily in their sequential version. + - It is parameterized (`quarto-install`, `quarto-version`, `quarto-artifact-name`, `ref`, `runners`, ...) so callers can run the suite against a built quarto instead of the dev source tree: the workflow installs the quarto under test outside the checkout and exports `QUARTO_TEST_BIN` (see "Binary mode" above). +- `test-smokes-built.yml` runs the smoke tests against a **built** quarto (weekly on Monday, plus `workflow_dispatch`) by calling `test-smokes.yml`. Three modes: `build` (build a linux-amd64 dist from the checkout, via the shared `.github/actions/build-dist-tarball` composite action also used by `create-release.yml`), `nightly` (reuse the signed artifacts of a completed create-release run, Linux and Windows), and `release` (install a published (pre-)release). From d68cd217065fa47245123584b7cd89a4bde55545 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:32:06 +0000 Subject: [PATCH 18/43] Add architecture diagrams for the test harness and CI modes Four mermaid diagrams in tests/README.md (GitHub renders them natively): the runQuarto dispatch seam (dev vs binary mode), the lifecycle of one binary-mode test (per-invocation child log, merge, synthetic-ERROR guard, tree kill), the CI workflow map (which workflow tests what, and how the modes relate to create-release and the shared build action), and the QUARTO_TEST_BIN handoff chain from workflow input to per-test dispatch. All four validated with mermaid-cli. Co-Authored-By: Claude Fable 5 --- tests/README.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/README.md b/tests/README.md index 62fd9043ba..ffea14a88c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -465,6 +465,116 @@ Authoring rules that keep tests working in both modes: - Never `import { quarto } from "../src/quarto.ts"` in tests — invoke quarto through `testQuartoCmd()` (`tests/test.ts`) or `runQuarto()` (`tests/quarto-cmd.ts`). - Tests that spawn quarto as a subprocess themselves should resolve the executable with `quartoDevCmd()` (`tests/utils.ts`, honors `QUARTO_TEST_BIN`) — or `quartoDevBinCmd()` (`tests/quarto-cmd.ts`) when the test must pin the locally-built dev CLI — and pass `quartoSpawnEnvOptions()` as spawn env options so the dev-tree env vars don't leak into the built quarto. +## How tests run (diagrams) + +### Test invocation: one seam, two modes + +Every `testQuartoCmd()`-based test goes through a single dispatch point, +`runQuarto()` in `tests/quarto-cmd.ts`. The verifiers never know which mode +ran — they only read the json-stream log file and the rendered outputs. + +```mermaid +flowchart TB + subgraph deno ["Deno test process (tests/ harness, always runs from the repo checkout)"] + TQC["testQuartoCmd / testRender / testSite / smoke-all driver"] + RQ{"runQuarto()
tests/quarto-cmd.ts"} + LOG[("json-stream log file
{msg, level, levelName} per line")] + OUT[("rendered output files")] + VER["verifiers (tests/verify.ts)
noErrors, printsMessage, ensureHtmlElements, ..."] + end + DEV["in-process quarto()
imported from src/quarto.ts
dev TS sources, version 99.9.9"] + BIN["spawned subprocess: built quarto
--log file --log-format json-stream
dev env vars stripped (QUARTO_SHARE_PATH, DENO_DIR, ...)"] + + TQC --> RQ + RQ -->|"dev mode (default:
QUARTO_TEST_BIN unset)"| DEV + RQ -->|"binary mode
(QUARTO_TEST_BIN set)"| BIN + DEV --> LOG + DEV --> OUT + BIN --> LOG + BIN --> OUT + LOG --> VER + OUT --> VER +``` + +### Binary mode: lifecycle of one test + +```mermaid +sequenceDiagram + participant T as test() (tests/test.ts) + participant R as runQuarto() + participant Q as built quarto (subprocess) + participant V as verifiers + + T->>T: create temp json-stream log file + Note over T: binary mode: harness logger NOT initialized
(the child owns log capture) + T->>R: execute(logFile) + R->>Q: spawn QUARTO_TEST_BIN render ...
--log (per-invocation temp) --log-format json-stream
env = ambient minus dev-tree vars, plus TestContext.env + Q->>Q: render, write log records + output files + Q-->>R: exit (code, stdout/stderr drained) + R->>T: merge child log into the test log file + alt exit != 0 and no ERROR record in child log + R->>T: append synthetic ERROR record
(exit code + stderr tail) — prevents silent green + end + alt timeout + R->>Q: kill process tree (launcher spawns deno and waits) + R->>T: append timeout ERROR record + end + T->>V: verify(log records) + verify(output files) +``` + +### CI: which workflow tests what + +```mermaid +flowchart LR + subgraph dev ["Dev mode (unchanged): quarto = in-process TS sources"] + PR["PR / push"] --> TSP["test-smokes-parallel.yml
sharded buckets"] + DAILY["daily schedule"] --> TSfull["full run"] + FFM["test-ff-matrix.yml
(ff-matrix qmd bucket)"] + end + subgraph built ["Binary mode: quarto = built distribution (QUARTO_TEST_BIN)"] + TSB["test-smokes-built.yml
weekly Monday + manual dispatch"] + BUILDM["source: build (default)
build linux-amd64 dist from this ref"] + NIGHTM["source: nightly
reuse SIGNED artifacts of a
create-release run (linux + windows quarto.exe)"] + RELM["source: release
install published (pre-)release,
checkout its v-tag"] + TSB --> BUILDM + TSB --> NIGHTM + TSB --> RELM + end + CR["create-release.yml
nightly build (no publish),
dispatch = publish"] + ACT[".github/actions/build-dist-tarball
(shared build recipe)"] + + TS["test-smokes.yml (reusable)
inputs: quarto-install, ref, runners,
buckets, quarto-artifact-*"] + TSP --> TS + DAILY --> TS + FFM --> TS + BUILDM --> TS + NIGHTM --> TS + RELM --> TS + BUILDM -. uses .-> ACT + CR -. "make-tarball jobs use" .-> ACT + NIGHTM -. "downloads artifacts from" .-> CR +``` + +### Binary mode in CI: how `QUARTO_TEST_BIN` reaches the tests + +`QUARTO_TEST_BIN` is never declared statically — the "Pin and verify test +target" step in `test-smokes.yml` computes it at runtime and exports it via +`$GITHUB_ENV`, making it visible to every later step of the job. When +`quarto-install` is `dev` (all existing callers), these steps are skipped +and nothing changes. + +```mermaid +flowchart TB + IN["workflow input
quarto-install: artifact | release"] + QD["quarto-dev action (ALWAYS runs)
provisions the harness Deno runtime"] + INST["install quarto under test
artifact: extract to RUNNER_TEMP (outside checkout)
release: quarto-actions/setup"] + PIN["Pin and verify test target
refuse 99.9.9 dev sentinel; check semver shape;
echo QUARTO_TEST_BIN=path >> GITHUB_ENV"] + RTS["./run-tests.sh (unchanged invocation)
sees QUARTO_TEST_BIN: banner + guard,
default selection = smoke/ only"] + QC["tests/quarto-cmd.ts
Deno.env.get('QUARTO_TEST_BIN')
every runQuarto spawns the built binary"] + + IN --> QD --> INST --> PIN --> RTS --> QC +``` + ## Debugging within tests `.vscode/launch.json` has a `Run Quarto test` configuration that can be used to debug when running tests. One need to modify the `program` and `args` fields to match the test to run. From b4d0db67151c5dab15c631f064170566c90282ad Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:38:05 +0000 Subject: [PATCH 19/43] Simplify built-version workflow: schedule uses nightly artifacts, gh CLI resolution The weekly schedule previously rebuilt a dist that create-release had already built hours earlier from nearly the same commit - unsigned and linux-only, where the nightly artifacts are signed and include the Windows quarto.exe. The schedule now resolves to source=nightly; source=build becomes dispatch-only, keeping its unique job: testing an arbitrary ref (PR branches, forks, pre-merge validation) without needing signing secrets or existing create-release runs. Also replaces the curl/jq run-resolution with the preinstalled gh CLI (one readable line per lookup), and documents the on-demand signed-build composition: dispatch create-release with publish-release=false, then source=nightly with that run id. Mode resolution is one expression repeated in job conditions: schedule => nightly; dispatch => the source input (default build). CI diagram and plan updated to match. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-smokes-built.yml | 64 +++++++++++++------------ tests/README.md | 10 ++-- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index a93629c203..6ad6d98e3c 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -1,17 +1,24 @@ # Runs the smoke test suite against a BUILT quarto instead of the dev source tree. -# Three modes: -# - build (default; also the weekly schedule path): build a linux-amd64 dist from -# this checkout with `quarto-bld prepare-dist`, then run the smokes against it -# (harness and binary come from the same commit). -# - nightly (workflow_dispatch only): reuse the SIGNED artifacts of a completed -# create-release run (its nightly schedule builds without publishing) - the -# linux 'Deb Zip' tarball AND the signed 'Windows Zip' (the real quarto.exe) - -# and run the smokes at that run's commit. Preventive Windows coverage without -# any extra build or signing infrastructure. -# - release (workflow_dispatch only): install a published (pre-)release with -# quarto-actions/setup and run the smokes at the matching refs/tags/v. +# Three sources for the quarto under test: +# - nightly (the weekly schedule path; also dispatchable): reuse the SIGNED +# artifacts of a completed create-release run (its nightly schedule builds +# without publishing) - the linux 'Deb Zip' tarball AND the signed +# 'Windows Zip' (the real quarto.exe) - and run the smokes at that run's +# commit. No duplicate build, and Windows coverage for free. To test a +# signed build of a specific commit on demand: dispatch create-release with +# publish-release=false, then dispatch this workflow with source=nightly and +# that run's id. +# - build (dispatch default): build a linux-amd64 dist from THIS ref with the +# same recipe as create-release (shared build-dist-tarball action) and test +# it. The only mode that works for arbitrary refs/PR branches and on forks +# (no signing secrets or create-release runs required). +# - release: install a published (pre-)release with quarto-actions/setup and +# run the smokes at the matching refs/tags/v. # nightly/release only work when the target commit's harness has binary-mode # support (tests/quarto-cmd.ts) - see dev-docs/smoke-tests-built-version-plan.md §5.3. +# +# Mode resolution (repeated in each job's `if`): schedule => nightly; +# dispatch => the source input (default build). name: Smoke Tests (Built Version) on: schedule: @@ -19,7 +26,7 @@ on: workflow_dispatch: inputs: source: - description: "Test a quarto built from this ref (build), the artifacts of a create-release run (nightly), or a published (pre-)release (release)" + description: "Where the quarto under test comes from: build = build from this ref now; nightly = artifacts of a create-release run; release = published (pre-)release. The weekly schedule always uses nightly." required: false type: choice options: @@ -49,13 +56,10 @@ permissions: contents: read jobs: - # source: build (default; also the schedule path) + # source: build (dispatch default) - build from THIS ref build-artifact: name: Build quarto dist (linux-amd64) - if: >- - (github.event_name != 'schedule' || github.repository == 'quarto-dev/quarto-cli') - && github.event.inputs.source != 'release' - && github.event.inputs.source != 'nightly' + if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'build' runs-on: ubuntu-latest outputs: sha: ${{ steps.rec.outputs.sha }} @@ -87,7 +91,7 @@ jobs: run-smokes-artifact: name: Smoke tests against built artifact - if: github.event.inputs.source != 'release' && github.event.inputs.source != 'nightly' + if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'build' needs: [build-artifact] uses: ./.github/workflows/test-smokes.yml with: @@ -159,11 +163,14 @@ jobs: quarto-version: ${{ needs.resolve-release.outputs.version }} buckets: ${{ github.event.inputs.buckets }} # empty = full run - # source: nightly (workflow_dispatch only) - reuse the signed artifacts of a - # completed no-publish create-release run (linux tarball + signed Windows zip) + # source: nightly (the weekly schedule path; also dispatchable) - reuse the + # signed artifacts of a completed no-publish create-release run (linux + # tarball + signed Windows zip) resolve-nightly: name: Resolve create-release run - if: github.event.inputs.source == 'nightly' + if: >- + (github.event_name != 'schedule' || github.repository == 'quarto-dev/quarto-cli') + && (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' runs-on: ubuntu-latest outputs: run-id: ${{ steps.r.outputs.run-id }} @@ -177,19 +184,14 @@ jobs: run: | run_id="${{ github.event.inputs.run-id }}" if [ -z "$run_id" ]; then - run_id="$(curl -fsSL \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/create-release.yml/runs?status=success&per_page=1" \ - | jq -r '.workflow_runs[0].id // empty')" + run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow create-release.yml \ + --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')" fi if [ -z "$run_id" ]; then echo "::error::no successful create-release run found to take artifacts from" exit 1 fi - sha="$(curl -fsSL \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" \ - | jq -r '.head_sha // empty')" + sha="$(gh run view "$run_id" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha // empty')" if [ -z "$sha" ]; then echo "::error::could not resolve head_sha for run ${run_id}" exit 1 @@ -200,7 +202,7 @@ jobs: run-smokes-nightly-linux: name: Smoke tests against nightly build (linux) - if: github.event.inputs.source == 'nightly' + if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' needs: [resolve-nightly] uses: ./.github/workflows/test-smokes.yml with: @@ -214,7 +216,7 @@ jobs: run-smokes-nightly-windows: name: Smoke tests against nightly build (windows) - if: github.event.inputs.source == 'nightly' + if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' needs: [resolve-nightly] uses: ./.github/workflows/test-smokes.yml with: diff --git a/tests/README.md b/tests/README.md index ffea14a88c..93911b96b8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -533,12 +533,12 @@ flowchart LR end subgraph built ["Binary mode: quarto = built distribution (QUARTO_TEST_BIN)"] TSB["test-smokes-built.yml
weekly Monday + manual dispatch"] - BUILDM["source: build (default)
build linux-amd64 dist from this ref"] - NIGHTM["source: nightly
reuse SIGNED artifacts of a
create-release run (linux + windows quarto.exe)"] + BUILDM["source: build (dispatch default)
build linux-amd64 dist from this ref
(any ref, works on forks)"] + NIGHTM["source: nightly (weekly schedule)
reuse SIGNED artifacts of a
create-release run (linux + windows quarto.exe)"] RELM["source: release
install published (pre-)release,
checkout its v-tag"] - TSB --> BUILDM - TSB --> NIGHTM - TSB --> RELM + TSB -->|"dispatch"| BUILDM + TSB -->|"weekly schedule
+ dispatch"| NIGHTM + TSB -->|"dispatch"| RELM end CR["create-release.yml
nightly build (no publish),
dispatch = publish"] ACT[".github/actions/build-dist-tarball
(shared build recipe)"] From 3617271796d1f34fa07cfb3991dc397c66774487 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:38:46 +0000 Subject: [PATCH 20/43] Plan: record that the weekly schedule uses nightly mode Co-Authored-By: Claude Fable 5 --- dev-docs/smoke-tests-built-version-plan.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index 6d27aa257e..7b03a07cc4 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -546,7 +546,10 @@ Studied post-review (maintainer suggestion): the signing steps in `publish-release` — so every create-release run, including the nightly no-publish schedule, already uploads a **signed** `Windows Zip` (the real `quarto.exe`) and the linux `Deb Zip` as workflow artifacts. The -implemented `source: nightly` dispatch mode reuses them: resolve a +implemented `source: nightly` mode — **the weekly schedule path** (a +scheduled `build` would duplicate what create-release built hours +earlier, unsigned and linux-only; `build` stays dispatch-only for +arbitrary refs and forks) — reuses them: resolve a create-release run (explicit `run-id` input or latest successful), check out its `head_sha` for the harness (same-commit, no skew), download the artifacts cross-run (`test-smokes.yml` input `quarto-artifact-run-id` → From 3f7da72042ddc5e84b9dbcc45c494416d35ed639 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:51:38 +0000 Subject: [PATCH 21/43] Built-version testing goes daily with three-OS coverage Implements the conclusions of two studies (create-release branch dispatch; daily dev-vs-built migration): - test-smokes-built.yml now triggers via workflow_run after every completed nightly create-release build instead of a weekly cron: once per build, right after it, no stale-artifact risk (the previous cron resolved 'latest successful run', silently testing yesterday's commit when a build failed or was slow). A failed nightly build means a visibly skipped day, not a false green. - New macOS smoke leg from the nightly signed+notarized Mac Zip - the only macOS smoke coverage in CI (previously the Mac build only ever got quarto check). test-smokes.yml gains a macOS system-deps step (brew poppler/librsvg) and the artifact install step now detects layout instead of hardcoding strip-components (the linux tarball has a version top dir; the Mac tarball and Windows Zip are flat). - create-release.yml gains a smoke-artifacts-only dispatch input that skips source/arm64 tarballs, deb/rpm installers and the macOS build, making 'signed build of a branch tip' cheap (verified safe: all publish/tag/docker/cloudsmith paths are input-gated, no protected environments restrict secrets on branches, version_commit resolves empty so jobs build the dispatched ref); publish-release is guarded against partial builds. - The daily dev-mode test-smokes.yml schedule is deliberately left untouched: the plan records the remaining migration decisions (dedicated daily unit/integration job, then downgrade daily dev to weekly) as maintainer calls after a ~2-week green track record. Co-Authored-By: Claude Fable 5 --- .github/workflows/create-release.yml | 13 ++++- .github/workflows/test-smokes-built.yml | 55 ++++++++++++++++------ .github/workflows/test-smokes.yml | 33 ++++++++----- dev-docs/smoke-tests-built-version-plan.md | 21 +++++++++ tests/README.md | 8 ++-- 5 files changed, 98 insertions(+), 32 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index fb800e66e0..ff242d4d5e 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -14,6 +14,11 @@ on: required: false type: boolean default: true + smoke-artifacts-only: + description: "Build only the artifacts consumed by Smoke Tests (Built Version) nightly mode: the linux amd64 tarball and the signed Windows zip. Skips source/arm64 tarballs, deb/rpm installers, and the macOS build. For cheap signed builds of a branch (dispatch with publish-release=false); incompatible with publishing." + required: false + type: boolean + default: false env: NFPM_VERSION: "2.43.1" @@ -100,6 +105,7 @@ jobs: default_author: github_actions make-source-tarball: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] steps: @@ -142,6 +148,7 @@ jobs: artifact-name: Deb Zip make-arm64-tarball: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] steps: @@ -212,6 +219,7 @@ jobs: path: ./package/quarto-${{needs.configure.outputs.version}}-linux-rhel7-amd64.tar.gz make-installer-linux: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] strategy: @@ -443,6 +451,7 @@ jobs: quarto --version make-installer-mac: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: macos-latest needs: [configure] steps: @@ -512,6 +521,7 @@ jobs: security delete-keychain build.keychain test-zip-mac: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: macos-latest needs: [configure, make-installer-mac] steps: @@ -551,7 +561,8 @@ jobs: quarto --version publish-release: - if: ${{ inputs.publish-release }} + # never publish from a partial (smoke-artifacts-only) build + if: ${{ inputs.publish-release && !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [ configure, diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 6ad6d98e3c..1468749b4a 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -1,10 +1,12 @@ # Runs the smoke test suite against a BUILT quarto instead of the dev source tree. # Three sources for the quarto under test: -# - nightly (the weekly schedule path; also dispatchable): reuse the SIGNED -# artifacts of a completed create-release run (its nightly schedule builds -# without publishing) - the linux 'Deb Zip' tarball AND the signed -# 'Windows Zip' (the real quarto.exe) - and run the smokes at that run's -# commit. No duplicate build, and Windows coverage for free. To test a +# - nightly (the workflow_run path - runs after every nightly create-release +# build; also dispatchable): reuse the SIGNED artifacts of a completed +# create-release run (its nightly schedule builds without publishing) - +# the linux 'Deb Zip' tarball, the signed 'Windows Zip' (the real +# quarto.exe), AND the signed+notarized 'Mac Zip' - and run the smokes at +# that run's commit. No duplicate build, three-OS coverage for free +# (the ONLY macOS smoke coverage in CI). To test a # signed build of a specific commit on demand: dispatch create-release with # publish-release=false, then dispatch this workflow with source=nightly and # that run's id. @@ -17,12 +19,17 @@ # nightly/release only work when the target commit's harness has binary-mode # support (tests/quarto-cmd.ts) - see dev-docs/smoke-tests-built-version-plan.md §5.3. # -# Mode resolution (repeated in each job's `if`): schedule => nightly; +# Mode resolution (repeated in each job's `if`): workflow_run => nightly; # dispatch => the source input (default build). name: Smoke Tests (Built Version) on: - schedule: - - cron: "0 8 * * 1" # weekly, Monday + # Fires once per completed nightly create-release build (main only: + # workflow_run triggers require the workflow file on the default branch). + # This makes built-version testing DAILY, right after each build, with no + # stale-artifact risk (vs a cron that resolves "latest successful run"). + workflow_run: + workflows: ["Build Installers"] + types: [completed] workflow_dispatch: inputs: source: @@ -59,7 +66,7 @@ jobs: # source: build (dispatch default) - build from THIS ref build-artifact: name: Build quarto dist (linux-amd64) - if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'build' + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' runs-on: ubuntu-latest outputs: sha: ${{ steps.rec.outputs.sha }} @@ -91,7 +98,7 @@ jobs: run-smokes-artifact: name: Smoke tests against built artifact - if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'build' + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' needs: [build-artifact] uses: ./.github/workflows/test-smokes.yml with: @@ -168,9 +175,11 @@ jobs: # tarball + signed Windows zip) resolve-nightly: name: Resolve create-release run + # on workflow_run: only test successful builds (a skipped day is visible, + # a stale artifact is not) if: >- - (github.event_name != 'schedule' || github.repository == 'quarto-dev/quarto-cli') - && (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') runs-on: ubuntu-latest outputs: run-id: ${{ steps.r.outputs.run-id }} @@ -182,7 +191,8 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - run_id="${{ github.event.inputs.run-id }}" + # workflow_run: the triggering build IS the run to test + run_id="${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.event.inputs.run-id }}" if [ -z "$run_id" ]; then run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow create-release.yml \ --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')" @@ -202,7 +212,7 @@ jobs: run-smokes-nightly-linux: name: Smoke tests against nightly build (linux) - if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' needs: [resolve-nightly] uses: ./.github/workflows/test-smokes.yml with: @@ -216,7 +226,7 @@ jobs: run-smokes-nightly-windows: name: Smoke tests against nightly build (windows) - if: (github.event_name == 'schedule' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' needs: [resolve-nightly] uses: ./.github/workflows/test-smokes.yml with: @@ -227,3 +237,18 @@ jobs: quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["windows-latest"]' buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-mac: + name: Smoke tests against nightly build (macOS) + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # the SIGNED + notarized Mac layout produced by make-installer-mac - + # the only macOS smoke coverage in CI + quarto-artifact-name: Mac Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["macos-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 5a50ae9b0c..a9de8807f6 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -208,6 +208,14 @@ jobs: sudo apt-get update -y sudo apt-get install -y librsvg2-bin + - name: Install missing system deps (macOS) + if: runner.os == 'macOS' + # macos runners preinstall most toolchain deps; poppler (pdftotext/ + # pdftoppm) and librsvg (rsvg-convert) are the ones PDF tests need. + # Tests requiring other optional tools prereq-skip gracefully. + run: | + brew install poppler librsvg + - name: Restore R packages working-directory: tests run: | @@ -302,18 +310,19 @@ jobs: echo "::error::expected exactly one .tar.gz or .zip in the quarto artifact, found: ${artifacts[*]:-none}" exit 1 fi - case "${artifacts[0]}" in - *.tar.gz) - # linux tarball layout: quarto-/{bin,share} - tar -xzf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" --strip-components=1 - ;; - *.zip) - # Windows Zip layout: {bin,share} at the archive root; bsdtar on - # windows runners extracts zip archives (same as test-zip-win in - # create-release.yml) - tar -xf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" - ;; - esac + # layouts differ per artifact: the linux tarball has a + # quarto-/ top dir; the Mac tarball and Windows Zip are flat + # ({bin,share} at the archive root). Extract, then flatten if needed. + # bsdtar (windows/mac runners) also handles .zip via tar -xf. + tar -xf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" + if [ ! -d "$RUNNER_TEMP/quarto-under-test/bin" ]; then + top="$(find "$RUNNER_TEMP/quarto-under-test" -mindepth 1 -maxdepth 1 -type d | head -1)" + if [ -z "$top" ] || [ ! -d "$top/bin" ]; then + echo "::error::could not locate bin/ in the extracted quarto artifact" + exit 1 + fi + mv "$top"/* "$RUNNER_TEMP/quarto-under-test"/ + fi echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" - name: Pin and verify test target diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index 7b03a07cc4..5c9cbf6cbd 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -624,6 +624,27 @@ harness/binary decoupling (via `quarto inspect`) if testing binaries from a different commit than the harness (incl. true backfill) becomes worth the drift risk. +**Phase 5 — daily built-version as the primary signal (studied +2026-07-17, maintainer proposal).** Verified: `create-release.yml` can +be dispatched on any branch tip with `publish-release=false` — all +publish/tag/docker/cloudsmith paths are input-gated, secrets are +available on non-default branches (no protected environments), and +`version_commit` resolves empty so jobs build the dispatched ref. The +`smoke-artifacts-only` input makes such branch builds cheap (linux +tarball + signed Windows zip only). Implemented now (additive — Phase +A): `test-smokes-built.yml` triggers via **`workflow_run` after every +nightly create-release build** (once per build, no stale-artifact +risk; a failed build means a visibly skipped day, not a silent stale +test) and gained a **macOS leg** from the nightly `Mac Zip` — the only +macOS smoke coverage in CI. Remaining maintainer decisions, AFTER a +green track record (~2 weeks): (Phase B) give `unit/` + `integration/` +(74 files, currently covered by every PR run) a cheap dedicated daily +dev job so they keep daily env-drift coverage; (Phase C) downgrade the +daily dev `test-smokes.yml` cron to weekly — not delete: a weekly dev +run keeps the dev-only classes covered (QUARTO_DEBUG paths, `quarto +check` dev branch, in-process races, Windows playwright setup) and the +safety net for paths-ignored doc-only PRs. + ## 7. Risks & open items - **Per-spawn cost at corpus scale**: 1500+ spawns × product cold-start diff --git a/tests/README.md b/tests/README.md index 93911b96b8..f4a71f1a39 100644 --- a/tests/README.md +++ b/tests/README.md @@ -532,15 +532,15 @@ flowchart LR FFM["test-ff-matrix.yml
(ff-matrix qmd bucket)"] end subgraph built ["Binary mode: quarto = built distribution (QUARTO_TEST_BIN)"] - TSB["test-smokes-built.yml
weekly Monday + manual dispatch"] + TSB["test-smokes-built.yml
runs after every nightly build
(workflow_run) + manual dispatch"] BUILDM["source: build (dispatch default)
build linux-amd64 dist from this ref
(any ref, works on forks)"] - NIGHTM["source: nightly (weekly schedule)
reuse SIGNED artifacts of a
create-release run (linux + windows quarto.exe)"] + NIGHTM["source: nightly (daily via workflow_run)
reuse SIGNED artifacts of a create-release run:
linux + windows quarto.exe + macOS
(the only macOS smoke coverage)"] RELM["source: release
install published (pre-)release,
checkout its v-tag"] TSB -->|"dispatch"| BUILDM - TSB -->|"weekly schedule
+ dispatch"| NIGHTM + TSB -->|"after each nightly build
+ dispatch"| NIGHTM TSB -->|"dispatch"| RELM end - CR["create-release.yml
nightly build (no publish),
dispatch = publish"] + CR["create-release.yml
nightly build (no publish),
dispatch = publish; smoke-artifacts-only
input = cheap signed branch builds"] ACT[".github/actions/build-dist-tarball
(shared build recipe)"] TS["test-smokes.yml (reusable)
inputs: quarto-install, ref, runners,
buckets, quarto-artifact-*"] From 95b3847fa68245af77207e9087701fa81c60fb8b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:52:47 +0000 Subject: [PATCH 22/43] Document the macOS-runner policy on the runners input macOS smoke runs are reserved for the scheduled built-version path (test-smokes-built.yml nightly mode, once per nightly build); it must not be added to per-commit callers, which need to stay fast. Encodes maintainer intent so the constraint survives as policy rather than just current wiring. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-smokes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index a9de8807f6..7de41e5b1f 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -45,7 +45,7 @@ on: type: string default: "" runners: - description: "JSON list of runner labels for the OS matrix" + description: "JSON list of runner labels for the OS matrix. Policy: macos-latest is reserved for scheduled/built-version runs (test-smokes-built.yml nightly mode) - do NOT add it to per-commit callers (test-smokes-parallel.yml), which must stay fast." required: false type: string default: '["ubuntu-latest", "windows-latest"]' From 05233c4e64009643a07c7ca3085332d396e13ee1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 14:01:32 +0000 Subject: [PATCH 23/43] docs: purge stale weekly-cron wording after workflow_run migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trigger for test-smokes-built.yml changed from a weekly Monday cron to workflow_run on each nightly Build Installers run, and the nightly mode gained a macOS leg. The mermaid diagrams were updated at the time but three prose spots still described the old design: - tests/README.md CI section (weekly on Monday; nightly = Linux+Windows) - test-smokes-built.yml source input description and nightly job comment - plan doc §5.2 trigger sketch, §5.2b framing, and Phase 2 acceptance Co-Authored-By: Claude Fable 5 --- .github/workflows/test-smokes-built.yml | 8 ++++---- dev-docs/smoke-tests-built-version-plan.md | 24 +++++++++++++--------- tests/README.md | 2 +- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 1468749b4a..778d16450e 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -33,7 +33,7 @@ on: workflow_dispatch: inputs: source: - description: "Where the quarto under test comes from: build = build from this ref now; nightly = artifacts of a create-release run; release = published (pre-)release. The weekly schedule always uses nightly." + description: "Where the quarto under test comes from: build = build from this ref now; nightly = artifacts of a create-release run; release = published (pre-)release. The workflow_run trigger (after each nightly build) always uses nightly." required: false type: choice options: @@ -170,9 +170,9 @@ jobs: quarto-version: ${{ needs.resolve-release.outputs.version }} buckets: ${{ github.event.inputs.buckets }} # empty = full run - # source: nightly (the weekly schedule path; also dispatchable) - reuse the - # signed artifacts of a completed no-publish create-release run (linux - # tarball + signed Windows zip) + # source: nightly (the workflow_run path, daily after each build; also + # dispatchable) - reuse the signed artifacts of a completed no-publish + # create-release run (linux tarball + signed Windows zip + notarized Mac zip) resolve-nightly: name: Resolve create-release run # on workflow_run: only test successful builds (a skipped day is visible, diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index 5c9cbf6cbd..e423db20ae 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -470,10 +470,12 @@ automatically; language setup is shared. ```yaml name: Smoke Tests (Built Version) on: - schedule: [{ cron: "0 8 * * 1" }] # weekly, Monday + workflow_run: # daily: after each nightly build (§5.2b, §6 Phase 5) + workflows: ["Build Installers"] + types: [completed] workflow_dispatch: inputs: - source: { type: choice, options: [build, release], default: build } + source: { type: choice, options: [build, nightly, release], default: build } version: { type: string, default: "pre-release" } # for source: release jobs: @@ -546,10 +548,11 @@ Studied post-review (maintainer suggestion): the signing steps in `publish-release` — so every create-release run, including the nightly no-publish schedule, already uploads a **signed** `Windows Zip` (the real `quarto.exe`) and the linux `Deb Zip` as workflow artifacts. The -implemented `source: nightly` mode — **the weekly schedule path** (a -scheduled `build` would duplicate what create-release built hours -earlier, unsigned and linux-only; `build` stays dispatch-only for -arbitrary refs and forks) — reuses them: resolve a +implemented `source: nightly` mode — **the `workflow_run` path, firing +daily after each completed nightly build** (a scheduled `build` would +duplicate what create-release built hours earlier, unsigned and +linux-only; `build` stays dispatch-only for arbitrary refs and forks) — +reuses them: resolve a create-release run (explicit `run-id` input or latest successful), check out its `head_sha` for the harness (same-commit, no skew), download the artifacts cross-run (`test-smokes.yml` input `quarto-artifact-run-id` → @@ -558,7 +561,7 @@ OSes. This is the **preventive Windows coverage** path — signed shipped binaries, zero extra build/signing infrastructure. The build recipe itself is also shared now: `.github/actions/build-dist-tarball` is used by both `create-release.yml` (amd64 + arm64 tarballs) and the `build` -mode, so the weekly build exercises the release recipe by construction. +mode, so `build` runs exercise the release recipe by construction. Constraint: works only for create-release runs whose commit contains the binary-mode harness (post-merge); the preflight fails clearly otherwise. @@ -610,9 +613,10 @@ green locally; the corrupt-dist scenario fails red. **Phase 2 — CI.** Parameterize `test-smokes.yml` (§5.1 incl. `runners`, `buckets` default, Windows-gate fix); add `test-smokes-built.yml` -(weekly + dispatch, build-then-test, full smoke-all, linux). Acceptance: -first green (or fully triaged) weekly run; failures classified into -product bugs / harness assumptions / dev-only. +(dispatch build-then-test, full smoke-all, linux; the scheduled trigger +later became `workflow_run` on the nightly build — §6 Phase 5). +Acceptance: first green (or fully triaged) built-version run; failures +classified into product bugs / harness assumptions / dev-only. **Phase 3 — broaden.** `source: release` path for releases cut after Phase 1 (Windows coverage via `quarto.exe`; §5.3 scope), ff-matrix diff --git a/tests/README.md b/tests/README.md index f4a71f1a39..ce32b8eabd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -665,4 +665,4 @@ Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tes - If it was triggerred by `workflow_call`, then it will run each test in using `run-tests.[sh|ps1]` in a for-loop. - Scheduled tests are still run daily in their sequential version. - It is parameterized (`quarto-install`, `quarto-version`, `quarto-artifact-name`, `ref`, `runners`, ...) so callers can run the suite against a built quarto instead of the dev source tree: the workflow installs the quarto under test outside the checkout and exports `QUARTO_TEST_BIN` (see "Binary mode" above). -- `test-smokes-built.yml` runs the smoke tests against a **built** quarto (weekly on Monday, plus `workflow_dispatch`) by calling `test-smokes.yml`. Three modes: `build` (build a linux-amd64 dist from the checkout, via the shared `.github/actions/build-dist-tarball` composite action also used by `create-release.yml`), `nightly` (reuse the signed artifacts of a completed create-release run, Linux and Windows), and `release` (install a published (pre-)release). +- `test-smokes-built.yml` runs the smoke tests against a **built** quarto (daily, via `workflow_run` after each nightly `create-release.yml` build, plus `workflow_dispatch`) by calling `test-smokes.yml`. Three modes: `build` (build a linux-amd64 dist from the checkout, via the shared `.github/actions/build-dist-tarball` composite action also used by `create-release.yml`), `nightly` (reuse the signed artifacts of a completed create-release run — Linux, Windows, and macOS, the only macOS smoke coverage in CI), and `release` (install a published (pre-)release). From 4be0328be48980a562e0e345f9c2165490b85298 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 17 Jul 2026 16:21:24 +0200 Subject: [PATCH 24/43] Use gh api for release-tag preflight check gh (2.96+ on the runner) auto-reads GH_TOKEN and hits the same contents endpoint, so the manual Authorization/Accept headers and HTTP-status string compare go away in favor of a plain exit-code check. --- .github/workflows/test-smokes-built.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 778d16450e..4873ed9900 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -149,12 +149,8 @@ jobs: # tested in this mode (the harness at the tag lacks tests/quarto-cmd.ts); # test-smokes.yml's "Pin and verify" step re-checks this in the checkout. run: | - status="$(curl -s -o /dev/null -w '%{http_code}' \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - -H "Accept: application/vnd.github.raw+json" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/contents/tests/quarto-cmd.ts?ref=refs/tags/v${VERSION}")" - if [ "$status" != "200" ]; then - echo "::error::tests/quarto-cmd.ts not found at refs/tags/v${VERSION} (HTTP ${status}): either the tag does not exist or the release predates binary-mode harness support - it cannot be tested in release mode" + if ! gh api "repos/${GITHUB_REPOSITORY}/contents/tests/quarto-cmd.ts?ref=refs/tags/v${VERSION}" --silent 2>/dev/null; then + echo "::error::tests/quarto-cmd.ts not found at refs/tags/v${VERSION}: either the tag does not exist or the release predates binary-mode harness support - it cannot be tested in release mode" exit 1 fi echo "refs/tags/v${VERSION} has binary-mode harness support" From a88598c0958f4abde89911a8898f63bea61691c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 14:32:02 +0000 Subject: [PATCH 25/43] docs: add built-version testing architecture doc with design decision records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New llm-docs/built-version-testing-architecture.md following the llm-docs conventions (staleness frontmatter, *-architecture.md naming): architecture summary, mode-selection guidance, and ten decision records (D1-D10) capturing settled trade-offs - notably D1, the nightly wiring choice (workflow_run over create-release dispatching/calling the tests, and over the inversion where test-smokes-built would workflow_call create-release with smoke-artifacts-only). Also records the D1 analysis compactly in the plan doc (new §5.2c) and cross-references the new doc from tests/README.md, testing-patterns.md, and .claude/rules/testing/overview.md. Co-Authored-By: Claude Fable 5 --- .claude/rules/testing/overview.md | 2 +- dev-docs/smoke-tests-built-version-plan.md | 21 ++ .../built-version-testing-architecture.md | 189 ++++++++++++++++++ llm-docs/testing-patterns.md | 6 +- tests/README.md | 2 +- 5 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 llm-docs/built-version-testing-architecture.md diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index 43731e1467..46b79f198d 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -28,7 +28,7 @@ QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh test.ts # Linux/macOS $env:QUARTO_TESTS_NO_CONFIG=$true; .\run-tests.ps1 # Windows ``` -**Binary mode:** set `QUARTO_TEST_BIN` to a built quarto (extracted outside the checkout — the runner refuses the 99.9.9 dev sentinel) to spawn it instead of running the dev sources in-process; defaults to `smoke/` only (`unit/` and `integration/` are dev-only). See `tests/README.md` → "Binary mode". +**Binary mode:** set `QUARTO_TEST_BIN` to a built quarto (extracted outside the checkout — the runner refuses the 99.9.9 dev sentinel) to spawn it instead of running the dev sources in-process; defaults to `smoke/` only (`unit/` and `integration/` are dev-only). See `tests/README.md` → "Binary mode"; architecture and design decisions in `llm-docs/built-version-testing-architecture.md`. ## Test Types diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md index e423db20ae..8dbcaa411a 100644 --- a/dev-docs/smoke-tests-built-version-plan.md +++ b/dev-docs/smoke-tests-built-version-plan.md @@ -565,6 +565,27 @@ mode, so `build` runs exercise the release recipe by construction. Constraint: works only for create-release runs whose commit contains the binary-mode harness (post-merge); the preflight fails clearly otherwise. +### 5.2c Wiring decision: why `workflow_run` (2026-07, maintainer question) + +Considered and rejected: create-release *dispatching* the test workflow +(hand-rolled `workflow_run` needing `actions: write` + code in the +release workflow); create-release *calling* `test-smokes.yml` via +`workflow_call` (same-run artifacts, but smoke results would redden +`Build Installers` runs — including real publishes — and testing config +would live in the release workflow); and the *inversion* — +`test-smokes-built` owning the schedule and `workflow_call`ing +create-release (feasible: create-release uses the `inputs.` context +throughout, and `github.event_name` propagates from the caller; but +`smoke-artifacts-only` skips `make-installer-mac` so the daily would +need the full build anyway — zero compute saved, no double build exists +today — and it would make the nightly build canary depend on the test +workflow's health). `workflow_run` keeps the release pipeline untouched +and the failure signals separate ("Build Installers" red = pipeline +broke; "Smoke Tests (Built Version)" red = product broke). Accepted +weaknesses: the trigger couples on the workflow *name* string, and a +never-fired trigger is silent (mitigated by daily cadence). Full +decision record: `llm-docs/built-version-testing-architecture.md` (D1). + ### 5.3 Release mode scope **[R]** Release mode checks out the tag while the *workflow file and local diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md new file mode 100644 index 0000000000..3713000197 --- /dev/null +++ b/llm-docs/built-version-testing-architecture.md @@ -0,0 +1,189 @@ +--- +main_commit: a3f6218b7 +analyzed_date: 2026-07-17 +key_files: + - tests/quarto-cmd.ts + - tests/test.ts + - tests/run-tests.sh + - tests/run-tests.ps1 + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-built.yml + - .github/workflows/create-release.yml + - .github/actions/build-dist-tarball/action.yml +--- + +# Built-Version Testing Architecture + +How the smoke test suite runs against a **built** quarto distribution +(binary mode) in addition to the dev source tree, and — most importantly — +**why the design is the way it is**. This is the decision record; consult it +before changing the harness seam or the CI wiring, so settled trade-offs are +not accidentally relitigated. + +Document map: + +- **This doc** — architecture summary + design decisions (the durable "why"). +- `tests/README.md` → "Binary mode" and "How tests run (diagrams)" — the + operational "how" (local recipe, authoring rules, four mermaid diagrams). +- `llm-docs/testing-patterns.md` → "Dev Mode vs Binary Mode" — authoring + patterns for tests that must work in both modes. +- `dev-docs/smoke-tests-built-version-plan.md` — the original plan/spec with + full grounding evidence, roadmap phases, and open items (historical record; + this doc supersedes it as the go-to reference for the settled design). + +## Architecture in one paragraph + +Every `testQuartoCmd()`-based test invokes quarto through a single dispatch +point, `runQuarto()` in `tests/quarto-cmd.ts`. In **dev mode** (default) it +calls the in-process `quarto()` entry point from `src/quarto.ts`, exactly as +the harness always has. In **binary mode** (`QUARTO_TEST_BIN` set to a built +quarto extracted *outside* the checkout) it spawns that binary as a +subprocess with `--log --log-format json-stream`, merges the child's +log into the test's log file, and the verifiers run unchanged — they only +ever see log records and rendered outputs. CI-side, the reusable +`test-smokes.yml` gained `quarto-install: dev | release | artifact` inputs +(dev callers are untouched), and `test-smokes-built.yml` orchestrates three +sources for the binary under test: `build` (build from this ref, dispatch +only), `nightly` (reuse the signed artifacts of a nightly `create-release` +build — fires automatically via `workflow_run` after each one), and +`release` (install a published (pre-)release at its tag). + +## When to use which mode + +| Mode | Quarto under test | Trigger | Question answered | +|---|---|---|---| +| dev (`test-smokes.yml`) | in-process TS sources (99.9.9) | every PR/push + daily cron | did this code change break behavior? | +| nightly | signed nightly artifacts (Linux tarball, signed `quarto.exe`, notarized Mac zip) | automatic, after each nightly build | does what we *ship* work? (bundling/packaging/launcher bugs; only macOS smoke coverage in CI) | +| build | fresh linux-amd64 dist from the current ref (unsigned) | manual dispatch | will *this branch* survive packaging? (works on forks/PR branches) | +| release | published (pre-)release via quarto-actions/setup, harness at its `v` tag | manual dispatch | is the version users download healthy? (curative, post-publish) | + +Dev mode and built modes are complementary, not redundant: dev covers +`unit/`, `integration/`, `QUARTO_DEBUG` paths, and the `quarto check` dev +branch; built modes cover the packaged product dev mode never executes. + +## Design decisions + +Each entry: what was decided, why, and what would justify revisiting. + +### D1. Nightly wiring: `workflow_run`, not `workflow_call`/dispatch/inversion + +**Decision.** `test-smokes-built.yml` listens for completed "Build +Installers" (`create-release.yml`) runs via `workflow_run` and reuses their +artifacts cross-run (`quarto-artifact-run-id`). The release pipeline is not +modified for testing purposes. + +**Alternatives considered (2026-07, maintainer question):** + +- *create-release dispatches the test workflow at the end* — `workflow_run` + hand-rolled: needs `actions: write` + `gh workflow run` code inside the + release workflow, same default-branch constraint. Strictly dominated. +- *create-release `workflow_call`s `test-smokes.yml` after building* — + same-run artifacts (no run-id resolution) and an explicit DAG, but smoke + results would redden `Build Installers` runs including real publishes; + gating to schedule-only moves testing configuration (buckets, runner + policy) into the release workflow permanently. +- *Inversion: `test-smokes-built` owns the daily schedule and calls + `create-release` via `workflow_call`* — feasible (create-release uses the + `inputs.` context exclusively, which works under `workflow_call`; its + `github.event_name == 'schedule'` guard still behaves because a called + workflow sees the caller's event). Rejected because: (a) + `smoke-artifacts-only` skips `make-installer-mac`, so the daily run would + need the full build anyway — zero compute saved (there is no double + build today: one nightly build, one test pass reusing its artifacts); + (b) it reverses the dependency — the nightly build is also a + release-pipeline canary (signing certs, notarization, installer tooling) + and must not die when the test workflow is broken or paused; (c) nightly + builds would disappear from the "Build Installers" run history. + +**Why `workflow_run` wins:** zero risk to the most sensitive workflow in +the repo, and clean failure attribution — "Build Installers" red = the +pipeline broke; "Smoke Tests (Built Version)" red = the product broke. + +**Known weaknesses (accepted):** the trigger couples on the workflow +*display name* string (`workflows: ["Build Installers"]`; renaming +create-release's `name:` silently stops the trigger), and "trigger never +fired" is silent (mitigated by daily cadence — an absent run is visible). + +**Revisit when:** the system has a green track record and maintainers want +one atomic nightly build-and-test signal — then the inversion is the +principled consolidation, done as a deliberate follow-up. + +### D2. Version marker: semver *build metadata* (`X.Y.Z+test.YYYYMMDD`) + +Built test dists are stamped `$(cat version.txt)+test.$(date +%Y%m%d)`. +Never a `-suffix` (prerelease versions fail every plain `>=X.Y` +`quarto-required` range — the vendored semver has no `includePrerelease` +anywhere) and never a 4th dot component (not semver; the vendored +`deno.land/x/semver@1.4.0` throws). Build metadata is range-transparent for +every gate while still distinguishable from the `99.9.9` dev sentinel. + +### D3. Dist outside the checkout + `99.9.9` sentinel refusal + +Installed launchers detect dev mode via a sibling `src/quarto.ts`: an +in-repo `package/dist/bin/quarto` silently runs the TS sources instead of +the built code. Therefore the dist under test must be extracted *outside* +the repo, and both `run-tests.[sh|ps1]` and `assertTestBinary()` refuse a +binary reporting `99.9.9` (`kLocalDevelopment`). CI extracts artifacts to +`RUNNER_TEMP`. + +### D4. Child env: inherit ambient + strip dev vars (not clearEnv+allowlist) + +Binary-mode spawns inherit the ambient environment minus a strip list +(`QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, `DENO_DIR`, `QUARTO_DEBUG`, +`QUARTO_FORCE_VERSION`, ...), with `TestContext.env` overlaid last. A +clearEnv+allowlist was rejected: the Windows system-variable surface +(`SystemRoot`, `PATHEXT`, ...) is unenumerable in practice. The dev-tree +exports in `run-tests.[sh|ps1]` are kept in all modes — the *harness* +process still needs them; only the *child* is sanitized. + +### D5. Silent-green guard: synthetic ERROR records + +"Non-zero exit ⇒ ERROR record in the log" is NOT an invariant +(pre-logger-init failures, `quarto add/remove` commandFailed, pandoc/typst +passthroughs), and ~23% of smoke-all docs are verified only via the log. So +`runQuarto()` appends a synthetic `{level: 40}` record (exit code + stderr +tail) when a non-zero exit leaves the child log record-free, and a timeout +record when the process-tree kill fires (`pgrep -P` walk on POSIX — +portable to macOS, unlike `ps --ppid`; `taskkill /T` on Windows). + +### D6. `QUARTO_TEST_BIN` is set at runtime, never declared statically + +The "Pin and verify test target" step in `test-smokes.yml` resolves the +installed binary, verifies it (sentinel refusal, semver shape, optional +`QUARTO_TEST_EXPECTED_VERSION` match), and exports it via `$GITHUB_ENV` so +every later step — including the unchanged `run-tests.sh` invocation — +sees it. With `quarto-install: dev` (all pre-existing callers) these steps +are skipped and nothing changes. + +### D7. `smoke-artifacts-only` is for cheap *branch* builds, not the daily path + +The `create-release.yml` input skips source/arm64 tarballs and the Mac +installer, yielding a fast signed Linux+Windows build for on-demand testing +of a branch (`dispatch create-release publish-release=false +smoke-artifacts-only=true`, then `dispatch test-smokes-built source=nightly +run-id=`). It deliberately does NOT feed the daily path: the +daily needs the full build (Mac Zip = the only macOS smoke coverage), and +`publish-release` is hard-guarded against partial builds. + +### D8. macOS runners: scheduled/built runs only, never per-commit + +`test-smokes-parallel.yml` (per-commit) must stay fast, so it never passes +`runners` and keeps the `ubuntu-latest`/`windows-latest` default. The only +`macos-latest` smoke job is the nightly Mac leg in `test-smokes-built.yml`. +Encoded in the `runners` input description in `test-smokes.yml`. + +### D9. The daily dev cron stays (for now) + +Built-version nightly runs do not replace the daily dev `test-smokes.yml` +schedule: dev mode uniquely covers `QUARTO_DEBUG` paths, the `quarto check` +dev branch, in-process races, and doc-only PRs. Pending maintainer decision +after ~2 weeks of green nightly runs (plan §6 Phase 5): add a cheap daily +dev job for `unit/` + `integration/`, then downgrade the daily dev cron to +weekly — never delete it. + +### D10. Release mode only works for post-harness tags + +Release mode checks out the tag, so the harness at that tag must already +contain `tests/quarto-cmd.ts` — a preflight fails clearly for older +releases. True backfill (main-branch harness testing an older binary) +would require harness/binary decoupling (plan §6 Phase 4, not implemented). diff --git a/llm-docs/testing-patterns.md b/llm-docs/testing-patterns.md index 8aafadddf2..72882bdb22 100644 --- a/llm-docs/testing-patterns.md +++ b/llm-docs/testing-patterns.md @@ -36,8 +36,10 @@ under test: `QUARTO_DEBUG`, `DENO_DIR`, ...) are stripped from the child. `run-tests.sh`/`.ps1` refuse a binary reporting the `99.9.9` dev sentinel and default the selection to `smoke/` only (`unit/` and `integration/` are - dev-only). Exercised by `.github/workflows/test-smokes-built.yml`; full - design in `dev-docs/smoke-tests-built-version-plan.md`. + dev-only). Exercised by `.github/workflows/test-smokes-built.yml`. + Architecture and design decisions: + `llm-docs/built-version-testing-architecture.md` (original plan: + `dev-docs/smoke-tests-built-version-plan.md`). Consequences for writing smoke tests: diff --git a/tests/README.md b/tests/README.md index ce32b8eabd..de03ce9402 100644 --- a/tests/README.md +++ b/tests/README.md @@ -432,7 +432,7 @@ Don't do ### Binary mode (`QUARTO_TEST_BIN`) -By default, tests run quarto **in-process** from the dev sources: `runQuarto()` in `tests/quarto-cmd.ts` calls the `quarto()` entry point imported from `src/quarto.ts`. When the `QUARTO_TEST_BIN` environment variable points at an installed quarto, `runQuarto()` instead spawns that binary as a subprocess, with `--log --log-format json-stream` so the log-based verifiers keep working unchanged. This is used to run the smoke tests against a built distribution (see `dev-docs/smoke-tests-built-version-plan.md` for the design, and the `test-smokes-built.yml` CI workflow below). +By default, tests run quarto **in-process** from the dev sources: `runQuarto()` in `tests/quarto-cmd.ts` calls the `quarto()` entry point imported from `src/quarto.ts`. When the `QUARTO_TEST_BIN` environment variable points at an installed quarto, `runQuarto()` instead spawns that binary as a subprocess, with `--log --log-format json-stream` so the log-based verifiers keep working unchanged. This is used to run the smoke tests against a built distribution (see `llm-docs/built-version-testing-architecture.md` for the architecture and design decisions, `dev-docs/smoke-tests-built-version-plan.md` for the original plan, and the `test-smokes-built.yml` CI workflow below). To run in binary mode locally: From 4e03512be16f6c043996d8acb578773ea2003edd Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 14:34:55 +0000 Subject: [PATCH 26/43] docs: add practical when-to-trigger guidance for built-version modes The architecture doc had the mode table but not the operational guidance (nightly = automatic workhorse; build = on-demand for packaging/harness branches, with the smoke-artifacts-only recipe for signed branch builds; release = curative post-publish check). Add it there, and mirror a compact trigger/purpose table in tests/README.md next to the test-smokes-built.yml description where dispatchers will look. Co-Authored-By: Claude Fable 5 --- .../built-version-testing-architecture.md | 20 +++++++++++++++++++ tests/README.md | 10 +++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md index 3713000197..aff52b4150 100644 --- a/llm-docs/built-version-testing-architecture.md +++ b/llm-docs/built-version-testing-architecture.md @@ -61,6 +61,26 @@ Dev mode and built modes are complementary, not redundant: dev covers `unit/`, `integration/`, `QUARTO_DEBUG` paths, and the `quarto check` dev branch; built modes cover the packaged product dev mode never executes. +In practice: + +- **Mostly, trigger nothing.** `nightly` fires itself after each nightly + build and is the preventive workhorse: it tests the exact signed + binaries the release pipeline produces, on all three OSes, *before* + anything is published. Red here means we caught it before users did. +- **Dispatch `build`** when a branch touches packaging or the harness + itself (`prepare-dist`, `configure`, `tests/quarto-cmd.ts`, ...) and you + want built-version feedback on *that ref* now. Trades coverage + (linux-only, unsigned) for immediacy and fork-friendliness. To get + *signed* Windows binaries for a branch instead, dispatch + `create-release` with `publish-release=false` + + `smoke-artifacts-only=true`, then dispatch this workflow with + `source=nightly` and that run's id (see D7). +- **Dispatch `release`** after publishing, as post-publish verification — + e.g. the optional step in + `dev-docs/checklist-make-a-new-quarto-prerelease.md`. Curative by + nature: it can only detect a broken published version, never prevent + one. Only works for releases cut after the harness support merged (D10). + ## Design decisions Each entry: what was decided, why, and what would justify revisiting. diff --git a/tests/README.md b/tests/README.md index de03ce9402..734c61dbe2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -665,4 +665,12 @@ Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tes - If it was triggerred by `workflow_call`, then it will run each test in using `run-tests.[sh|ps1]` in a for-loop. - Scheduled tests are still run daily in their sequential version. - It is parameterized (`quarto-install`, `quarto-version`, `quarto-artifact-name`, `ref`, `runners`, ...) so callers can run the suite against a built quarto instead of the dev source tree: the workflow installs the quarto under test outside the checkout and exports `QUARTO_TEST_BIN` (see "Binary mode" above). -- `test-smokes-built.yml` runs the smoke tests against a **built** quarto (daily, via `workflow_run` after each nightly `create-release.yml` build, plus `workflow_dispatch`) by calling `test-smokes.yml`. Three modes: `build` (build a linux-amd64 dist from the checkout, via the shared `.github/actions/build-dist-tarball` composite action also used by `create-release.yml`), `nightly` (reuse the signed artifacts of a completed create-release run — Linux, Windows, and macOS, the only macOS smoke coverage in CI), and `release` (install a published (pre-)release). +- `test-smokes-built.yml` runs the smoke tests against a **built** quarto (daily, via `workflow_run` after each nightly `create-release.yml` build, plus `workflow_dispatch`) by calling `test-smokes.yml`. Which mode to trigger when: + + | Mode | Trigger | Use it to answer | + |---|---|---| + | `nightly` | automatic (after each nightly build); dispatchable with a `run-id` | does what we ship work? Signed artifacts from a create-release run — Linux, Windows (`quarto.exe`), macOS (the only macOS smoke coverage in CI) | + | `build` | dispatch (default) | will *this ref* survive packaging? Builds a linux-amd64 dist from the checkout via the shared `.github/actions/build-dist-tarball` action (also used by `create-release.yml`); the only mode that works on forks/PR branches | + | `release` | dispatch | is the *published* (pre-)release healthy? Post-publish verification; only works for releases whose tag contains the binary-mode harness | + + Full rationale and design decisions: `llm-docs/built-version-testing-architecture.md`. From e2633105d955151642a5ffbfe3d40295a046233e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 14:43:40 +0000 Subject: [PATCH 27/43] ci: harden built-version workflows against review findings Focused review of every if:/expression across test-smokes-built.yml, test-smokes.yml, and create-release.yml (three independent truth-table reviews). All mode-resolution expressions, needs-chain skip propagation, input defaults, and reusable-workflow contexts verified correct. Three real issues found and fixed: 1. workflow_run fires on EVERY completed create-release run, not only nightly schedules - a smoke-artifacts-only branch build (no Mac Zip) would fail the mac leg red at artifact download. resolve-nightly now probes the run's artifacts and each OS leg gates on its artifact existing. Side benefit: a smoke-artifacts-only dispatch now gets its linux+windows smokes automatically - one dispatch instead of two. 2. publish-release defaults to TRUE, so dispatching create-release with only smoke-artifacts-only checked reached the version commit+tag step and pushed an orphan tag while the publish job was guarded off. configure now fails fast on the publish+smoke combination. 3. Such runs also joined the shared 'building-releases-prerelease' concurrency group; the group expression now sends them to per-run groups. Header comments and docs (README trigger table, architecture doc D1/D7) updated to describe the actual trigger scope and the one-dispatch flow. Co-Authored-By: Claude Fable 5 --- .github/workflows/create-release.yml | 14 ++++- .github/workflows/test-smokes-built.yml | 58 ++++++++++++++----- .../built-version-testing-architecture.md | 25 +++++--- tests/README.md | 2 +- 4 files changed, 77 insertions(+), 22 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index ff242d4d5e..f4dfddeb7d 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -25,7 +25,9 @@ env: concurrency: # make publishing release concurrent (but others trigger not) - group: building-releases-${{ inputs.publish-release && 'prerelease' || github.run_id }} + # smoke-artifacts-only runs never publish, so they must not queue in the + # shared 'prerelease' group behind/ahead of a real release + group: building-releases-${{ inputs.publish-release && !inputs.smoke-artifacts-only && 'prerelease' || github.run_id }} jobs: configure: @@ -40,6 +42,16 @@ jobs: tag_pushed: ${{ steps.version_commit.outputs.tag_pushed }} if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'quarto-dev/quarto-cli') steps: + # publish-release defaults to TRUE, so checking only the + # smoke-artifacts-only box would otherwise reach the version + # commit+tag step below and push an orphan tag (the publish-release + # job itself is guarded, but the tag push lives in this job) + - name: Fail on publish-release + smoke-artifacts-only + if: ${{ inputs.publish-release && inputs.smoke-artifacts-only }} + run: | + echo "::error::smoke-artifacts-only builds cannot publish - re-dispatch with publish-release=false" + exit 1 + - uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 4873ed9900..2a5b2f2cc7 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -6,10 +6,14 @@ # the linux 'Deb Zip' tarball, the signed 'Windows Zip' (the real # quarto.exe), AND the signed+notarized 'Mac Zip' - and run the smokes at # that run's commit. No duplicate build, three-OS coverage for free -# (the ONLY macOS smoke coverage in CI). To test a -# signed build of a specific commit on demand: dispatch create-release with -# publish-release=false, then dispatch this workflow with source=nightly and -# that run's id. +# (the ONLY macOS smoke coverage in CI). Each OS leg is gated on its +# artifact existing in the run, so a smoke-artifacts-only build (no Mac +# Zip) gets linux+windows tested instead of a red mac job. To test a +# signed build of a specific commit on demand: dispatch create-release +# with publish-release=false (+ smoke-artifacts-only for a cheaper +# linux+windows build) - the workflow_run trigger then tests it +# automatically; dispatching this workflow with source=nightly and a +# run-id is only needed to re-test an older run. # - build (dispatch default): build a linux-amd64 dist from THIS ref with the # same recipe as create-release (shared build-dist-tarball action) and test # it. The only mode that works for arbitrary refs/PR branches and on forks @@ -23,10 +27,12 @@ # dispatch => the source input (default build). name: Smoke Tests (Built Version) on: - # Fires once per completed nightly create-release build (main only: - # workflow_run triggers require the workflow file on the default branch). - # This makes built-version testing DAILY, right after each build, with no - # stale-artifact risk (vs a cron that resolves "latest successful run"). + # Fires on EVERY completed create-release run - the nightly schedule + # (daily built-version testing, right after each build, no stale-artifact + # risk) but also manual create-release dispatches, whose builds get tested + # too (partial smoke-artifacts-only builds only run the legs whose + # artifacts exist). The workflow file must be on the default branch for + # workflow_run to fire; the triggering build may be on any branch. workflow_run: workflows: ["Build Installers"] types: [completed] @@ -180,8 +186,11 @@ jobs: outputs: run-id: ${{ steps.r.outputs.run-id }} sha: ${{ steps.r.outputs.sha }} + has-linux: ${{ steps.r.outputs.has-linux }} + has-windows: ${{ steps.r.outputs.has-windows }} + has-mac: ${{ steps.r.outputs.has-mac }} steps: - - name: Resolve run id and commit + - name: Resolve run id, commit and artifacts id: r shell: bash env: @@ -202,13 +211,32 @@ jobs: echo "::error::could not resolve head_sha for run ${run_id}" exit 1 fi - echo "Using create-release run ${run_id} at commit ${sha}" + # A smoke-artifacts-only build has no 'Mac Zip' (and workflow_run + # fires on EVERY completed create-release run, not only nightly + # schedules) - gate each OS leg on its artifact actually existing + # instead of failing red at download time. + names="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts" \ + --paginate --jq '.artifacts[].name')" + has() { grep -Fxq "$1" <<<"$names" && echo true || echo false; } + has_linux="$(has 'Deb Zip')" + has_windows="$(has 'Windows Zip')" + has_mac="$(has 'Mac Zip')" + if [ "$has_linux" = false ] && [ "$has_windows" = false ] && [ "$has_mac" = false ]; then + echo "::error::run ${run_id} produced none of the expected artifacts (Deb Zip / Windows Zip / Mac Zip)" + exit 1 + fi + echo "Using create-release run ${run_id} at commit ${sha} (linux=${has_linux} windows=${has_windows} mac=${has_mac})" echo "run-id=$run_id" >> "$GITHUB_OUTPUT" echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "has-linux=$has_linux" >> "$GITHUB_OUTPUT" + echo "has-windows=$has_windows" >> "$GITHUB_OUTPUT" + echo "has-mac=$has_mac" >> "$GITHUB_OUTPUT" run-smokes-nightly-linux: name: Smoke tests against nightly build (linux) - if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' needs: [resolve-nightly] uses: ./.github/workflows/test-smokes.yml with: @@ -222,7 +250,9 @@ jobs: run-smokes-nightly-windows: name: Smoke tests against nightly build (windows) - if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-windows == 'true' needs: [resolve-nightly] uses: ./.github/workflows/test-smokes.yml with: @@ -236,7 +266,9 @@ jobs: run-smokes-nightly-mac: name: Smoke tests against nightly build (macOS) - if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-mac == 'true' needs: [resolve-nightly] uses: ./.github/workflows/test-smokes.yml with: diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md index aff52b4150..091b403b4a 100644 --- a/llm-docs/built-version-testing-architecture.md +++ b/llm-docs/built-version-testing-architecture.md @@ -73,8 +73,10 @@ In practice: (linux-only, unsigned) for immediacy and fork-friendliness. To get *signed* Windows binaries for a branch instead, dispatch `create-release` with `publish-release=false` + - `smoke-artifacts-only=true`, then dispatch this workflow with - `source=nightly` and that run's id (see D7). + `smoke-artifacts-only=true` — the `workflow_run` trigger then tests the + build automatically (only the legs whose artifacts exist); a manual + `source=nightly` dispatch with a `run-id` is just for re-testing an + older run (see D7). - **Dispatch `release`** after publishing, as post-publish verification — e.g. the optional step in `dev-docs/checklist-make-a-new-quarto-prerelease.md`. Curative by @@ -123,6 +125,10 @@ pipeline broke; "Smoke Tests (Built Version)" red = the product broke. *display name* string (`workflows: ["Build Installers"]`; renaming create-release's `name:` silently stops the trigger), and "trigger never fired" is silent (mitigated by daily cadence — an absent run is visible). +Note the trigger fires on EVERY completed create-release run, not only +nightly schedules — manual dispatches (including partial +`smoke-artifacts-only` builds) get tested too, which is why each nightly +OS leg is gated on its artifact actually existing in the resolved run. **Revisit when:** the system has a green track record and maintainers want one atomic nightly build-and-test signal — then the inversion is the @@ -179,11 +185,16 @@ are skipped and nothing changes. The `create-release.yml` input skips source/arm64 tarballs and the Mac installer, yielding a fast signed Linux+Windows build for on-demand testing -of a branch (`dispatch create-release publish-release=false -smoke-artifacts-only=true`, then `dispatch test-smokes-built source=nightly -run-id=`). It deliberately does NOT feed the daily path: the -daily needs the full build (Mac Zip = the only macOS smoke coverage), and -`publish-release` is hard-guarded against partial builds. +of a branch: dispatch create-release with `publish-release=false` + +`smoke-artifacts-only=true` and the `workflow_run` trigger tests the build +automatically (mac leg skipped via the artifact-existence gate — one +dispatch total). Guards: `configure` fails fast if `publish-release` (which +defaults to true) is combined with `smoke-artifacts-only` — otherwise the +version commit+tag step would push an orphan tag — and such runs use a +per-run concurrency group so they never queue in the shared `prerelease` +group against a real release. It deliberately does NOT feed the daily +path: the daily needs the full build (Mac Zip = the only macOS smoke +coverage). ### D8. macOS runners: scheduled/built runs only, never per-commit diff --git a/tests/README.md b/tests/README.md index 734c61dbe2..d68dfdb1f0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -669,7 +669,7 @@ Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tes | Mode | Trigger | Use it to answer | |---|---|---| - | `nightly` | automatic (after each nightly build); dispatchable with a `run-id` | does what we ship work? Signed artifacts from a create-release run — Linux, Windows (`quarto.exe`), macOS (the only macOS smoke coverage in CI) | + | `nightly` | automatic (after each completed create-release run, scheduled or dispatched); dispatchable with a `run-id` to re-test an older run | does what we ship work? Signed artifacts from a create-release run — Linux, Windows (`quarto.exe`), macOS (the only macOS smoke coverage in CI); each OS leg runs only if its artifact exists in the run | | `build` | dispatch (default) | will *this ref* survive packaging? Builds a linux-amd64 dist from the checkout via the shared `.github/actions/build-dist-tarball` action (also used by `create-release.yml`); the only mode that works on forks/PR branches | | `release` | dispatch | is the *published* (pre-)release healthy? Post-publish verification; only works for releases whose tag contains the binary-mode harness | From c881e92cb07c8793f4212f2b219c0a7aab2e1560 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 14:55:52 +0000 Subject: [PATCH 28/43] tests: sanitize env for QUARTO_TEST_BIN --version probes The binary-mode guards in run-tests.[sh|ps1] and assertTestBinary() probed the built quarto with the harness env intact. The installed launcher keeps an inherited QUARTO_SHARE_PATH (exported by the runner for the harness), and its --version fast path is 'cat $QUARTO_SHARE_PATH/version' - pointing at the dev tree, where src/resources/version does not exist. A healthy built quarto therefore reported an EMPTY version (not the 99.9.9 sentinel a review suggested - that file doesn't exist in the dev tree): cosmetically wrong banners, and a false-red QUARTO_TEST_EXPECTED_VERSION mismatch in release-mode CI. Probe with the same strip-list as test spawns (env -u chain in bash, save/clear/restore in pwsh, buildBinaryEnv()+clearEnv in the harness) and fail loudly on an empty version (incomplete distribution). Co-Authored-By: Claude Fable 5 --- tests/quarto-cmd.ts | 12 ++++++++++++ tests/run-tests.ps1 | 29 ++++++++++++++++++++++++++++- tests/run-tests.sh | 14 +++++++++++++- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index 9e9c005212..cf1cd6a597 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -151,10 +151,16 @@ export function assertTestBinary(bin: string) { if (checkedBinary === bin) { return; } + // Probe with the sanitized env used for test spawns: the installed + // launcher keeps an inherited QUARTO_SHARE_PATH (exported by + // run-tests.[sh|ps1] for the harness), under which --version reads the + // dev tree's nonexistent src/resources/version and reports empty. const result = new Deno.Command(bin, { args: ["--version"], stdout: "piped", stderr: "piped", + env: buildBinaryEnv(), + clearEnv: true, }).outputSync(); const version = new TextDecoder().decode(result.stdout).trim(); if (result.code !== 0) { @@ -163,6 +169,12 @@ export function assertTestBinary(bin: string) { `QUARTO_TEST_BIN (${bin}) failed to report a version (exit ${result.code}):\n${stderr}`, ); } + if (version.length === 0) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reported an empty version. ` + + `The distribution is likely incomplete (missing share/version).`, + ); + } if (version === kLocalDevelopment) { throw new Error( `QUARTO_TEST_BIN (${bin}) reports the dev version sentinel ${kLocalDevelopment}. ` + diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index 58b6bef88d..c334bcdd52 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -77,7 +77,34 @@ If ($null -ne $Env:QUARTO_TEST_BIN) { Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) does not exist" Exit 1 } - $QUARTO_TEST_BIN_VERSION = & $Env:QUARTO_TEST_BIN --version + # Probe with the dev-tree env stripped (same list as tests/quarto-cmd.ts): + # the installed launcher keeps an inherited QUARTO_SHARE_PATH, and the dev + # exports above would make a healthy built quarto read the dev tree's + # (nonexistent) src/resources/version and report an EMPTY version. + $probeStrip = @( + "QUARTO_SHARE_PATH", "QUARTO_BIN_PATH", "QUARTO_DEBUG", "DENO_DIR", + "QUARTO_DENO", "QUARTO_DENO_DOM", "QUARTO_ROOT", "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION" + ) + $probeSaved = @{} + ForEach ($name in $probeStrip) { + $probeSaved[$name] = [Environment]::GetEnvironmentVariable($name) + Remove-Item "Env:$name" -ErrorAction SilentlyContinue + } + Try { + $QUARTO_TEST_BIN_VERSION = & $Env:QUARTO_TEST_BIN --version + } Finally { + ForEach ($name in $probeStrip) { + If ($null -ne $probeSaved[$name]) { + [Environment]::SetEnvironmentVariable($name, $probeSaved[$name]) + } + } + } + If ([string]::IsNullOrWhiteSpace($QUARTO_TEST_BIN_VERSION)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) did not report a version." + Write-Host -ForegroundColor red "The distribution is likely incomplete (missing share/version)." + Exit 1 + } If ($QUARTO_TEST_BIN_VERSION -eq "99.9.9") { Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." Write-Host -ForegroundColor red "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 3744c741da..a6b6510e28 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -73,7 +73,19 @@ if [[ -n "$QUARTO_TEST_BIN" ]]; then echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) does not exist or is not executable" exit 1 fi - QUARTO_TEST_BIN_VERSION="$("$QUARTO_TEST_BIN" --version 2>/dev/null)" + # Probe with the dev-tree env stripped (same list as tests/quarto-cmd.ts): + # the installed launcher keeps an inherited QUARTO_SHARE_PATH, and the dev + # exports above would make a healthy built quarto read the dev tree's + # (nonexistent) src/resources/version and report an EMPTY version. + QUARTO_TEST_BIN_VERSION="$(env -u QUARTO_SHARE_PATH -u QUARTO_BIN_PATH \ + -u QUARTO_DEBUG -u DENO_DIR -u QUARTO_DENO -u QUARTO_DENO_DOM \ + -u QUARTO_ROOT -u QUARTO_SRC_PATH -u QUARTO_FORCE_VERSION \ + "$QUARTO_TEST_BIN" --version 2>/dev/null)" + if [[ -z "$QUARTO_TEST_BIN_VERSION" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) did not report a version." + echo "The distribution is likely incomplete (missing share/version)." + exit 1 + fi if [[ "$QUARTO_TEST_BIN_VERSION" == "99.9.9" ]]; then echo "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." echo "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." From 4a360f17e224224be56a4047419e596e29de25c6 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 17 Jul 2026 17:19:15 +0200 Subject: [PATCH 29/43] docs: drop the historical plan doc, point to the ADR instead The 744-line dev-docs/smoke-tests-built-version-plan.md was the working plan/spec produced while building this. The settled design and its rationale now live in llm-docs/built-version-testing-architecture.md (the decision record); the plan added nothing durable and duplicated the ADR. Repoint the eight in-tree references at the ADR. --- .github/workflows/test-smokes-built.yml | 2 +- dev-docs/debugging-flaky-tests.md | 2 +- dev-docs/smoke-tests-built-version-plan.md | 744 ------------------ .../built-version-testing-architecture.md | 3 - llm-docs/testing-patterns.md | 3 +- tests/README.md | 2 +- tests/quarto-cmd.ts | 2 +- tests/run-tests.ps1 | 2 +- tests/run-tests.sh | 2 +- 9 files changed, 7 insertions(+), 755 deletions(-) delete mode 100644 dev-docs/smoke-tests-built-version-plan.md diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 2a5b2f2cc7..932dc1a289 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -21,7 +21,7 @@ # - release: install a published (pre-)release with quarto-actions/setup and # run the smokes at the matching refs/tags/v. # nightly/release only work when the target commit's harness has binary-mode -# support (tests/quarto-cmd.ts) - see dev-docs/smoke-tests-built-version-plan.md §5.3. +# support (tests/quarto-cmd.ts) - see llm-docs/built-version-testing-architecture.md (D10). # # Mode resolution (repeated in each job's `if`): workflow_run => nightly; # dispatch => the source input (default build). diff --git a/dev-docs/debugging-flaky-tests.md b/dev-docs/debugging-flaky-tests.md index 70816a80d2..2534b0f08b 100644 --- a/dev-docs/debugging-flaky-tests.md +++ b/dev-docs/debugging-flaky-tests.md @@ -186,7 +186,7 @@ for test in test1.ts test2.ts test3.ts; do done # Run against a built quarto instead of the in-process dev sources -# (binary mode; see dev-docs/smoke-tests-built-version-plan.md) +# (binary mode; see llm-docs/built-version-testing-architecture.md) QUARTO_TEST_BIN=/path/to/installed/quarto ./run-tests.sh path/to/test.ts ``` diff --git a/dev-docs/smoke-tests-built-version-plan.md b/dev-docs/smoke-tests-built-version-plan.md deleted file mode 100644 index 8dbcaa411a..0000000000 --- a/dev-docs/smoke-tests-built-version-plan.md +++ /dev/null @@ -1,744 +0,0 @@ -# Plan: Running Smoke Tests Against a Built Quarto - -Status: **detailed design — v3** (grounded + adversarially reviewed; -ready for Phase 0 spike). - -Goal: run the existing smoke test suites — the full smoke-all corpus first, -plus the `.test.ts` smokes and the feature-format matrix — against a -**built** Quarto (an artifact produced in the same CI run, or a published -GitHub (pre)release), instead of only the dev source tree. - -Decisions (maintainer, 2026-07-17): - -- Weekly `schedule` **plus** manual `workflow_dispatch`. -- Scope: **full smoke-all** (ff-matrix bucket optional extra). -- Primary mode is **preventive build-then-test**: build the artifact in the - same workflow run and test it — not only after-the-fact testing of - published prereleases (that stays as a secondary dispatch input). -- No release-pipeline gating until the mode has a track record. - -Review history: v2 was attacked by a 5-lens adversarial review (19 agents; -every critical/major finding independently verification-checked). All 32 -standing findings are folded into this v3; the design-changing ones are -marked **[R]** below. - ---- - -## 1. How testing works today (established facts) - -- `testQuartoCmd` invokes Quarto **in-process**: it imports `quarto()` from - `src/quarto.ts` (`tests/test.ts:13`) and calls it directly - (`tests/test.ts:159-162`). There is no subprocess and no configurable seam. -- Before each test the harness redirects Quarto's logger to a temp file in - `json-stream` format (`tests/test.ts:243-249`); after execution it parses - the file into `ExecuteOutput { msg, level, levelName }` records - (`test.ts:176-180, 386-392`) consumed by every log verifier - (`noErrors`, `noErrorsOrWarnings`, `shouldError`, `printsMessage` — - `tests/verify.ts:107-205`). Errors thrown by `execute()` are caught and - converted into ERROR log records before verification (`test.ts:262-266`). -- All file-content verifiers read produced output files off disk. -- `tests/run-tests.sh` runs the harness under the repo-bundled Deno with - `--importmap=src/import_map.json` and exports dev paths: - `QUARTO_BIN_PATH=package/dist/bin` (`:47`), - `QUARTO_SHARE_PATH=src/resources` (`:60`), `QUARTO_DEBUG=true` (`:61`), - `DENO_DIR=package/dist/bin/deno_cache` (`:50-54`). -- `smoke-all.test.ts` globs `docs/smoke-all/**/*.{md,qmd,ipynb}` - (**1424 documents** as of 2026-07-17, many with multiple format specs), - parses `_quarto.tests` with dev-source YAML code, dispatches spec keys to - verifiers via an inline `verifyMap`, and renders via - `testQuartoCmd("render", [input, "--to", format])` (`:462`); project - pre-renders call `quarto(["render", projectPath])` directly (`:434`). -- The **feature-format matrix is a smoke-all bucket**: - `test-ff-matrix.yml:44-50` calls `test-smokes.yml` with a qmd glob that - `run-tests.sh:132-164` routes to `smoke-all.test.ts`. Anything that fixes - smoke-all fixes the matrix. -- `create-release.yml` builds per-OS artifacts and uploads them as workflow - artifacts. **[R]** The nightly `schedule` runs are build-only (the - tag-creating commit and the whole `publish-release` job are gated on - `inputs.publish-release`, and `inputs` is empty on schedule events); - published + `v`-tagged (pre)releases come from - `workflow_dispatch` runs, where `publish-release` defaults to true - (`create-release.yml:88-100,678-703`). The linux tarball recipe is - `./configure.sh` → `quarto-bld prepare-dist --set-version ` → tar of - `package/pkg-working` (`create-release.yml:136-159`). - -## 2. Strategy - -1. **The dev-tree coupling is concentrated in the execution path** — the - body of `test.execute()` plus the logger lifecycle around it in - `test()` (two touch points, both in `tests/test.ts`). The verifier - layer and the `_quarto.tests` dispatch consume only the json-stream log - file and files on disk, both reproducible by a subprocess via - `--log --log-format json-stream`. -2. **Version skew is avoided by construction.** The primary workflow builds - the artifact from the same SHA the harness checks out. For - published-release testing, check out `refs/tags/v` — with the - scope limitation in §5.3 **[R]**. The harness keeps importing `src/` - for *parsing and path math* only; *execution* moves to the binary. - -**What this design does NOT test [R]:** signing/notarization, MSI/pkg/deb -installer logic and installer PATH setup, Cloudsmith packages, arm64 -tarballs, and (until Phase 3 release mode) the Windows Rust launcher. A -green "Smoke Tests (Built Version)" badge means the linux-amd64 built -*layout* renders correctly — not that release artifacts are safe -end-to-end. Keep this list next to any badge/gate. - -## 3. Ground truths verified in code (2026-07-17) - -**Log pipeline (CONFIRMED):** - -- `--log`, `--log-level`, `--log-format` are global options appended to - every command (`src/core/log.ts:49-95`; applied via the `cmdHandler` in - `src/quarto.ts:231-234`), and — decisive — the logger is initialized in - `mainRunner` from a **raw-args parse before any command runs** - (`src/core/main.ts:23-27`, `logOptions` at `src/core/log.ts:97-118`). Env - fallbacks `QUARTO_LOG`, `QUARTO_LOG_LEVEL`, `QUARTO_LOG_FORMAT` exist but - explicit flags win. No command bypasses logger init. -- json-stream writes one `JSON.stringify(logRecord)` per line with - `msg`/`level`/`levelName` (`log.ts:262-263`) — byte-compatible with - `readExecuteOutput`. The file handler **flushes after every record** - (`log.ts:267-273`). -- On render failure an ERROR record is written before exit 1: errors - propagate to `mainRunner`'s catch → `logError(e)` (`main.ts:70-73`) → - ERROR record in the file; then `exitWithCleanup(1)` (`main.ts:74-81`). - `CommandError` likewise (`src/quarto.ts:202-208`). -- **[R] But "non-zero exit ⇒ ERROR record in the log" is NOT an - invariant.** It fails for (a) any child failure *before* `mainRunner`'s - logger init — deno startup errors, bundle/import-resolution failures - loading the esbuild `quarto.js`, missing modules (`main.ts:26-27` runs - after the module graph loads; the launcher just execs deno, - `common/quarto:205-209`); (b) `quarto add`/`remove` failures via - `commandFailed()`/`signalCommandFailure` → `exitWithCleanup(1)` with - only INFO output (`src/quarto.ts:199-201`, - `src/command/add/cmd.ts:48-51,59-61`); (c) the `pandoc`/`typst`/`run` - passthroughs, which `Deno.exit(code)` and route child stderr directly - (`src/quarto.ts:100,119,139`). Because the harness pre-creates the log - file, an untouched log parses to `[]` and `noErrors`/ - `noErrorsOrWarnings` pass **vacuously** — and ~23% of smoke-all docs - (326/1424) use the default log-only spec with no file verifier. The - seam therefore MUST synthesize an ERROR record for record-free non-zero - exits (§4.1) — otherwise the exact failure class binary mode exists to - catch (broken bundle, missing share) reports green on those docs. - -**`QUARTO_FORCE_VERSION=99.9.9` is rejected (do not use):** honored at -`src/core/quarto.ts:34-45` but `99.9.9` equals `kLocalDevelopment`, which -(a) satisfies every lower-bound version gate (extensions -`src/extension/extension.ts:776-788`, document `quarto-required` -`src/command/render/render-files.ts:130-144`, engines -`src/execute/engine.ts:62-80`, freezer `src/core/cache/cache.ts:81`) — -masking real "version too old" behavior — and (b) flips `quarto check` -into a dev-mode branch that shells `git rev-parse` in `$QUARTO_ROOT` -(`src/command/check/check.ts:288-302`). The binary must report its real -version; the fixtures get fixed instead (§4.6). - -**The `>=99.9.0` fixtures are scaffolding artifacts (git archaeology):** -`quarto create extension` computes `quarto-required` from the *running* -version truncated to `major.minor.0` -(`src/command/create/artifacts/artifact-shared.ts:143`); on a dev build -(`99.9.9`) that yields exactly `>=99.9.0`. All ten fixtures were authored -on dev checkouts (four separate lipsum vendorings 2023; brand/typst -fixtures 2025–2026). No test anywhere exercises the version-gate error -path; sibling extensions in the same `_extensions/` dirs use `>=1.3.0`, -as does upstream `quarto-ext/lipsum` and Quarto's own bundled copy -(`src/resources/extensions/quarto/lipsum/_extension.yml:4`). **All ten -are safe to relax.** - -**[R] Semver behavior of version markers (verified by executing the -vendored `deno.land/x/semver@v1.4.0`):** `satisfies()` **throws** on -non-semver strings (no try/catch around `new SemVer` in `Range.test`, -unlike node-semver), and **prerelease versions do not satisfy plain -ranges** (`1.10.16-test` fails `>=1.9`; no call site passes -`includePrerelease`). Only **build metadata** (`1.10.15+test.20260717`) -is both valid and range-transparent. This dictates the §5.2 version -marker. - -**Launcher & env behavior:** - -- The installed bash launcher inherits `QUARTO_SHARE_PATH` and - `QUARTO_DEBUG` if already set (`package/scripts/common/quarto:102-110, - 68-70`; `quarto.cmd:80-81, 47`), recomputes `QUARTO_ROOT`/ - `QUARTO_BIN_PATH` unconditionally, **never sets `DENO_DIR` outside dev - mode**, and inherits `QUARTO_DENO` and `QUARTO_DENO_DOM` - (`common/quarto:167-173`) — note it is `QUARTO_DENO_DOM` that is - inherited; `DENO_DOM_PLUGIN` itself is unconditionally overwritten by - both launchers **[R]**. -- **Dev-mode trap:** the bash/cmd launchers decide dev vs installed by - finding a sibling `src/quarto.ts` relative to their own path - (`common/quarto:22-37`). `QUARTO_TEST_BIN` pointing at the in-repo - `package/dist/bin/quarto` silently runs **dev mode** (TS sources, - `--check`, dev env defaults). The dist must be extracted **outside the - git checkout**, and the seam must fail loudly if ` --version` - reports `99.9.9`. -- **Windows built layout ships two entry points.** `prepare-dist` copies - `quarto.cmd` into the dist (`copyQuartoScript`, - `package/src/common/configure.ts:143-149`); the `make-installer-win` - job additionally builds and signs the **Rust launcher `quarto.exe`** - (`package/launcher/src/main.rs`; `create-release.yml:350-429`) — what - the MSI/zip put on PATH, i.e. what Windows users actually run. - `QUARTO_TEST_BIN` should target `quarto.exe` for published-release - testing; a `prepare-dist`-only Windows artifact has `quarto.cmd`, which - spawns **directly** with `Deno.Command` — no `cmd /c` (established - pattern: `quartoDevCmd()` `tests/utils.ts:244-246`). The Rust launcher - has **no dev-mode branch**, reads `--version` straight from - `share/version`, and inherits `QUARTO_SHARE_PATH`, `QUARTO_DENO`, and - `QUARTO_DENO_DOM` from the environment (`main.rs:21-23,50-60,65-68`). -- A genuinely installed binary runs a single esbuild-bundled `quarto.js` - with `--no-check` (`common/quarto:91-121, 205-208`), a real - `share/version` file, inlined Lua filters, and arch-specific - deno/deno_dom (`package/src/common/prepare-dist.ts:128-147, 191-224`). - Binary mode covers bundling/import-resolution errors, missing share - resources, version wiring, and the `--no-check` gap. -- `QUARTO_DEBUG=true` (dev default) adds stack traces to ERROR `msg` text - (`src/core/log.ts:371-386`) plus dev-only reconfigure/watch paths. - Binary mode spawns the child **without** it. ERROR counts are - unaffected; no smoke-all `printsMessage` depends on stack text. - -**CI structure:** - -- `run-tests.sh` hardcodes the harness runtime at - `package/dist/bin/tools//deno` + `package/dist/bin/deno_cache` + - `src/import_map.json` (`run-tests.sh:47,57,164`), all produced only by - `configure.sh`. **Consequence: the `quarto-dev` action must run in - every CI mode** — it provisions the harness runtime; the built binary - is added *on top* (PATH override + `QUARTO_TEST_BIN`), not substituted. - **[R]** Nuance: this "never uses PATH quarto" claim is CI-scoped — - local runs source `configure-test-env.sh` (which calls PATH `quarto - install tinytex/verapdf`) unless `QUARTO_TESTS_NO_CONFIG` is set; the - Phase 0/1 local recipe must set it or order PATH deliberately. -- The dev quarto reaches PATH via a `/usr/local/bin` symlink on - Linux/macOS (`package/src/common/configure.ts:81-133`, - `suggestUserBinPaths`) and via a `GITHUB_PATH` append of - `package/dist/bin` on Windows CI **[R]**; in both cases a later - `$GITHUB_PATH` prepend shadows it for subsequent steps, so - `quarto install tinytex/verapdf/chrome-headless-shell` - (`test-smokes.yml:236,243,251`) and PATH-based tests automatically use - the binary under test. `merge-extension-tests` is a plain `cp -r` — - mode independent. -- **[R]** `prepare-dist` populates `package/pkg-working` (disjoint from - `package/dist`; `package/src/common/config.ts:65-89`) **but also - regenerates artifacts inside `src/`** (quarto-preview JS via - `build.ts --force`; schema/yaml-intelligence assets under - `src/resources` via `buildAssets()`), so a build job and a test run - sharing one checkout can race on the harness's own - `QUARTO_SHARE_PATH=src/resources`. Separate build/test jobs (the - chosen design) are the mitigation, not merely a preference. - `configure.sh` + `prepare-dist` need network (public URLs) but **no - secrets** for the linux tarball. - -## 4. Design — harness "binary mode" - -Activated by `QUARTO_TEST_BIN=` (on Windows prefer `quarto.exe` — what releases ship, §3). -Unset ⇒ current behavior, byte-for-byte unchanged. - -### 4.1 The execution seam (`tests/quarto-cmd.ts` + `tests/test.ts`) - -```ts -export function binaryMode(): string | undefined; - -export async function runQuarto( - args: string[], - options?: { - env?: Record; - cwd?: string; - logFile?: string; // json-stream target (binary mode) - timeoutMs?: number; - throwOnFailure?: boolean; // [R] default TRUE for direct call sites - logLevel?: string; // [R] honor per-test log intent - }, -): Promise -``` - -- **Dev branch** (no `QUARTO_TEST_BIN`): call in-process `quarto(args, - undefined, options?.env)` exactly as today. -- **Binary branch**: spawn the binary with - `--log --log-format json-stream --log-level ` - appended — but **only** for seam-driven invocations; never for tests - that spawn quarto themselves and own their flags - (`logging/log-level-and-formats.test.ts`). **[R]** If a test overlays - `QUARTO_LOG_LEVEL` via `context.env` or passes `logConfig`, the seam - uses that level/format for the flags instead of silently overriding - (flags beat env in `logOptions`, so passing the env var through is not - enough). -- **[R] Failure semantics — the silent-green invariant.** The seam never - relies on "child exited non-zero ⇒ ERROR record exists" (§3). After the - child exits: parse the log; if exit ≠ 0 **and** the parsed records - contain no ERROR-level entry, append a synthetic ERROR record (exit - code + stderr tail) to the log before verification. With that - guarantee: - - `test.execute()` uses `throwOnFailure: false` (mirrors today's - catch-and-log, `test.ts:262-266`); verifiers see the synthetic record. - - **Direct call sites keep throw semantics** (`throwOnFailure: true`, - the default): the smoke-all module-level project pre-render and the - `context.setup` pre-renders (`render-freeze`, extension installs, …) - run *outside* the test try/catch today and fail loudly on error — - a never-throwing helper would convert those into false greens against - stale outputs. -- **[R] stdout/stderr**: pipe both and drain concurrently (undrained - pipes deadlock at 64 KiB on verbose renders — the child runs without - `--quiet`, so its StdErr handler emits all progress). Keep the stderr - tail for the synthetic ERROR record and for failure reports. Do not - pass `--quiet` (shipped console behavior stays exercised and available - for triage). -- **[R] Timeout kills the process tree, not the direct child.** - `QUARTO_TEST_BIN` is a launcher; the bash script and `quarto.exe` both - spawn-and-wait on deno (`common/quarto:205-209` does not `exec`), so - `child.kill()` orphans the actual renderer — which also still holds - the log file at its own offset (mode `"w"`, no `O_APPEND`), corrupting - any harness-appended record. Implementation: POSIX — spawn in its own - process group and signal the group (or, better long-term, change the - installed launcher's non-dev branch to `exec "${QUARTO_DENO}" …`, a - one-line shipped improvement); Windows — `taskkill /PID /T /F`. - Append the timeout ERROR record only after the tree is confirmed dead. -- **[R] Logger lifecycle in `test()`**: in binary mode, `test()` skips - `initializeLogger`/`cleanupLogger`/`flushLoggers` entirely (the child - owns the log file), and the execute-catch appends a synthetic ERROR - record to the file instead of calling `logError` (which, with no - initialized logger, would go to the console handler and — worse — - `cleanupLogger()` would permanently destroy the default handlers for - subsequent tests in the same file). -- `testQuartoCmd`'s `cwd` context keeps chdir-ing the harness process - (relative-path verifiers and teardowns depend on it) *and* passes cwd - to the subprocess. -- Startup guard, once per run: execute ` --version`; **fail hard** - on `99.9.9` (dev-mode trap) or mismatch with - `QUARTO_TEST_EXPECTED_VERSION` when provided; print the banner. - -### 4.2 Env contract for the spawned binary - -**[R] Mechanism: inherit ambient env + strip, not `clearEnv` + -allowlist.** The v2 `clearEnv` design required enumerating every var the -toolchains need; the Windows system-var surface alone (`SystemRoot`, -`ComSpec`, `PATHEXT`, `APPDATA`, `LOCALAPPDATA`, `ProgramData`, -`PROGRAMFILES`, `windir`, …) is large, failure modes are obscure -(CPython misbehaves without `SystemRoot`; TinyTeX resolves from -`APPDATA` — `tinyTexInstallDir()`; `quartoDataDir()` is -`LOCALAPPDATA`-based for chrome-headless-shell/verapdf), and a -linux-only Phase 0 cannot validate it. The dangerous set is the small, -known one. Contract: - -1. **Inherit** the ambient environment (toolchain vars `QUARTO_R`, - `QUARTO_PYTHON`, `QUARTO_TYPST`, `QUARTO_ESBUILD`, - `QUARTO_DART_SASS`, `QUARTO_CHROMIUM`, `QUARTO_TEXLIVE_BINPATH`, - `QUARTO_TINYTEX_REPOSITORY`, `QUARTO_KNITR_RSCRIPT_ARGS` are ambient - CI env and flow through, as do PATH/HOME/locale/proxy/venv/renv/Julia - and all platform system vars). -2. **Strip** (dev-tree identity + stale-leak guards): - `QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, `QUARTO_DEBUG`, `DENO_DIR`, - `QUARTO_DENO`, `QUARTO_DENO_DOM` **[R]**, `QUARTO_ROOT`, - `QUARTO_SRC_PATH`, `QUARTO_FORCE_VERSION`, - `QUARTO_VERSION_REQUIREMENT`, `QUARTO_PROJECT_DIR`, `QUARTO_PROFILE` - (unless test-provided), `QUARTO_LOG`/`_LEVEL`/`_FORMAT` (seam owns - logging via flags), `RSTUDIO` (must be affirmatively absent for the - "not RStudio" test). -3. **Overlay `context.env` last** (per-test intent wins): - `QUARTO_PROFILE`, `QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES/ - _OUTPUT_FILES`, `QUARTO_PDF_STANDARD`, `RSTUDIO=1`, `LUA_PATH`, etc. - -**[R]** The same strip must apply to the `quartoDevCmd()`/`execProcess` -spawns once they target `QUARTO_TEST_BIN` (§4.5b/c) — the one-line -path switch alone leaves them inheriting `DENO_DIR` etc. Route them -through a shared `spawnQuartoEnv()` helper. - -### 4.3 smoke-all driver changes (`tests/smoke/smoke-all.test.ts`) - -- Route the project pre-render (`:434`; `throwOnFailure: true`) and the - `editor-support-crossref` pseudo-format (2 docs) through `runQuarto`. -- Everything else (discovery, `_quarto.tests` parsing, `verifyMap` - dispatch, cleanup, `run:` skip logic) is harness-side and unchanged. -- The YAML-intelligence bootstrap stays (harness-process-only). - -### 4.4 Runner plumbing (`run-tests.sh` / `run-tests.ps1`) - -When `QUARTO_TEST_BIN` is set (or `--bin ` given): - -- Still resolve the repo Deno + import map — the harness always needs - them. **[R] Keep exporting `QUARTO_SHARE_PATH` (and the dev env - generally) for the harness process in ALL modes** — the harness itself - requires it (`smoke-all.test.ts`'s module-level - `initYamlIntelligenceResourcesFromFilesystem()` → - `quartoConfig.sharePath()` → `getenv()` **throws** when unset, - `src/core/env.ts:9-16`). Child isolation is achieved solely by the - §4.2 strip, applied at spawn time. -- **[R] Default test selection**: `run-tests.sh` cannot take a directory - argument today (the arg-validation loop rejects non-`.ts`/doc paths - and exits 1). In binary mode, set `TESTS_TO_RUN` internally to the - smoke tree, and classify `tests/integration/*.test.ts` explicitly - (include or document as dev-only) rather than losing them silently. - `tests/unit/` is excluded (dev-only by definition). -- Print the ` --version` banner; refuse `99.9.9` (§4.1 guard). -- Local recipe: set `QUARTO_TESTS_NO_CONFIG=true` (or order PATH - deliberately) so `configure-test-env.sh` doesn't invoke a PATH - `quarto` you didn't intend (§3 **[R]**). - -### 4.5 Test adaptations (from the 133-file classification, §8) - -- **Shared-helper migrations** (~14 files importing `src/quarto.ts`): - replace direct `quarto()` calls with `runQuarto` (setup pre-renders - keep throw semantics — §4.1 **[R]**). Trivial `testQuartoCmd` rewrites - where possible (`jupyter/issue-10097`, `issue-12374`). - `engine/invalid-engine-in-project` converts from `assertRejects` - (currently un-awaited — a latent bug) to exit-code + ERROR-log - assertion. -- **`quartoDevCmd()` switch** (`tests/utils.ts:244`): return - `QUARTO_TEST_BIN` when set — migrates `run/*`, `lua-unit`, `logging`, - `create`; consolidate the hardcoded `../package/dist/bin/quarto` - spawns (`filters/editor-support`, `typst-gather`) onto it. **[R]** - Pair the switch with the shared env-strip helper (§4.2) — the path - change alone is insufficient. -- **Semantic one-offs**: `env/check.test.ts` — compute expected version - from the binary instead of hardcoding `99.9.9`; - `inspect-standalone-rstudio` — `RSTUDIO=1` via `context.env` in binary - mode. -- **Anti-pattern fix**: `website/drafts-env.test.ts` converts - `Deno.env.set("QUARTO_PROFILE", ...)` to `context.env`. -- **Move, don't flag**: the 2 dev-only files - (`yaml-intelligence/yaml-intelligence.test.ts`, - `yaml-intelligence-folded-block-strings.test.ts`) move to - `tests/unit/`. `TestContext.requiresDevQuarto` is added as an escape - hatch (sets Deno `ignore` in binary mode), starting with zero users. -- **Guard rails** (lint/CI checks): forbid `import ... from - ".*src/quarto.ts"` under `tests/smoke/`; forbid quarto spawns not - routed through `quartoDevCmd()`/`runQuarto`. - -### 4.6 Fixture fixes - -- Relax the ten `quarto-required: '>=99.9.0'` fixture extensions to - `'>=1.9'` (evidence in §3; matches their sibling extensions). Safe, - standalone, dev-mode no-ops (`99.9.9 >= 1.9`) — can land first. - -## 5. CI design - -### 5.1 Parameterize `test-smokes.yml` - -New `workflow_call` inputs: `quarto-install` (`dev`|`release`|`artifact`, -default `dev`), `quarto-version`, `quarto-artifact-name`, `ref` -(checkout ref), and **[R]** `runners` (JSON list for the OS matrix, -default `'["ubuntu-latest","windows-latest"]'`). **[R]** `buckets` -changes from `required: true` to `required: false, default: ""` (both -existing callers always pass it — behavior-neutral). - -Additional adjustments **[R]**: - -- The seven Windows setup steps gated - `runner.os != 'Windows' || github.event_name == 'schedule'` - (node/playwright/multiplex) must also fire for full non-dev runs - (`inputs.quarto-install != 'dev'`), or a dispatch-triggered full - Windows release-mode run executes in an environment no full Windows - run has ever used. -- All new steps carry `shell: bash` explicitly (they run on the Windows - leg too). - -Step changes — inserted after the existing `quarto-dev` step -(`test-smokes.yml:230`), which runs **unconditionally in every mode** -(harness runtime, §3): - -```yaml -- uses: ./.github/workflows/actions/quarto-dev # ALWAYS - -- name: Set up release quarto - if: inputs.quarto-install == 'release' - uses: quarto-dev/quarto-actions/setup@v2 - with: { version: "${{ inputs.quarto-version }}" } - -- name: Download built quarto artifact - if: inputs.quarto-install == 'artifact' - uses: actions/download-artifact@v8 # repo standard [R] - with: { name: "${{ inputs.quarto-artifact-name }}" } -- name: Install built quarto outside the checkout - if: inputs.quarto-install == 'artifact' - shell: bash - run: | # outside repo (§3 dev-mode trap) - mkdir -p "$RUNNER_TEMP/quarto-under-test" - tar -xzf built-quarto-*.tar.gz -C "$RUNNER_TEMP/quarto-under-test" --strip-components=1 - echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" - -- name: Pin and verify test target - if: inputs.quarto-install != 'dev' - shell: bash - run: | - v="$(quarto --version)" - [ "$v" != "99.9.9" ] || { echo "dev sentinel detected"; exit 1; } - echo "QUARTO_TEST_BIN=$(command -v quarto)" >> "$GITHUB_ENV" -``` - -`quarto install tinytex/verapdf/…` pick up the PATH override -automatically; language setup is shared. - -### 5.2 New `test-smokes-built.yml` - -```yaml -name: Smoke Tests (Built Version) -on: - workflow_run: # daily: after each nightly build (§5.2b, §6 Phase 5) - workflows: ["Build Installers"] - types: [completed] - workflow_dispatch: - inputs: - source: { type: choice, options: [build, nightly, release], default: build } - version: { type: string, default: "pre-release" } # for source: release - -jobs: - build-artifact: # source: build (default; also the schedule path) - if: > - (github.event_name != 'schedule' || github.repository == 'quarto-dev/quarto-cli') - && (github.event.inputs.source != 'release') # fork guard [R] - runs-on: ubuntu-latest - outputs: { sha: "${{ steps.rec.outputs.sha }}" } - steps: - - uses: actions/checkout@v6 - - id: rec - run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - run: ./configure.sh - - run: | # [R] BUILD METADATA marker — - pushd package/src # semver-valid, range-transparent, - ./quarto-bld prepare-dist \ # still != 99.9.9. NEVER use a - --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" \ - --log-level info # '-prerelease' suffix or a 4th - popd # component (both fail all gates) - - run: | - pushd package - mv pkg-working quarto-built-test - tar czf built-quarto-linux-amd64.tar.gz quarto-built-test - mv quarto-built-test pkg-working - popd - - uses: actions/upload-artifact@v7 - with: { name: built-quarto-linux-amd64, - path: package/built-quarto-linux-amd64.tar.gz } - - run-smokes-artifact: - if: github.event.inputs.source != 'release' - needs: [build-artifact] - uses: ./.github/workflows/test-smokes.yml - with: - ref: "${{ needs.build-artifact.outputs.sha }}" # harness == binary commit - quarto-install: artifact - quarto-artifact-name: built-quarto-linux-amd64 - runners: '["ubuntu-latest"]' # linux only [R] - buckets: "" # full run [R] - - resolve-release: # [R] resolve step needs its - if: github.event.inputs.source == 'release' # own job (uses:-jobs have no steps) - runs-on: ubuntu-latest - outputs: { version: "${{ steps.r.outputs.version }}" } - steps: - - id: r - run: | # input 'pre-release'/'release' -> concrete 1.x.y via _prerelease.json/_download.json - ... - - run-smokes-release: - if: github.event.inputs.source == 'release' - needs: [resolve-release] - uses: ./.github/workflows/test-smokes.yml - with: - ref: "refs/tags/v${{ needs.resolve-release.outputs.version }}" - quarto-install: release - quarto-version: "${{ needs.resolve-release.outputs.version }}" - buckets: "" -``` - -**[R] Version-marker guard**: Phase 0 asserts (with the vendored semver) -`satisfies("", ">=1.3.0") === true` before adopting any marker; -the CI "Pin and verify" step gains the same assertion. - -### 5.2b `source: nightly` — signed artifacts from a no-publish create-release run - -Studied post-review (maintainer suggestion): the signing steps in -`make-installer-win` are **unconditional** — not gated on -`publish-release` — so every create-release run, including the nightly -no-publish schedule, already uploads a **signed** `Windows Zip` (the real -`quarto.exe`) and the linux `Deb Zip` as workflow artifacts. The -implemented `source: nightly` mode — **the `workflow_run` path, firing -daily after each completed nightly build** (a scheduled `build` would -duplicate what create-release built hours earlier, unsigned and -linux-only; `build` stays dispatch-only for arbitrary refs and forks) — -reuses them: resolve a -create-release run (explicit `run-id` input or latest successful), check -out its `head_sha` for the harness (same-commit, no skew), download the -artifacts cross-run (`test-smokes.yml` input `quarto-artifact-run-id` → -`download-artifact` `run-id`/`github-token`), and run the smokes on both -OSes. This is the **preventive Windows coverage** path — signed shipped -binaries, zero extra build/signing infrastructure. The build recipe -itself is also shared now: `.github/actions/build-dist-tarball` is used -by both `create-release.yml` (amd64 + arm64 tarballs) and the `build` -mode, so `build` runs exercise the release recipe by construction. -Constraint: works only for create-release runs whose commit contains the -binary-mode harness (post-merge); the preflight fails clearly otherwise. - -### 5.2c Wiring decision: why `workflow_run` (2026-07, maintainer question) - -Considered and rejected: create-release *dispatching* the test workflow -(hand-rolled `workflow_run` needing `actions: write` + code in the -release workflow); create-release *calling* `test-smokes.yml` via -`workflow_call` (same-run artifacts, but smoke results would redden -`Build Installers` runs — including real publishes — and testing config -would live in the release workflow); and the *inversion* — -`test-smokes-built` owning the schedule and `workflow_call`ing -create-release (feasible: create-release uses the `inputs.` context -throughout, and `github.event_name` propagates from the caller; but -`smoke-artifacts-only` skips `make-installer-mac` so the daily would -need the full build anyway — zero compute saved, no double build exists -today — and it would make the nightly build canary depend on the test -workflow's health). `workflow_run` keeps the release pipeline untouched -and the failure signals separate ("Build Installers" red = pipeline -broke; "Smoke Tests (Built Version)" red = product broke). Accepted -weaknesses: the trigger couples on the workflow *name* string, and a -never-fired trigger is silent (mitigated by daily cadence). Full -decision record: `llm-docs/built-version-testing-architecture.md` (D1). - -### 5.3 Release mode scope **[R]** - -Release mode checks out the tag while the *workflow file and local -composite actions* resolve from the checked-out tree. Consequences: - -- The harness at the tag must already contain the binary-mode plumbing — - **no existing tag has it**. Release mode therefore only works for - releases cut **after Phase 1 merges**. "Backfill any historical - version" is out of scope (would require the Phase-4 harness/binary - decoupling: main-branch harness testing an older binary). -- Local-action interface drift is real (`.github/actions/cache-typst` is - referenced unconditionally and only exists since 2026-05; older tags - fail with "action not found"). -- Add a preflight step that fails with a clear message when the - checked-out ref lacks `QUARTO_TEST_BIN` support in `tests/run-tests.sh`. - -### 5.4 Later (Phase 4+) - -Call the parameterized workflow from `create-release.yml` as an -initially non-blocking gate before `publish-release`; revisit the §2 -non-goals list before presenting it as a release gate. - -## 6. Phased roadmap - -**Phase 0 — spike (≈1–2 days).** Locally: `configure.sh` + -`quarto-bld prepare-dist`, copy `pkg-working` **outside the checkout**, -`QUARTO_TESTS_NO_CONFIG=true`, hack the seam, run ~20–30 representative -smoke-all docs + a few `.test.ts` smokes. Acceptance: failure-class -catalog; log-file contract confirmed end-to-end **including the -synthetic-ERROR path** (kill a child mid-render; corrupt the dist and -confirm red, not green); semver marker assertion passes; **measured -per-spawn overhead** (product cold-start, not just process launch — the -corpus is 1424 docs ⇒ 1500+ spawns; if overhead is high, the -`run-parallel-tests` bucket matrix becomes the Phase 2 default rather -than a contingency). - -**Phase 1 — fixtures + harness binary mode.** Land the ten -`quarto-required` relaxations and the `drafts-env` fix (standalone, -dev-mode no-ops). Implement `runQuarto` (+ `throwOnFailure`, synthetic -ERROR, tree-kill, stderr capture), the env strip helper, `quartoDevCmd()` -switch, runner plumbing (internal `TESTS_TO_RUN`, `integration/` -classification, version guard), logger-lifecycle branching in `test()`, -`requiresDevQuarto`, the ~14 migrations, the 2 file moves. Acceptance: -full dev-mode suite green (default path untouched); binary-mode subset -green locally; the corrupt-dist scenario fails red. - -**Phase 2 — CI.** Parameterize `test-smokes.yml` (§5.1 incl. `runners`, -`buckets` default, Windows-gate fix); add `test-smokes-built.yml` -(dispatch build-then-test, full smoke-all, linux; the scheduled trigger -later became `workflow_run` on the nightly build — §6 Phase 5). -Acceptance: first green (or fully triaged) built-version run; failures -classified into product bugs / harness assumptions / dev-only. - -**Phase 3 — broaden.** `source: release` path for releases cut after -Phase 1 (Windows coverage via `quarto.exe`; §5.3 scope), ff-matrix -bucket job, guard-rail lint checks. Acceptance: dispatch run against the -first post-Phase-1 prerelease succeeds on both OSes. - -**Phase 4 — optional.** Non-blocking gate in `create-release.yml`; -harness/binary decoupling (via `quarto inspect`) if testing binaries -from a different commit than the harness (incl. true backfill) becomes -worth the drift risk. - -**Phase 5 — daily built-version as the primary signal (studied -2026-07-17, maintainer proposal).** Verified: `create-release.yml` can -be dispatched on any branch tip with `publish-release=false` — all -publish/tag/docker/cloudsmith paths are input-gated, secrets are -available on non-default branches (no protected environments), and -`version_commit` resolves empty so jobs build the dispatched ref. The -`smoke-artifacts-only` input makes such branch builds cheap (linux -tarball + signed Windows zip only). Implemented now (additive — Phase -A): `test-smokes-built.yml` triggers via **`workflow_run` after every -nightly create-release build** (once per build, no stale-artifact -risk; a failed build means a visibly skipped day, not a silent stale -test) and gained a **macOS leg** from the nightly `Mac Zip` — the only -macOS smoke coverage in CI. Remaining maintainer decisions, AFTER a -green track record (~2 weeks): (Phase B) give `unit/` + `integration/` -(74 files, currently covered by every PR run) a cheap dedicated daily -dev job so they keep daily env-drift coverage; (Phase C) downgrade the -daily dev `test-smokes.yml` cron to weekly — not delete: a weekly dev -run keeps the dev-only classes covered (QUARTO_DEBUG paths, `quarto -check` dev branch, in-process races, Windows playwright setup) and the -safety net for paths-ignored doc-only PRs. - -## 7. Risks & open items - -- **Per-spawn cost at corpus scale**: 1500+ spawns × product cold-start - (bundle load, schema init) — measured in Phase 0; bucket-matrix - sharding is the fallback. -- **Binary-only flakiness classes**: jupyter daemon behavior across - separate quarto processes, per-process knitr/renv startup, memory/CPU - contention under deno-test parallelism — watch in Phase 2 triage. -- **`shouldError` + passthrough/add/remove commands**: §3's record-free - non-zero exits are handled by the synthetic-ERROR invariant; the - passthrough tests keep their own `execProcess` assertions. -- **`quarto check` dev branch**: with a real version the dev branch is - skipped — `check`-based tests assert against installed behavior (§4.5). -- **Intra- vs inter-process concurrency**: `issues/9133` reproduces an - in-process race; the two-subprocess variant needs a runtime check that - the regression still triggers. -- **`configure.sh` cost in binary-mode CI**: still downloads the dev - toolchain the binary won't use — wasted minutes, correctness-neutral. -- **Skip-list drift**: guard-rail greps + `requiresDevQuarto` convention. -- **Scope honesty**: keep the §2 "does not test" list current. - ---- - -## 8. Appendix — classification sweep (2026-07-17) - -All 133 `tests/smoke/**/*.test.ts` read in full: **107 compatible / 24 -adapt / 2 dev-only**. (`tests/integration/*.test.ts` — 3 files — were -outside this sweep and get classified in Phase 1, §4.4.) - -### 8.1 Compatible as-is (107) - -Everything funneling through `testQuartoCmd` or its wrappers -(`testRender`, `testSite`, `testProjectRender`, `testManuscriptRender`): -`render` (28/31), `crossref`+`site`+`website` (25/26), `project` (8/8), -`inspect` (5/6), `extensions` (5/7), `yaml`, `ojs`, `use`, `verify`, -`jats`, `book`, `shortcodes`, `search`, `scholar`, `manuscript`, `embed`, -`authors`, `build-ts-extension`, `check`, and more. Their `src/` imports -are expectation/path/cleanup helpers only. `TestContext.env` is already a -parameter (not `Deno.env.set`) and forwards cleanly. - -### 8.2 Adapt (24) — four mechanical patterns - -(a) direct `quarto()` import (~14 files; greppable) → `runQuarto` helper / -trivial `testQuartoCmd` rewrites; (b) `quartoDevCmd()` PATH spawns -(`run/*`, `lua-unit`, `logging`, `create`) → env-var switch **plus the -shared env-strip helper (§4.2)**; (c) hardcoded -`../package/dist/bin/quarto` spawns (`filters/editor-support`, -`typst-gather` 7–12) → consolidate onto (b); (d) semantic one-offs -(`env/check` 99.9.9 expectation; `inspect-standalone-rstudio` in-process -hook → `RSTUDIO=1` child env). Note: "uses `unitTest()`" is NOT a -dev-only signal — several files use it as a generic wrapper around -subprocess spawns. - -### 8.3 Dev-only (2) - -`yaml-intelligence/yaml-intelligence.test.ts` and -`yaml-intelligence-folded-block-strings.test.ts` — in-process -yaml-intelligence internals with no CLI surface → move to `tests/unit/`. - -### 8.4 Smoke-all documents - -Binary-clean except the ten `quarto-required: '>=99.9.0'` fixtures -(§4.6): no doc executes quarto from a code cell, no runtime dev-tree -paths, no pre/post-render scripts, every `printsMessage` uses -INFO/WARN/ERROR (never DEBUG). Corpus: 1424 docs; ~326 use the default -log-only spec (the silent-green exposure §3 [R] closes); 1032 of the -1098 spec-bearing docs also have file-content verifiers that fail loudly -when nothing renders. Driver adaptations: project pre-renders (24 docs) -and `editor-support-crossref` (2 docs) through `runQuarto`. One-time -runtime checks: `engine/class-override` (`>=1.9.17`), -`QUARTO_EXECUTE_INFO`/`QUARTO_PROJECT_ROOT` env docs, the pandoc-args -INFO echo docs (`2025/12/09/13775-*`). diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md index 091b403b4a..b5d980c6cc 100644 --- a/llm-docs/built-version-testing-architecture.md +++ b/llm-docs/built-version-testing-architecture.md @@ -27,9 +27,6 @@ Document map: operational "how" (local recipe, authoring rules, four mermaid diagrams). - `llm-docs/testing-patterns.md` → "Dev Mode vs Binary Mode" — authoring patterns for tests that must work in both modes. -- `dev-docs/smoke-tests-built-version-plan.md` — the original plan/spec with - full grounding evidence, roadmap phases, and open items (historical record; - this doc supersedes it as the go-to reference for the settled design). ## Architecture in one paragraph diff --git a/llm-docs/testing-patterns.md b/llm-docs/testing-patterns.md index 72882bdb22..af5c0091c9 100644 --- a/llm-docs/testing-patterns.md +++ b/llm-docs/testing-patterns.md @@ -38,8 +38,7 @@ under test: and default the selection to `smoke/` only (`unit/` and `integration/` are dev-only). Exercised by `.github/workflows/test-smokes-built.yml`. Architecture and design decisions: - `llm-docs/built-version-testing-architecture.md` (original plan: - `dev-docs/smoke-tests-built-version-plan.md`). + `llm-docs/built-version-testing-architecture.md`. Consequences for writing smoke tests: diff --git a/tests/README.md b/tests/README.md index d68dfdb1f0..88c3d9ce19 100644 --- a/tests/README.md +++ b/tests/README.md @@ -432,7 +432,7 @@ Don't do ### Binary mode (`QUARTO_TEST_BIN`) -By default, tests run quarto **in-process** from the dev sources: `runQuarto()` in `tests/quarto-cmd.ts` calls the `quarto()` entry point imported from `src/quarto.ts`. When the `QUARTO_TEST_BIN` environment variable points at an installed quarto, `runQuarto()` instead spawns that binary as a subprocess, with `--log --log-format json-stream` so the log-based verifiers keep working unchanged. This is used to run the smoke tests against a built distribution (see `llm-docs/built-version-testing-architecture.md` for the architecture and design decisions, `dev-docs/smoke-tests-built-version-plan.md` for the original plan, and the `test-smokes-built.yml` CI workflow below). +By default, tests run quarto **in-process** from the dev sources: `runQuarto()` in `tests/quarto-cmd.ts` calls the `quarto()` entry point imported from `src/quarto.ts`. When the `QUARTO_TEST_BIN` environment variable points at an installed quarto, `runQuarto()` instead spawns that binary as a subprocess, with `--log --log-format json-stream` so the log-based verifiers keep working unchanged. This is used to run the smoke tests against a built distribution (see `llm-docs/built-version-testing-architecture.md` for the architecture and design decisions, and the `test-smokes-built.yml` CI workflow below). To run in binary mode locally: diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index cf1cd6a597..e0e5c280b2 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -11,7 +11,7 @@ * subprocess instead, with `--log --log-format json-stream` so the * existing log-record verifiers keep working unchanged. * -* See dev-docs/smoke-tests-built-version-plan.md for the full design. +* See llm-docs/built-version-testing-architecture.md for the design and rationale. * * Copyright (C) 2020-2026 Posit Software, PBC * diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index c334bcdd52..b21fc726dc 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -71,7 +71,7 @@ If ($null -eq $Env:QUARTO_DENO_DIR) { # itself still runs from the dev tree — the dev env vars above stay — and # tests/quarto-cmd.ts dispatches quarto invocations to the binary, stripping # the dev env from the child process. See -# dev-docs/smoke-tests-built-version-plan.md. +# llm-docs/built-version-testing-architecture.md. If ($null -ne $Env:QUARTO_TEST_BIN) { If (-not (Test-Path $Env:QUARTO_TEST_BIN)) { Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) does not exist" diff --git a/tests/run-tests.sh b/tests/run-tests.sh index a6b6510e28..8ae24f82b0 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -67,7 +67,7 @@ QUARTO_DENO_OPTIONS="--config test-conf.json --v8-flags=--enable-experimental-re # itself still runs from the dev tree — the dev env exports above stay — and # tests/quarto-cmd.ts dispatches quarto invocations to the binary, stripping # the dev env from the child process. See -# dev-docs/smoke-tests-built-version-plan.md. +# llm-docs/built-version-testing-architecture.md. if [[ -n "$QUARTO_TEST_BIN" ]]; then if [[ ! -x "$QUARTO_TEST_BIN" ]]; then echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) does not exist or is not executable" From 12f932a7d8067a23127576500034b4f1ae853c56 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 15:12:28 +0000 Subject: [PATCH 30/43] tests: surface swallowed failures in custom-execute tests The harness converts execute() errors into log records and only fails a test through its verifiers. Three migrated custom-execute tests relied on that contract without a log-reading verifier: - convert/issue-12318: assertion lived inside execute() with verify: [] - a convert failure or failed assertion passed SILENTLY green. Moved the assertion into a verifier and prepended noErrors. Also clean up the roundtrip output file the teardown previously leaked. - crossref/syntax: only verifier was the body comparison (ignores log records) - render failures surfaced as an unhelpful ENOENT. Prepended noErrors. - jupyter/cache: not broken (printsMessage reads the log) but noErrors now reports the actual render error instead of 'expected message not found'. New anti-pattern rule: assertions belong in verify, and throwOnFailure:false requires a log-reading verifier. Co-Authored-By: Claude Fable 5 --- .claude/rules/testing/test-anti-patterns.md | 10 +++++++ tests/smoke/convert/issue-12318.test.ts | 32 +++++++++++++++------ tests/smoke/crossref/syntax.test.ts | 8 ++++-- tests/smoke/jupyter/cache.test.ts | 4 ++- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/.claude/rules/testing/test-anti-patterns.md b/.claude/rules/testing/test-anti-patterns.md index a0ccef6a24..88cc67927a 100644 --- a/.claude/rules/testing/test-anti-patterns.md +++ b/.claude/rules/testing/test-anti-patterns.md @@ -6,6 +6,16 @@ paths: # Test Anti-Patterns +## Don't: Rely on `execute()` Throwing to Fail a Test + +The harness catches every error thrown from a `TestDescriptor.execute` (including +`runQuarto` failures and inline `assert`s) and converts it into a log record — +the test then passes unless a **verifier** fails. A custom `execute` with +`verify: []`, or with verifiers that never read the log, is silently green on +failure. Rules: put assertions in `verify`, not in `execute`; and any test using +`runQuarto(..., { throwOnFailure: false })` must include a log-reading verifier +(`noErrors` at minimum) so command failures surface with their actual error. + ## Don't: Import `src/quarto.ts` in Tests A direct `import { quarto } from "src/quarto.ts"` bypasses binary mode (`QUARTO_TEST_BIN`) and always tests the dev sources. Invoke quarto through `testQuartoCmd()`/`runQuarto()` (`tests/quarto-cmd.ts` is the single dispatch point), and resolve subprocess spawns with `quartoDevCmd()` (`tests/utils.ts`). diff --git a/tests/smoke/convert/issue-12318.test.ts b/tests/smoke/convert/issue-12318.test.ts index 812bd90905..4d14adc85e 100644 --- a/tests/smoke/convert/issue-12318.test.ts +++ b/tests/smoke/convert/issue-12318.test.ts @@ -12,37 +12,53 @@ import { } from "../../test.ts"; import { assert } from "testing/asserts"; import { runQuarto } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; (() => { const input = "docs/convert/issue-12318"; + const roundtrip = "issue-12318-2.qmd"; test({ // The name of the test name: "issue-12318", - + // Sets up the test context: { teardown: async () => { if (existsSync(input + '.ipynb')) { Deno.removeSync(input + '.ipynb'); } + if (existsSync(roundtrip)) { + Deno.removeSync(roundtrip); + } } }, - + // Executes the test execute: async (logFile?: string) => { - await runQuarto(["convert", "docs/convert/issue-12318.qmd"], { + await runQuarto(["convert", input + ".qmd"], { logFile, throwOnFailure: false, }); - await runQuarto(["convert", "docs/convert/issue-12318.ipynb", "--output", "issue-12318-2.qmd"], { + await runQuarto(["convert", input + ".ipynb", "--output", roundtrip], { logFile, throwOnFailure: false, }); - const txt = Deno.readTextFileSync("issue-12318-2.qmd"); - assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); }, - - verify: [], + + // The harness catches execute() errors and turns them into log records, + // so assertions must live in verifiers (an empty verify list would pass + // silently on failure) - noErrors surfaces convert failures first. + verify: [ + noErrors, + { + name: "no triple backticks mid-line after roundtrip", + verify: (_outputs: ExecuteOutput[]) => { + const txt = Deno.readTextFileSync(roundtrip); + assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); + return Promise.resolve(); + }, + }, + ], type: "unit" }); })(); diff --git a/tests/smoke/crossref/syntax.test.ts b/tests/smoke/crossref/syntax.test.ts index 968d6dc0f0..09d0e3d63f 100644 --- a/tests/smoke/crossref/syntax.test.ts +++ b/tests/smoke/crossref/syntax.test.ts @@ -4,7 +4,7 @@ * Copyright (C) 2020-2022 Posit Software, PBC */ -import { ensureFileRegexMatches } from "../../verify.ts"; +import { ensureFileRegexMatches, noErrors } from "../../verify.ts"; import { testRender } from "../render/render.ts"; import { crossref } from "./utils.ts"; import { @@ -70,7 +70,9 @@ const testDesc: TestDescriptor = { // FIXME: why is this test flaky now? Ask @dr name: "test html produced by different figure syntax", context, execute: async (logFile?: string) => { - // failures reach the harness as log records, mirroring testQuartoCmd + // render failures land in the log as ERROR records; noErrors below + // surfaces them (otherwise the comparison verify would just hit an + // unhelpful ENOENT on the missing output file) await runQuarto(["render", imgQmd.input], { logFile, throwOnFailure: false, @@ -80,7 +82,7 @@ const testDesc: TestDescriptor = { // FIXME: why is this test flaky now? Ask @dr throwOnFailure: false, }); }, - verify: [verify], + verify: [noErrors, verify], type: "smoke", }; test(testDesc); diff --git a/tests/smoke/jupyter/cache.test.ts b/tests/smoke/jupyter/cache.test.ts index 932279c824..63a347e694 100644 --- a/tests/smoke/jupyter/cache.test.ts +++ b/tests/smoke/jupyter/cache.test.ts @@ -7,7 +7,7 @@ import { dirname, join } from "path"; import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { docs } from "../../utils.ts"; -import { folderExists, printsMessage } from "../../verify.ts"; +import { folderExists, noErrors, printsMessage } from "../../verify.ts"; import { fileLoader } from "../../utils.ts"; import { safeExistsSync, safeRemoveSync } from "../../../src/core/path.ts"; @@ -43,6 +43,7 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder), // this will check only for the second render that should be read from cache printsMessage({ level: "INFO", regex: /Notebook read from cache/}) @@ -88,6 +89,7 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder2), // this will check only for the second render that should be read from cache printsMessage({level: "INFO", regex: /Notebook read from cache/}) From 64f3fe31d057baa47efac493380628fbd70e1d24 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 15:19:57 +0000 Subject: [PATCH 31/43] tests: make editor-support and logging json-stream tests failable Audit follow-up to the custom-execute silent-green class: - filters/editor-support: all four tests had verify: [] with the spawn and assertions inside execute() - every failure mode (quarto missing, non-zero exit, non-JSON output, wrong crossref entries) was swallowed by the harness and the tests could never fail. The work now runs inside verifiers. - logging/log-level-and-formats: the json-stream branch wrapped its level assertions in an empty catch {} - any violation (e.g. DEBUG records leaking into an --log-level error log) was silently ignored. Also the shouldContainLevel check never asserted presence (vacuous on zero matching records) and JSON.parsed each record's plain-text msg (guaranteed to fail - which the empty catch then hid). Rewritten: narrow parse-only try/catch, real absence assertions with the offending message, real presence assertions per expected level. These 11 test registrations were previously unfailable; they may surface genuine regressions on their first live CI run. Co-Authored-By: Claude Fable 5 --- tests/smoke/filters/editor-support.test.ts | 31 ++++++++---- .../logging/log-level-and-formats.test.ts | 47 +++++++++---------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 720bc3deb5..6f23372fb6 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -35,15 +35,23 @@ async function runEditorSupportCrossref(doc: string) { return json; } +// The harness swallows errors thrown from execute() (they become log +// records nothing here reads), so the spawn + assertions must live in a +// verifier for these tests to be able to fail at all. test({ name: "editor-support:crossref:smoke-1", context: {}, - execute: async () => { - const json = await runEditorSupportCrossref(docs("crossrefs/sections.qmd")); - assertEquals(json.entries[0].key, "sec-introduction"); - assertEquals(json.entries[0].caption, "Introduction"); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref output", + verify: async (_outputs) => { + const json = await runEditorSupportCrossref( + docs("crossrefs/sections.qmd"), + ); + assertEquals(json.entries[0].key, "sec-introduction"); + assertEquals(json.entries[0].caption, "Introduction"); + }, + }], type: "smoke", }); @@ -51,10 +59,13 @@ function smokeTestCrossref(name: string, doc: string) { test({ name, context: {}, - execute: async () => { - await runEditorSupportCrossref(doc); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref runs cleanly", + verify: async (_outputs) => { + await runEditorSupportCrossref(doc); + }, + }], type: "smoke", }); } diff --git a/tests/smoke/logging/log-level-and-formats.test.ts b/tests/smoke/logging/log-level-and-formats.test.ts index bbd01d2859..1b7e7b05b7 100644 --- a/tests/smoke/logging/log-level-and-formats.test.ts +++ b/tests/smoke/logging/log-level-and-formats.test.ts @@ -139,35 +139,34 @@ function testLogDirectly(options: { } - // If JSON format is specified, verify the output is valid JSON + // If JSON format is specified, verify the log file parses and the + // record levels match expectations. NOTE: keep the parse try/catch + // NARROW - a previous version wrapped the level assertions in a + // catch-all that silently swallowed every failure. if (logFile && options.format === "json-stream") { assert(existsSync(logFile), "Log file should exist"); - let foundValidJson = false; + let outputs; try { - const outputs = readExecuteOutput(logFile); - foundValidJson = true; - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldNotContainLevel?.includes(out.levelName)).forEach( - (out) => { - assert(false, `JSON output should not contain level ${out.levelName}, but found: ${out.msg}`); - } + outputs = readExecuteOutput(logFile); + } catch { + outputs = undefined; + } + assert(outputs !== undefined, "JSON format should produce valid JSON output"); + const records = outputs!.filter((out) => out.msg !== ""); + const levels = new Set(records.map((out) => out.levelName)); + for (const lvl of options.expectedOutputs?.shouldNotContainLevel ?? []) { + const offending = records.find((out) => out.levelName === lvl); + assert( + offending === undefined, + `JSON log should not contain level ${lvl}, but found: ${offending?.msg}` ); - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldContainLevel?.includes(out.levelName)).forEach( - (out) => { - let json = undefined; - try { - json = JSON.parse(out.msg); - } catch { - assert(false, "Error parsing JSON returned by quarto meta"); - } - assert( - Object.keys(json).length > 0, - "JSON returned by quarto meta seems invalid", - ); - } + } + for (const lvl of options.expectedOutputs?.shouldContainLevel ?? []) { + assert( + levels.has(lvl), + `JSON log should contain at least one ${lvl} record; found levels: ${[...levels].join(", ") || "(none)"}` ); - - } catch (e) {} - assert(foundValidJson, "JSON format should produce valid JSON output"); + } } } finally { // Clean up log file if it exists From f2ce7589da109e0ba8807226784d0aff044cb1b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 15:27:01 +0000 Subject: [PATCH 32/43] tests: close silent-green paths found by testQuartoCmd call-site audit Harness (tests/test.ts): - verifiers now read logTarget (logConfig.log support was desynced: the logger and binary-mode child wrote to logTarget while verification read the temp file - any test using logConfig.log would hand every log verifier an empty array) - a missing log file now FAILS the test instead of silently skipping every verifier - the failure-report path guards its log re-read so a corrupt log cannot clobber the assembled report with a secondary parse error Call sites: - render-freeze: projectOutputExists constructed outputCreated's Verify and DISCARDED it (guaranteed no-op); combined with the purely-negative useFrozen check, a completely failed freeze render passed two tests. Now delegates properly + noErrors on all three tests. - verify/pdf-metadata + pdf-text-position: assertions lived in teardown, which skips cleanup on failure - leaving a stale fixture.pdf that satisfies the next run even if its render fails (self-perpetuating). Assertions moved into verifiers; setup+teardown both remove the pdf. - project-stdout: verify was [] - the test asserted nothing at all. - typst-gather: cached typst/ dirs are gitignored and survive local runs, keeping existence verifiers green against stale caches; each test now cleans its cache in setup, plus noErrors (incl. the negative-only no-packages-staged test). - project-prepost: teardown assertions moved into verifiers (extension and issue-10828 tests), noErrors added. Co-Authored-By: Claude Fable 5 --- tests/smoke/project/project-prepost.test.ts | 33 ++++++++------- tests/smoke/project/project-stdout.test.ts | 4 +- tests/smoke/render/render-freeze.test.ts | 15 +++---- tests/smoke/typst-gather/typst-gather.test.ts | 21 ++++++++-- tests/smoke/verify/pdf-metadata.test.ts | 27 ++++++++---- tests/smoke/verify/pdf-text-position.test.ts | 41 ++++++++++++------- tests/test.ts | 21 ++++++++-- 7 files changed, 112 insertions(+), 50 deletions(-) diff --git a/tests/smoke/project/project-prepost.test.ts b/tests/smoke/project/project-prepost.test.ts index b7895f4a5f..7dd2ba57bb 100644 --- a/tests/smoke/project/project-prepost.test.ts +++ b/tests/smoke/project/project-prepost.test.ts @@ -63,18 +63,19 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/extension")], - [{ - name: "i-exist.txt exists", + // assertions live in verify (teardown assertions skip cleanup on failure); + // noErrors keeps the purely-negative i-exist check from passing vacuously + // on a failed render + [noErrors, { + name: "prepost extension file effects", verify: async () => { - const path = join(docs("project/prepost/extension"), "i-exist.txt"); - verifyNoPath(path); + verifyNoPath(join(docs("project/prepost/extension"), "i-exist.txt")); + verifyPath(join(docs("project/prepost/extension"), "i-was-created.txt")); } }], { teardown: async () => { - const path = join(docs("project/prepost/extension"), "i-was-created.txt"); - verifyPath(path); - safeRemoveIfExists(path); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); const siteDir = join(docs("project/prepost/extension"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); @@ -85,19 +86,23 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/issue-10828")], - [], + // assertions live in verify, not teardown (an empty verify list + // asserted nothing about the render itself) + [noErrors, { + name: "project input/output files written", + verify: async () => { + verifyPath(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + verifyPath(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + } + }], { env: { "QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES": normalizePath(docs("project/prepost/issue-10828/input-files.txt")), "QUARTO_USE_FILE_FOR_PROJECT_OUTPUT_FILES": normalizePath(docs("project/prepost/issue-10828/output-files.txt")) }, teardown: async () => { - const inputPath = normalizePath(docs("project/prepost/issue-10828/input-files.txt")); - const outputPath = normalizePath(docs("project/prepost/issue-10828/output-files.txt")); - verifyPath(inputPath); - safeRemoveIfExists(inputPath); - verifyPath(outputPath); - safeRemoveIfExists(outputPath); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); const siteDir = join(docs("project/prepost/issue-10828"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); diff --git a/tests/smoke/project/project-stdout.test.ts b/tests/smoke/project/project-stdout.test.ts index dfa38d8629..b0f2aec8fe 100644 --- a/tests/smoke/project/project-stdout.test.ts +++ b/tests/smoke/project/project-stdout.test.ts @@ -14,6 +14,7 @@ import { docs } from "../../utils.ts"; import { directoryEmptyButFor, fileExists, + noErrors, verifyYamlFile, } from "../../verify.ts"; @@ -30,7 +31,8 @@ const siteOutDir = join(siteProjDir, outDir); testQuartoCmd( "render", [siteProjDir, "-o", "-"], - [], + // an empty verify list asserted nothing - a failed stdout render passed + [noErrors], { teardown: async () => { if (existsSync(siteOutDir)) { diff --git a/tests/smoke/render/render-freeze.test.ts b/tests/smoke/render/render-freeze.test.ts index 6e4d5affe4..2298d4f41c 100644 --- a/tests/smoke/render/render-freeze.test.ts +++ b/tests/smoke/render/render-freeze.test.ts @@ -12,7 +12,7 @@ import { Metadata } from "../../../src/config/types.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; import { runQuarto } from "../../quarto-cmd.ts"; import { ExecuteOutput, Verify } from "../../test.ts"; -import { outputCreated } from "../../verify.ts"; +import { noErrors, outputCreated } from "../../verify.ts"; import { testRender } from "./render.ts"; const regex = /output file: .*\.knit\.md/m; @@ -67,9 +67,10 @@ const ignoreFrozen = { const projectOutputExists: Verify = { name: "Make sure project output exists", - verify: (_output: ExecuteOutput[]) => { - outputCreated(path, "html"); - return Promise.resolve(); + verify: (output: ExecuteOutput[]) => { + // delegate to outputCreated's verify - a previous version constructed + // the Verify and discarded it, asserting nothing + return outputCreated(path, "html").verify(output); }, }; @@ -130,7 +131,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "clean fzr - auto", ...testContext, @@ -142,7 +143,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, ignoreFrozen], + [noErrors, projectOutputExists, ignoreFrozen], { name: "dirty fzr - auto", setup: async () => { @@ -167,7 +168,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "dirty fzr - freeze", setup: async () => { diff --git a/tests/smoke/typst-gather/typst-gather.test.ts b/tests/smoke/typst-gather/typst-gather.test.ts index e048cb9341..791128eab8 100644 --- a/tests/smoke/typst-gather/typst-gather.test.ts +++ b/tests/smoke/typst-gather/typst-gather.test.ts @@ -5,6 +5,17 @@ import { join } from "../../../src/deno_ral/path.ts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; + +// The cached typst/ dirs are gitignored and SURVIVE across local runs - a +// stale cache would keep the existence verifiers green even if typst-gather +// broke entirely. Each test removes its cache first so verification is +// always against a fresh gather. +const freshCache = (cacheDir: string) => async () => { + if (existsSync(cacheDir)) { + Deno.removeSync(cacheDir, { recursive: true }); + } +}; // Test 1: Auto-detection from _extension.yml const verifyPackagesCreated: Verify = { @@ -40,9 +51,10 @@ const verifyExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyPackagesCreated, verifyExamplePackageCached], + [noErrors, verifyPackagesCreated, verifyExamplePackageCached], { cwd: () => "smoke/typst-gather", + setup: freshCache("_extensions/test-format/typst"), }, "typst-gather caches preview packages from extension templates", ); @@ -80,9 +92,10 @@ const verifyConfigExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyConfigPackagesCreated, verifyConfigExamplePackageCached], + [noErrors, verifyConfigPackagesCreated, verifyConfigExamplePackageCached], { cwd: () => "smoke/typst-gather/with-config", + setup: freshCache("_extensions/config-format/typst"), }, "typst-gather uses rootdir from config file", ); @@ -248,7 +261,9 @@ const verifyNoPackagesStaged: Verify = { testQuartoCmd( "render", [join(noPackagesProjectDir, "index.qmd"), "--to", "typst"], - [verifyNoPackagesStaged], + // noErrors: a render failing before staging would trivially satisfy the + // purely-negative "nothing staged" assertions + [noErrors, verifyNoPackagesStaged], { teardown: async () => { try { diff --git a/tests/smoke/verify/pdf-metadata.test.ts b/tests/smoke/verify/pdf-metadata.test.ts index 31ad2c775c..bf694d8a73 100644 --- a/tests/smoke/verify/pdf-metadata.test.ts +++ b/tests/smoke/verify/pdf-metadata.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfMetadata } from "../../verify-pdf-metadata.ts"; import { assert } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,14 +42,26 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +// Test: Render fixture and run assertions. The assertions live in a +// verifier (not teardown): teardown assertions skip cleanup on failure, +// leaving a stale fixture.pdf that would satisfy the next run even if its +// render failed. setup also removes any stale pdf from a crashed run. +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf metadata assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/smoke/verify/pdf-text-position.test.ts b/tests/smoke/verify/pdf-text-position.test.ts index 2f40fa0e26..0ea195b7f7 100644 --- a/tests/smoke/verify/pdf-text-position.test.ts +++ b/tests/smoke/verify/pdf-text-position.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfTextPositions, PdfTextPositionAssertion } from "../../verify-pdf-text-position.ts"; import { assert, AssertionError } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,21 +42,33 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +// Test: Render fixture and run assertions. The assertions live in a +// verifier (not teardown): teardown assertions skip cleanup on failure, +// leaving a stale fixture.pdf that would satisfy the next run even if its +// render failed. setup also removes any stale pdf from a crashed run. +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf text position assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + await runSemanticTagTests(); + await runPageRoleTests(); + await runEdgeOverrideTests(); + await runDistanceConstraintTests(); + await runDistanceConstraintErrorTests(); + await runPageRoleWithEdgeTests(); + await runCenterEdgeTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - await runSemanticTagTests(); - await runPageRoleTests(); - await runEdgeOverrideTests(); - await runDistanceConstraintTests(); - await runDistanceConstraintErrorTests(); - await runPageRoleWithEdgeTests(); - await runCenterEdgeTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/test.ts b/tests/test.ts index ef2ddbaf09..06b70ef2fd 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -299,9 +299,15 @@ export function test(test: TestDescriptor) { flushLoggers(handlers); } - // Read the output - const testOutput = logOutput(log); - if (testOutput) { + // Read the output. Verifiers must read logTarget - the harness + // logger and the binary-mode child both write there; reading the + // temp file would hand every log verifier an empty array whenever + // logConfig.log is set. And a missing log is a FAILURE - skipping + // verification would pass the test without checking anything. + const testOutput = logOutput(logTarget); + if (testOutput === undefined) { + fail(`test log file is missing: ${logTarget}`); + } else { for (const ver of test.verify) { lastVerify = ver; if (userSession) { @@ -349,7 +355,14 @@ export function test(test: TestDescriptor) { ? colors.brightGreen(verifyFailed) : verifyFailed; - const logMessages = logOutput(log); + // guarded: a corrupt/unparseable log must not clobber the + // assembled failure report with a secondary parse error + let logMessages: ExecuteOutput[] | undefined; + try { + logMessages = logOutput(logTarget); + } catch { + logMessages = undefined; + } // Create distinctive failure marker for easy log navigation // This helps users find the failure when clicking GitHub Actions annotations From 583933ca0bef9455e41e26907547e1656d44979e Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 17 Jul 2026 19:01:13 +0200 Subject: [PATCH 33/43] tests: fix latent bugs the hardening commits surfaced The preceding hardening commits made three tests fail-capable that had been silently green, exposing pre-existing defects (all present on main): - editor-support: runEditorSupportCrossref called writer.releaseLock() before writer.close(), so close() threw "A writable stream is not associated with the writer". close() releases the lock itself - drop the releaseLock() call. - convert/issue-12318: the test converted docs/convert/issue-12318.qmd, which never existed. Add the fixture from GH #12318 (qmd->ipynb->qmd roundtrip repro) so the test exercises the round trip it claims to. - editor-support all.qmd: referenced since #6136 but never created. Add a combined-crossref fixture (sec/fig/tbl/eq/thm). --- tests/docs/convert/issue-12318.qmd | 28 +++++++++++++++++++++ tests/docs/crossrefs/editor-support/all.qmd | 26 +++++++++++++++++++ tests/smoke/filters/editor-support.test.ts | 4 ++- 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 tests/docs/convert/issue-12318.qmd create mode 100644 tests/docs/crossrefs/editor-support/all.qmd diff --git a/tests/docs/convert/issue-12318.qmd b/tests/docs/convert/issue-12318.qmd new file mode 100644 index 0000000000..8859466ef0 --- /dev/null +++ b/tests/docs/convert/issue-12318.qmd @@ -0,0 +1,28 @@ +--- +title: Quarto Crossrefs +format: html +jupyter: python3 +categories: [plot, python, equation] +--- + +## Overview + +See @fig-simple in @sec-plot for a demonstration of a simple plot. + +See @eq-stddev to better understand standard deviation. + +## Plot {#sec-plot} + +```{python} +#| label: fig-simple +#| fig-cap: "Simple Plot" +import matplotlib.pyplot as plt +plt.plot([1,23,2,4]) +plt.show() +``` + +## Equation {#sec-equation} + +$$ +s = \sqrt{\frac{1}{N-1} \sum_{i=1}^N (x_i - \overline{x})^2} +$$ {#eq-stddev} diff --git a/tests/docs/crossrefs/editor-support/all.qmd b/tests/docs/crossrefs/editor-support/all.qmd new file mode 100644 index 0000000000..b027c14e31 --- /dev/null +++ b/tests/docs/crossrefs/editor-support/all.qmd @@ -0,0 +1,26 @@ +--- +title: All crossref types +--- + +## Introduction {#sec-intro} + +See @sec-intro for the overview, @fig-plot for the figure, @tbl-data for the +table, @eq-var for the equation, and @thm-line for the theorem. + +![A simple plot.](plot.png){#fig-plot} + +| Column A | Column B | +|----------|----------| +| 1 | 2 | + +: A small table. {#tbl-data} + +$$ +\sigma^2 = \frac{1}{N} \sum_{i=1}^N (x_i - \mu)^2 +$$ {#eq-var} + +::: {#thm-line} +## Line + +The equation of a straight line is $y = mx + b$. +::: diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 6f23372fb6..407f48c568 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -25,7 +25,9 @@ async function runEditorSupportCrossref(doc: string) { Deno.readTextFileSync(doc), ); await writer.write(buf); - writer.releaseLock(); + // close() flushes and releases the lock; calling releaseLock() first + // detaches the writer and makes close() throw "A writable stream is not + // associated with the writer". await writer.close(); const outputBuf = await child.output(); const status = await child.status; From 2d9999157a3e0c6b0dc51ddec44bb0a142e632e3 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 17 Jul 2026 21:27:56 +0200 Subject: [PATCH 34/43] tests: stop teardown ENOENT from masking render-failure reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The figure-syntax comparison test in crossref/syntax.test.ts runs runQuarto with throwOnFailure:false and relies on the noErrors verifier to surface a failed render. Its teardown removed the output/support paths with raw Deno.removeSync, so when a render fails (outputs never created) the finally-phase teardown threw NotFound and replaced the in-flight noErrors assertion — the surfaced failure became an unhelpful ENOENT instead of the actual render errors, defeating the reason noErrors was added. Switch to safeRemoveSync, which tolerates already-absent paths. Also correct the editor-support stdin comment: writer.close() closes the stream but does not release the writer's lock (releaseLock is separate); the held lock is inert here and leaks no resource (verified under Deno's resource/op sanitizers). --- tests/smoke/crossref/syntax.test.ts | 13 +++++++++---- tests/smoke/filters/editor-support.test.ts | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/smoke/crossref/syntax.test.ts b/tests/smoke/crossref/syntax.test.ts index 09d0e3d63f..8217557d48 100644 --- a/tests/smoke/crossref/syntax.test.ts +++ b/tests/smoke/crossref/syntax.test.ts @@ -16,6 +16,7 @@ import { } from "../../test.ts"; import { assert, fail } from "testing/asserts"; import { runQuarto } from "../../quarto-cmd.ts"; +import { safeRemoveSync } from "../../../src/deno_ral/fs.ts"; const syntaxQmd = crossref("syntax.qmd", "html"); testRender(syntaxQmd.input, "html", false, [ @@ -58,11 +59,15 @@ const verify: Verify = { }; const context: TestContext = { teardown: () => { - Deno.removeSync(imgQmd.output.outputPath); - Deno.removeSync(imgQmd.output.supportPath, { recursive: true }); + // safeRemoveSync tolerates a missing path: when a render fails, noErrors + // fails first and these outputs never exist. Raw Deno.removeSync would then + // throw NotFound from this finally-phase teardown and mask the noErrors + // report with an unhelpful ENOENT. + safeRemoveSync(imgQmd.output.outputPath); + safeRemoveSync(imgQmd.output.supportPath, { recursive: true }); - Deno.removeSync(divQmd.output.outputPath); - Deno.removeSync(divQmd.output.supportPath, { recursive: true }); + safeRemoveSync(divQmd.output.outputPath); + safeRemoveSync(divQmd.output.supportPath, { recursive: true }); return Promise.resolve(); }, }; diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 407f48c568..11be1abe0a 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -25,8 +25,10 @@ async function runEditorSupportCrossref(doc: string) { Deno.readTextFileSync(doc), ); await writer.write(buf); - // close() flushes and releases the lock; calling releaseLock() first - // detaches the writer and makes close() throw "A writable stream is not + // close() flushes the write and closes the stream (sends EOF to the child); + // it does not release the writer's lock, but nothing reuses the stream so + // the held lock is inert and leaks no resource. Do NOT releaseLock() first — + // that detaches the writer and makes close() throw "A writable stream is not // associated with the writer". await writer.close(); const outputBuf = await child.output(); From f8368cf6f222d7a759003c13498c3c4916b6c9c4 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 17 Jul 2026 21:37:49 +0200 Subject: [PATCH 35/43] tests: clean prepost artifacts in setup so verifyPath proves creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit project-prepost.test.ts checks that i-was-created.txt (extension) and input-files.txt/output-files.txt (issue-10828) exist after render, but only removed them in teardown. Those are fixed repo paths, so a run that crashes between creating them and teardown leaves a stale copy — after which a regression that stops the render creating them still passes verifyPath. Add setup cleanup (matching the safeRemoveIfExists the file already uses in teardown) so each verifyPath proves this render wrote the file, not a leftover. Also clears i-exist.txt up front so pre-render's "must not exist" guard starts clean after a prior crashed run. --- tests/smoke/project/project-prepost.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/smoke/project/project-prepost.test.ts b/tests/smoke/project/project-prepost.test.ts index 7dd2ba57bb..370195ca03 100644 --- a/tests/smoke/project/project-prepost.test.ts +++ b/tests/smoke/project/project-prepost.test.ts @@ -74,6 +74,13 @@ testQuartoCmd( } }], { + // remove artifacts a prior crashed run may have left behind, so verifyPath + // proves this render created i-was-created.txt (not a stale copy) and + // pre-render's i-exist.txt guard starts from a clean slate + setup: async () => { + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-exist.txt")); + }, teardown: async () => { safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); const siteDir = join(docs("project/prepost/extension"), "_site"); @@ -100,6 +107,12 @@ testQuartoCmd( "QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES": normalizePath(docs("project/prepost/issue-10828/input-files.txt")), "QUARTO_USE_FILE_FOR_PROJECT_OUTPUT_FILES": normalizePath(docs("project/prepost/issue-10828/output-files.txt")) }, + // remove artifacts a prior crashed run may have left behind, so verifyPath + // proves this render wrote input-files.txt/output-files.txt, not a stale copy + setup: async () => { + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + }, teardown: async () => { safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); From c1c6d21da1e1042b9a3d4204df1c8c49f2474b84 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 17 Jul 2026 22:10:51 +0200 Subject: [PATCH 36/43] tests: split runQuarto by mode and add withCwd for chdir safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanups surfaced while reviewing the binary-mode test harness. runQuarto had grown into one 125-line function interleaving the in-process dev path with the spawn/timeout/log-merge binary path. Split into runDevQuarto, runBinaryQuarto, and a mergeChildLog helper so each mode's mechanics read on their own; behavior is unchanged. Three extension tests chdir'd into a working dir inside setup() and restored cwd only on the success path. When "use/update template" throws (the network step), cwd leaked into later tests, which then ran in — and whose teardown deleted — a directory they never meant to be in. withCwd() restores cwd in a finally, matching the existing withTempDir idiom. Also corrected a run-tests.sh comment that claimed its --version env strip list matched quarto-cmd.ts's kStripEnvVars; it is deliberately the subset that affects version resolution. --- tests/quarto-cmd.ts | 152 +++++++++++------- tests/run-tests.sh | 11 +- .../extension-render-journals.test.ts | 18 +-- .../extension-render-typst-templates.test.ts | 18 +-- .../render/render-format-extension.test.ts | 25 ++- tests/utils.ts | 16 ++ 6 files changed, 150 insertions(+), 90 deletions(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index e0e5c280b2..0caecd236d 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -52,6 +52,9 @@ const kStripEnvVars = [ // (level 40 == std/log LogLevels.ERROR) const kErrorLevel = 40; +// Default per-invocation render timeout (dev and binary mode alike). +const kDefaultRenderTimeoutMs = 600000; + // Path of the built quarto under test, when binary mode is active. export function quartoTestBin(): string | undefined { const bin = Deno.env.get("QUARTO_TEST_BIN"); @@ -267,31 +270,50 @@ export interface RunQuartoResult { stderrTail?: string; } +// Dispatches to the in-process dev sources or, when QUARTO_TEST_BIN is set, +// the built binary. The two modes have distinct process/logging/timeout +// mechanics, so each lives in its own helper below. export async function runQuarto( args: string[], options: RunQuartoOptions = {}, ): Promise { const bin = quartoTestBin(); - const timeoutMs = options.timeoutMs ?? 600000; + return bin + ? runBinaryQuarto(bin, args, options) + : runDevQuarto(args, options); +} - if (!bin) { - // dev mode: in-process call, preserving existing semantics exactly - // (a timeout rejects but does not kill the in-process render) - let timer: ReturnType | undefined; - const timeout = new Promise((_resolve, reject) => { - timer = setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); - }); - try { - await Promise.race([quarto(args, undefined, options.env), timeout]); - } finally { - if (timer !== undefined) { - clearTimeout(timer); - } +// dev mode: in-process call, preserving existing semantics exactly +// (a timeout rejects but does not kill the in-process render) +async function runDevQuarto( + args: string[], + options: RunQuartoOptions, +): Promise { + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; + let timer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); + }); + try { + await Promise.race([quarto(args, undefined, options.env), timeout]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); } - return { code: 0, timedOut: false }; } + return { code: 0, timedOut: false }; +} +// binary mode: spawn the built quarto, enforce the timeout by killing the +// process tree, reconcile the child's log into the caller's, and apply the +// failure policy. +async function runBinaryQuarto( + bin: string, + args: string[], + options: RunQuartoOptions, +): Promise { assertTestBinary(bin); + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; const throwOnFailure = options.throwOnFailure ?? true; // The binary's LogFileHandler opens --log in truncate mode, so two @@ -341,45 +363,13 @@ export async function runQuarto( const commandLine = `quarto ${args.join(" ")}`; if (options.logFile && childLog) { - // merge this invocation's records into the caller's log file - let childContent = ""; - try { - childContent = Deno.readTextFileSync(childLog); - } catch { - // child never wrote the log (e.g. failed before logger init) - } - try { - Deno.removeSync(childLog); - } catch { - // best effort - } - if (childContent.length > 0) { - let existing = ""; - try { - existing = Deno.readTextFileSync(options.logFile); - } catch { - // log file may not exist yet - } - const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; - Deno.writeTextFileSync(options.logFile, existing + sep + childContent); - } - if (timedOut) { - appendLogError( - options.logFile, - `${commandLine} timed out after ${timeoutMs}ms and was killed`, - ); - } else if (output.code !== 0 && !hasErrorRecordText(childContent)) { - // A child can exit non-zero without any ERROR record: failures - // before logger init (deno startup, bundle load, missing share) and - // commandFailed paths (quarto add/remove). Without this synthetic - // record, log-only verifiers (noErrorsOrWarnings — the default - // smoke-all spec) would pass vacuously against an empty log. - appendLogError( - options.logFile, - `${commandLine} exited with code ${output.code} without logging an error\n` + - `stderr (tail):\n${stderrTail}`, - ); - } + mergeChildLog(options.logFile, childLog, { + timedOut, + code: output.code, + timeoutMs, + commandLine, + stderrTail, + }); } if ((output.code !== 0 || timedOut) && throwOnFailure) { @@ -392,3 +382,57 @@ export async function runQuarto( return { code: output.code, timedOut, stderrTail }; } + +// Merges a binary invocation's temp log into the caller's log file and +// synthesizes an ERROR record when the child failed without logging one, so +// log-reading verifiers can see the failure. Deletes the temp log. +function mergeChildLog( + logFile: string, + childLog: string, + outcome: { + timedOut: boolean; + code: number; + timeoutMs: number; + commandLine: string; + stderrTail: string; + }, +) { + let childContent = ""; + try { + childContent = Deno.readTextFileSync(childLog); + } catch { + // child never wrote the log (e.g. failed before logger init) + } + try { + Deno.removeSync(childLog); + } catch { + // best effort + } + if (childContent.length > 0) { + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // log file may not exist yet + } + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + childContent); + } + if (outcome.timedOut) { + appendLogError( + logFile, + `${outcome.commandLine} timed out after ${outcome.timeoutMs}ms and was killed`, + ); + } else if (outcome.code !== 0 && !hasErrorRecordText(childContent)) { + // A child can exit non-zero without any ERROR record: failures + // before logger init (deno startup, bundle load, missing share) and + // commandFailed paths (quarto add/remove). Without this synthetic + // record, log-only verifiers (noErrorsOrWarnings — the default + // smoke-all spec) would pass vacuously against an empty log. + appendLogError( + logFile, + `${outcome.commandLine} exited with code ${outcome.code} without logging an error\n` + + `stderr (tail):\n${outcome.stderrTail}`, + ); + } +} diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 8ae24f82b0..eeee0ea1db 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -73,10 +73,13 @@ if [[ -n "$QUARTO_TEST_BIN" ]]; then echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) does not exist or is not executable" exit 1 fi - # Probe with the dev-tree env stripped (same list as tests/quarto-cmd.ts): - # the installed launcher keeps an inherited QUARTO_SHARE_PATH, and the dev - # exports above would make a healthy built quarto read the dev tree's - # (nonexistent) src/resources/version and report an EMPTY version. + # Probe with the dev-tree env stripped. This is the subset of + # tests/quarto-cmd.ts's kStripEnvVars that affects `--version` resolution + # (share/root/deno paths) — the logging/profile vars in the full list don't + # change --version output, so they're omitted here. The installed launcher + # keeps an inherited QUARTO_SHARE_PATH, and the dev exports above would make + # a healthy built quarto read the dev tree's (nonexistent) + # src/resources/version and report an EMPTY version. QUARTO_TEST_BIN_VERSION="$(env -u QUARTO_SHARE_PATH -u QUARTO_BIN_PATH \ -u QUARTO_DEBUG -u DENO_DIR -u QUARTO_DENO -u QUARTO_DENO_DOM \ -u QUARTO_ROOT -u QUARTO_SRC_PATH -u QUARTO_FORCE_VERSION \ diff --git a/tests/smoke/extensions/extension-render-journals.test.ts b/tests/smoke/extensions/extension-render-journals.test.ts index 3ed045c42b..9137c9f981 100644 --- a/tests/smoke/extensions/extension-render-journals.test.ts +++ b/tests/smoke/extensions/extension-render-journals.test.ts @@ -9,6 +9,7 @@ import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const journalRepos = [ // { repo: "acm", noSupporting: true }, TODO this format needs changes after this merge. @@ -42,15 +43,14 @@ for (const journalRepo of journalRepos) { // Sets up the test setup: async () => { console.log(`using quarto-journals/${journalRepo.repo}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await runQuarto([ - "use", - "template", - `quarto-journals/${journalRepo.repo}`, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + `quarto-journals/${journalRepo.repo}`, + "--no-prompt", + ]); + }); }, // Cleans up the test diff --git a/tests/smoke/extensions/extension-render-typst-templates.test.ts b/tests/smoke/extensions/extension-render-typst-templates.test.ts index 3511e59022..1d62194c5b 100644 --- a/tests/smoke/extensions/extension-render-typst-templates.test.ts +++ b/tests/smoke/extensions/extension-render-typst-templates.test.ts @@ -9,6 +9,7 @@ import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const GITHUB_REPO = "quarto-ext/typst-templates"; @@ -40,15 +41,14 @@ for (const name of typstTemplates) { setup: async () => { const source = `${GITHUB_REPO}/${name}`; console.log(`using template: ${source}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await runQuarto([ - "use", - "template", - source, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + source, + "--no-prompt", + ]); + }); }, teardown: async () => { diff --git a/tests/smoke/render/render-format-extension.test.ts b/tests/smoke/render/render-format-extension.test.ts index e9743fb08a..09eaa5ab75 100644 --- a/tests/smoke/render/render-format-extension.test.ts +++ b/tests/smoke/render/render-format-extension.test.ts @@ -12,7 +12,7 @@ // Both files serve different purposes and should remain separate. import { safeRemoveSync } from "../../../src/core/path.ts"; -import { docs } from "../../utils.ts"; +import { docs, withCwd } from "../../utils.ts"; import { runQuarto } from "../../quarto-cmd.ts"; import { testRender } from "./render.ts"; @@ -28,19 +28,16 @@ import { testRender } from "./render.ts"; const updateExtensions = async () => { try { console.log("Updating quarto-journals extensions to latest versions..."); - const wd = Deno.cwd(); - Deno.chdir(docs("extensions/format/academic")); - - for (const repo of ["acs", "elsevier"]) { - await runQuarto([ - "update", - "extension", - `quarto-journals/${repo}`, - "--no-prompt", - ]); - } - - Deno.chdir(wd); + await withCwd(docs("extensions/format/academic"), async () => { + for (const repo of ["acs", "elsevier"]) { + await runQuarto([ + "update", + "extension", + `quarto-journals/${repo}`, + "--no-prompt", + ]); + } + }); console.log("Extensions updated successfully"); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/tests/utils.ts b/tests/utils.ts index 813a371a27..4766aa6212 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -32,6 +32,22 @@ export async function withTempDir( } } +// Runs fn with the process cwd changed to dir, restoring the original cwd +// even if fn throws. Bare Deno.chdir(dir) ... Deno.chdir(wd) leaks the cwd +// into later tests when fn rejects. +export async function withCwd( + dir: string, + fn: () => T | Promise, +): Promise { + const wd = Deno.cwd(); + Deno.chdir(dir); + try { + return await fn(); + } finally { + Deno.chdir(wd); + } +} + // Find a _quarto.yaml file in the directory hierarchy of the input file export function findProjectDir(input: string, until?: RegExp | undefined): string | undefined { let dir = dirname(input); From ada430548b6626d9e601a0d7603abe72002a602d Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 17 Jul 2026 22:18:47 +0200 Subject: [PATCH 37/43] ci: TEMP push trigger + small bucket for branch shakedown Temporary so the built-version smoke workflow runs on this PR branch before it lands on the default branch (workflow_dispatch is not available until then). Push trigger + a single-file bucket keep the run cheap. Both are reverted before merge. --- .github/workflows/test-smokes-built.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 932dc1a289..2cda02ae91 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -27,6 +27,9 @@ # dispatch => the source input (default build). name: Smoke Tests (Built Version) on: + # TEMP: shakedown trigger for this PR branch - REMOVE before merge. + push: + branches: [test/smoke-tests-built-version] # Fires on EVERY completed create-release run - the nightly schedule # (daily built-version testing, right after each build, no stale-artifact # risk) but also manual create-release dispatches, whose builds get tested @@ -113,7 +116,9 @@ jobs: quarto-install: artifact quarto-artifact-name: built-quarto-linux-amd64 runners: '["ubuntu-latest"]' - buckets: ${{ github.event.inputs.buckets }} # empty = full run + # TEMP: small bucket for PR-branch shakedown - restore to + # `${{ github.event.inputs.buckets }}` before merge. + buckets: '["smoke/render/render-output-dir.test.ts"]' # source: release (workflow_dispatch only) resolve-release: From fcc7a15e1a86e14e80c2ca7a691fc4b0c50dcdf6 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Mon, 20 Jul 2026 11:45:40 +0200 Subject: [PATCH 38/43] ci: remove PR-branch shakedown scaffolding from built-version workflow The push trigger and the hardcoded single-bucket override existed only to exercise this workflow on the feature branch during development. Restore the buckets input passthrough (empty = full run) and drop the branch push trigger so the workflow fires only via workflow_run/dispatch as designed. --- .github/workflows/test-smokes-built.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 2cda02ae91..932dc1a289 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -27,9 +27,6 @@ # dispatch => the source input (default build). name: Smoke Tests (Built Version) on: - # TEMP: shakedown trigger for this PR branch - REMOVE before merge. - push: - branches: [test/smoke-tests-built-version] # Fires on EVERY completed create-release run - the nightly schedule # (daily built-version testing, right after each build, no stale-artifact # risk) but also manual create-release dispatches, whose builds get tested @@ -116,9 +113,7 @@ jobs: quarto-install: artifact quarto-artifact-name: built-quarto-linux-amd64 runners: '["ubuntu-latest"]' - # TEMP: small bucket for PR-branch shakedown - restore to - # `${{ github.event.inputs.buckets }}` before merge. - buckets: '["smoke/render/render-output-dir.test.ts"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run # source: release (workflow_dispatch only) resolve-release: From 12a7eb761c7a7dd3af9b295f1b72667fbeab18c2 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Mon, 20 Jul 2026 11:46:27 +0200 Subject: [PATCH 39/43] tests: create the caller log even when a built quarto logs nothing test.ts treats a missing log target as a hard failure ("test log file is missing"). mergeChildLog only wrote the caller log when the child produced output, so a quiet successful command (empty child log, exit 0, no synthetic ERROR) left no file at all. Harmless for the always-created temp log, but a smoke test pinning a custom logConfig.log for a quiet command would fail spuriously. Ensure the caller log exists (empty) in that case so verifiers read an empty record array instead of the harness reporting a missing file. --- tests/quarto-cmd.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index 0caecd236d..bfea93237c 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -408,15 +408,21 @@ function mergeChildLog( } catch { // best effort } + // Always ensure logFile exists, even when the child logged nothing on a + // successful run: test.ts treats a MISSING logTarget as a hard failure + // ("test log file is missing"). A quiet successful command must leave an + // empty log (which verifiers read as an empty record array), not no file. + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // log file may not exist yet + } if (childContent.length > 0) { - let existing = ""; - try { - existing = Deno.readTextFileSync(logFile); - } catch { - // log file may not exist yet - } const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; Deno.writeTextFileSync(logFile, existing + sep + childContent); + } else if (existing.length === 0) { + Deno.writeTextFileSync(logFile, ""); } if (outcome.timedOut) { appendLogError( From 6ebb6d55acf511fbe5d44ed20a300f0a9cd65840 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Mon, 20 Jul 2026 11:47:04 +0200 Subject: [PATCH 40/43] tests: keep a failed process-tree kill from crashing the test process killProcessTree runs fire-and-forget inside the timeout callback (child.output() resolves once the kill lands). The POSIX path guards every syscall, but the Windows path awaited an unguarded taskkill Deno.Command: if taskkill can't be spawned that rejection had no handler and could surface as an unhandled rejection under Deno's default. Guard the taskkill call and attach a catch at the call site so a failed kill degrades to a timeout rather than a crash. --- tests/quarto-cmd.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index bfea93237c..f0d94aa695 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -204,11 +204,15 @@ export function assertTestBinary(bin: string) { // and kill can escape.) async function killProcessTree(pid: number) { if (isWindows) { - await new Deno.Command("taskkill", { - args: ["/PID", String(pid), "/T", "/F"], - stdout: "null", - stderr: "null", - }).output(); + try { + await new Deno.Command("taskkill", { + args: ["/PID", String(pid), "/T", "/F"], + stdout: "null", + stderr: "null", + }).output(); + } catch { + // taskkill unavailable or the tree already exited + } return; } const pids: number[] = []; @@ -349,7 +353,9 @@ async function runBinaryQuarto( let timedOut = false; const timer = setTimeout(() => { timedOut = true; - killProcessTree(child.pid); + // fire-and-forget: child.output() below resolves once the kill lands. + // Swallow any rejection so it never surfaces as an unhandled rejection. + killProcessTree(child.pid).catch(() => {}); }, timeoutMs); // output() drains both streams (undrained pipes deadlock at the 64KiB From 2ed6485608323abe60ab65d4d86ea2f84084578b Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Mon, 20 Jul 2026 11:47:34 +0200 Subject: [PATCH 41/43] tests: tolerate a torn log line so a timeout reports as a timeout readExecuteOutput called bare JSON.parse and threw on the first unparseable line, while hasErrorRecordText already tolerates partial lines. A timeout that kills a built quarto mid-write leaves a torn record; mergeChildLog appends a clean synthetic timeout ERROR after it, but the throw fired before any verifier saw that record, so the test failed with a JSON parse error instead of the timeout. Skip unparseable lines so the synthetic ERROR surfaces. --- tests/test.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/test.ts b/tests/test.ts index 06b70ef2fd..2b52c8cf2a 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -428,7 +428,18 @@ export function test(test: TestDescriptor) { export function readExecuteOutput(log: string) { const jsonStream = Deno.readTextFileSync(log); const lines = jsonStream.split("\n").filter((line) => !!line); - return lines.map((line) => { - return JSON.parse(line) as ExecuteOutput; - }); + const records: ExecuteOutput[] = []; + for (const line of lines) { + // Tolerate a torn final line: a timeout-killed built quarto can leave a + // partial record before mergeChildLog appends the synthetic timeout + // ERROR. Skipping it lets that ERROR reach the verifiers instead of a + // JSON.parse throw masking the real timeout (matches hasErrorRecordText + // in quarto-cmd.ts). + try { + records.push(JSON.parse(line) as ExecuteOutput); + } catch { + // ignore unparseable line + } + } + return records; } From cc83cb0c49b7cae19f09a2b1fec3ebf732847557 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Mon, 20 Jul 2026 11:54:12 +0200 Subject: [PATCH 42/43] tests: fall back to a direct kill when taskkill cannot be spawned The Windows catch swallowed a failed taskkill and returned, killing nothing; runBinaryQuarto then keeps awaiting child.output(), so a hung child could block the test process instead of degrading to a timeout. Best-effort Deno.kill on the direct child so output() can resolve. This does not reach an orphaned grandchild renderer, but is strictly better than killing nothing on the near-impossible case that taskkill itself is unavailable. --- tests/quarto-cmd.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index f0d94aa695..8bff470227 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -211,7 +211,16 @@ async function killProcessTree(pid: number) { stderr: "null", }).output(); } catch { - // taskkill unavailable or the tree already exited + // taskkill unavailable (should not happen on a real Windows runner) or + // the tree already exited. Fall back to killing the direct child so the + // awaited child.output() can resolve instead of blocking on a still-live + // launcher; this cannot reach an orphaned grandchild renderer, but is + // strictly better than killing nothing. + try { + Deno.kill(pid, "SIGKILL"); + } catch { + // already exited + } } return; } From 44d04711c0952566ec38ccfc89023cdc37ee57c1 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Mon, 20 Jul 2026 11:58:01 +0200 Subject: [PATCH 43/43] tests: strip a torn log line at the source instead of loosening the reader readExecuteOutput was made to skip every unparseable line, which also disabled the malformed-JSON detection log-level-and-formats.test.ts relies on (it treats a parse throw as a failed "valid JSON output" assertion). Restore the strict reader and instead drop a torn trailing line in mergeChildLog, which is the only place that knows a timeout kill happened and could have interrupted the child mid-write. The synthetic timeout ERROR still reaches the verifiers, and genuine malformed output is caught again. --- tests/quarto-cmd.ts | 27 +++++++++++++++++++++++++++ tests/test.ts | 21 +++++++-------------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index 8bff470227..1915473e30 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -127,6 +127,29 @@ export function appendLogError(logFile: string, msg: string) { Deno.writeTextFileSync(logFile, existing + sep + record + "\n"); } +// A timeout kill can interrupt the child mid-write, leaving a torn (partial) +// final JSON line. Drop a trailing unparseable line so the merged log stays +// valid json-stream — readExecuteOutput is deliberately strict (a parse throw +// is how the logging tests detect malformed output), so the torn line must be +// removed at the source rather than tolerated by every reader. +function stripTornTrailingLine(content: string): string { + const lines = content.split("\n"); + let i = lines.length - 1; + while (i >= 0 && lines[i] === "") { + i--; + } + if (i < 0) { + return content; + } + try { + JSON.parse(lines[i]); + return content; + } catch { + lines.splice(i, 1); + return lines.join("\n"); + } +} + function hasErrorRecordText(content: string): boolean { for (const line of content.split("\n")) { if (!line) continue; @@ -423,6 +446,10 @@ function mergeChildLog( } catch { // best effort } + // Only a timeout kill can tear a line; a clean exit flushes whole records. + if (outcome.timedOut) { + childContent = stripTornTrailingLine(childContent); + } // Always ensure logFile exists, even when the child logged nothing on a // successful run: test.ts treats a MISSING logTarget as a hard failure // ("test log file is missing"). A quiet successful command must leave an diff --git a/tests/test.ts b/tests/test.ts index 2b52c8cf2a..31a37fddfd 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -425,21 +425,14 @@ export function test(test: TestDescriptor) { Deno.test(args); } +// Strict on purpose: a JSON.parse throw is how log-level-and-formats.test.ts +// detects that quarto emitted malformed JSON-stream output. A timeout-killed +// built quarto can leave a torn final line, but that is stripped at the source +// in mergeChildLog (tests/quarto-cmd.ts) so the merged log stays valid here. export function readExecuteOutput(log: string) { const jsonStream = Deno.readTextFileSync(log); const lines = jsonStream.split("\n").filter((line) => !!line); - const records: ExecuteOutput[] = []; - for (const line of lines) { - // Tolerate a torn final line: a timeout-killed built quarto can leave a - // partial record before mergeChildLog appends the synthetic timeout - // ERROR. Skipping it lets that ERROR reach the verifiers instead of a - // JSON.parse throw masking the real timeout (matches hasErrorRecordText - // in quarto-cmd.ts). - try { - records.push(JSON.parse(line) as ExecuteOutput); - } catch { - // ignore unparseable line - } - } - return records; + return lines.map((line) => { + return JSON.parse(line) as ExecuteOutput; + }); }