diff --git a/.claude/rules/testing/built-version-ci.md b/.claude/rules/testing/built-version-ci.md new file mode 100644 index 00000000000..cf82f36fa22 --- /dev/null +++ b/.claude/rules/testing/built-version-ci.md @@ -0,0 +1,18 @@ +--- +paths: + - ".github/workflows/test-smokes.yml" + - ".github/workflows/test-smokes-built.yml" + - ".github/workflows/test-ff-matrix.yml" + - ".github/actions/build-dist-tarball/**" +--- + +# Built-Version CI Workflows + +Before changing these workflows, read `llm-docs/built-version-testing-architecture.md` — the decision record (D1–D10) for the resolver + scheduler topology, the three test legs (smoke / playwright / ff-matrix), per-leg OS policy, and the reusable `test-ff-matrix.yml` interface. Settled trade-offs (no windows playwright leg, per-OS `playwright-report-*` artifact names, `workflow_run` wiring, the ff-matrix concurrency-group suffix) are recorded there — don't relitigate them by accident. + +Invariants when editing: + +- Every playwright/ff-matrix leg's `if:` in `test-smokes-built.yml` must keep BOTH the mode-resolution expression and the `github.event.inputs.buckets == ''` clause (a new leg that forgets the latter silently breaks the buckets-override semantics, with green CI). +- `test-ff-matrix.yml` owns the ff-matrix bucket glob — never duplicate it in a caller. +- Per-leg OS scope is set via the explicit `runners:` inputs on the scheduler jobs in `test-smokes-built.yml`, and only there. +- `test-ff-matrix.yml`'s top-level `concurrency` group needs its per-call suffix: a called workflow's concurrency evaluates in the caller's context, and a shared cancel-in-progress group would let one leg cancel a sibling. diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index bec82f921b2..9ada45b9496 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -28,6 +28,8 @@ QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh test.ts # Linux/macOS $env:QUARTO_TESTS_NO_CONFIG=$true; .\run-tests.ps1 # Windows ``` +**Binary mode:** set `QUARTO_TEST_BIN` to a built quarto (extracted outside the checkout — the runner refuses the 99.9.9 dev sentinel) to spawn it instead of running the dev sources in-process; defaults to `smoke/` (`unit/` is dev-only; the playwright suite and ff-matrix corpus are binary-compatible and run as their own CI legs in `test-smokes-built.yml`, or locally by passing them explicitly). See `tests/README.md` → "Binary mode"; architecture and design decisions in `llm-docs/built-version-testing-architecture.md`. + ## Test Types | Type | Location | File Pattern | Details | @@ -59,6 +61,7 @@ Managed via: | File | Purpose | |------|---------| | `test.ts` | Test infrastructure (`testQuartoCmd`, `unitTest`) | +| `quarto-cmd.ts` | Quarto invocation dispatch (`runQuarto`; in-process dev vs `QUARTO_TEST_BIN` binary mode) | | `verify.ts` | Verification functions | | `utils.ts` | Path utilities (`docs()`, `outputForInput()`) | | `README.md` | Comprehensive documentation | diff --git a/.claude/rules/testing/playwright-tests.md b/.claude/rules/testing/playwright-tests.md index 8cc3781a043..59bd0e749ce 100644 --- a/.claude/rules/testing/playwright-tests.md +++ b/.claude/rules/testing/playwright-tests.md @@ -44,6 +44,14 @@ The wrapper (`playwright-tests.test.ts`): 3. Runs `npx playwright test` 4. Cleans up rendered output +The suite runs in dev CI (the per-commit parallel shards) AND against a +built quarto (`test-smokes-built.yml` playwright legs, binary mode). Two +constraints follow: the wrapper's render spawns must keep +`quartoSpawnEnvOptions()` (without it the built quarto inherits the +dev-tree env and silently renders with dev resources), and the browser +assertions are ignored on Windows CI (the wrapper's `ignore:` gate) — which +is why built-version CI has no windows playwright leg. + ## Test Structure Tests use `@playwright/test` framework: diff --git a/.claude/rules/testing/test-anti-patterns.md b/.claude/rules/testing/test-anti-patterns.md index ab18275a715..88cc67927ae 100644 --- a/.claude/rules/testing/test-anti-patterns.md +++ b/.claude/rules/testing/test-anti-patterns.md @@ -6,6 +6,20 @@ paths: # Test Anti-Patterns +## Don't: Rely on `execute()` Throwing to Fail a Test + +The harness catches every error thrown from a `TestDescriptor.execute` (including +`runQuarto` failures and inline `assert`s) and converts it into a log record — +the test then passes unless a **verifier** fails. A custom `execute` with +`verify: []`, or with verifiers that never read the log, is silently green on +failure. Rules: put assertions in `verify`, not in `execute`; and any test using +`runQuarto(..., { throwOnFailure: false })` must include a log-reading verifier +(`noErrors` at minimum) so command failures surface with their actual error. + +## Don't: Import `src/quarto.ts` in Tests + +A direct `import { quarto } from "src/quarto.ts"` bypasses binary mode (`QUARTO_TEST_BIN`) and always tests the dev sources. Invoke quarto through `testQuartoCmd()`/`runQuarto()` (`tests/quarto-cmd.ts` is the single dispatch point), and resolve subprocess spawns with `quartoDevCmd()` (`tests/utils.ts`). + ## Don't: Modify Environment Variables `Deno.env.set()` modifies process-global state. Deno runs test files in parallel by default, so other tests can see modified values. diff --git a/.claude/rules/testing/typescript-tests.md b/.claude/rules/testing/typescript-tests.md index d34043a4aae..901914931ad 100644 --- a/.claude/rules/testing/typescript-tests.md +++ b/.claude/rules/testing/typescript-tests.md @@ -22,7 +22,15 @@ TypeScript-based tests using Deno. Smoke tests render documents; unit tests veri ## Core Infrastructure -Core test files (`test.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. +Core test files (`test.ts`, `quarto-cmd.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. + +### Binary mode compatibility + +Tests must keep working when `QUARTO_TEST_BIN` targets a built quarto (binary mode) instead of the in-process dev sources: + +- Never `import { quarto } from "../../src/quarto.ts"` in tests — invoke quarto via `testQuartoCmd()`/`runQuarto()`. +- Subprocess spawns of quarto: resolve the executable with `quartoDevCmd()` (`tests/utils.ts`, honors `QUARTO_TEST_BIN`) or `quartoDevBinCmd()` (`tests/quarto-cmd.ts`, pins the local dev build), and pass `quartoSpawnEnvOptions()` as spawn env options. +- `TestContext.requiresDevQuarto: true` ignores a test in binary mode — rare escape hatch for tests exercising quarto internals in-process. ### Search for an existing verifier before writing one diff --git a/.github/actions/build-dist-tarball/action.yml b/.github/actions/build-dist-tarball/action.yml new file mode 100644 index 00000000000..9b4810c8ca8 --- /dev/null +++ b/.github/actions/build-dist-tarball/action.yml @@ -0,0 +1,53 @@ +name: "Build Quarto dist tarball" +description: | + Builds a Quarto distribution tarball from the current checkout: + configure.sh, quarto-bld prepare-dist --set-version, then a tar.gz of + package/pkg-working (top-level directory quarto-) uploaded as a + workflow artifact. Shared by create-release.yml (make-tarball / + make-arm64-tarball) and test-smokes-built.yml. +inputs: + version: + description: "Version to stamp into the distribution (--set-version)" + required: true + arch: + description: "Target architecture: amd64 or arm64" + required: false + default: "amd64" + tarball-name: + description: "File name of the produced .tar.gz" + required: true + artifact-name: + description: "Workflow artifact name to upload the tarball as" + required: true +runs: + using: "composite" + steps: + - name: Configure + shell: bash + run: ./configure.sh + + - name: Prepare Distribution + shell: bash + run: | + pushd package/src/ + if [ "${{ inputs.arch }}" = "arm64" ]; then + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --arch aarch64 --log-level info + else + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --log-level info + fi + popd + + - name: Make Tarball + shell: bash + run: | + pushd package/ + mv pkg-working quarto-${{ inputs.version }} + tar --owner=root --group=root -czf "${{ inputs.tarball-name }}" quarto-${{ inputs.version }} + mv quarto-${{ inputs.version }} pkg-working + popd + + - name: Upload Artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact-name }} + path: ./package/${{ inputs.tarball-name }} diff --git a/.github/actions/merge-extension-tests/action.yml b/.github/actions/merge-extension-tests/action.yml index 2091b9c95f8..9516a9080c4 100644 --- a/.github/actions/merge-extension-tests/action.yml +++ b/.github/actions/merge-extension-tests/action.yml @@ -11,6 +11,17 @@ runs: - name: Merge julia-engine tests shell: bash run: | + # TODO(binary-mode): the julia-engine subtree tests spawn quarto with + # raw Deno.Command (inherited env), so in binary mode the built quarto + # inherits the harness dev env (QUARTO_DEBUG, QUARTO_SHARE_PATH, ...) + # and crashes in checkReconfiguration / renders with dev-tree + # resources. Until the tests sanitize their spawn env upstream in + # PumasAI/quarto-julia-engine, skip them in binary mode. Plan: + # dev-docs/ci-julia-engine-binary-mode-followup.md + if [[ -n "$QUARTO_TEST_BIN" ]]; then + echo "::notice title=julia-engine tests skipped in binary mode::subtree tests do not sanitize their quarto spawn env yet (see dev-docs/ci-julia-engine-binary-mode-followup.md)" + exit 0 + fi SUBTREE=src/resources/extension-subtrees/julia-engine/tests cp -r "$SUBTREE/docs/julia-engine" tests/docs/julia-engine cp -r "$SUBTREE/smoke/julia-engine" tests/smoke/julia-engine diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 04ebc42003e..f4dfddeb7de 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -14,13 +14,20 @@ on: required: false type: boolean default: true + smoke-artifacts-only: + description: "Build only the artifacts consumed by Smoke Tests (Built Version) nightly mode: the linux amd64 tarball and the signed Windows zip. Skips source/arm64 tarballs, deb/rpm installers, and the macOS build. For cheap signed builds of a branch (dispatch with publish-release=false); incompatible with publishing." + required: false + type: boolean + default: false env: NFPM_VERSION: "2.43.1" concurrency: # make publishing release concurrent (but others trigger not) - group: building-releases-${{ inputs.publish-release && 'prerelease' || github.run_id }} + # smoke-artifacts-only runs never publish, so they must not queue in the + # shared 'prerelease' group behind/ahead of a real release + group: building-releases-${{ inputs.publish-release && !inputs.smoke-artifacts-only && 'prerelease' || github.run_id }} jobs: configure: @@ -35,6 +42,16 @@ jobs: tag_pushed: ${{ steps.version_commit.outputs.tag_pushed }} if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'quarto-dev/quarto-cli') steps: + # publish-release defaults to TRUE, so checking only the + # smoke-artifacts-only box would otherwise reach the version + # commit+tag step below and push an orphan tag (the publish-release + # job itself is guarded, but the tag push lives in this job) + - name: Fail on publish-release + smoke-artifacts-only + if: ${{ inputs.publish-release && inputs.smoke-artifacts-only }} + run: | + echo "::error::smoke-artifacts-only builds cannot publish - re-dispatch with publish-release=false" + exit 1 + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -100,6 +117,7 @@ jobs: default_author: github_actions make-source-tarball: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] steps: @@ -133,32 +151,16 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-amd64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-amd64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-amd64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: amd64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-amd64.tar.gz + artifact-name: Deb Zip make-arm64-tarball: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] steps: @@ -170,30 +172,13 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --arch aarch64 --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-arm64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-arm64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Arm64 Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-arm64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: arm64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-arm64.tar.gz + artifact-name: Deb Arm64 Zip make-tarball-rhel: runs-on: ubuntu-latest @@ -246,6 +231,7 @@ jobs: path: ./package/quarto-${{needs.configure.outputs.version}}-linux-rhel7-amd64.tar.gz make-installer-linux: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] strategy: @@ -477,6 +463,7 @@ jobs: quarto --version make-installer-mac: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: macos-latest needs: [configure] steps: @@ -546,6 +533,7 @@ jobs: security delete-keychain build.keychain test-zip-mac: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: macos-latest needs: [configure, make-installer-mac] steps: @@ -585,7 +573,8 @@ jobs: quarto --version publish-release: - if: ${{ inputs.publish-release }} + # never publish from a partial (smoke-artifacts-only) build + if: ${{ inputs.publish-release && !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [ configure, diff --git a/.github/workflows/test-ff-matrix.yml b/.github/workflows/test-ff-matrix.yml index 29645fcd5bd..5cf0f21b9c6 100644 --- a/.github/workflows/test-ff-matrix.yml +++ b/.github/workflows/test-ff-matrix.yml @@ -1,5 +1,50 @@ name: Feature-Format Tests on: + # Reusable interface: lets test-smokes-built.yml run the feature-format + # corpus against a BUILT quarto (its ff-matrix legs) while this workflow + # stays the single owner of the ff-matrix bucket glob. All inputs are + # forwarded to test-smokes.yml; their defaults preserve today's dev + # behavior, and the `inputs.x || default` fallbacks in the job keep the + # schedule/push/PR triggers (where inputs.* is empty) on those defaults. + # Nesting: test-smokes-built.yml -> this -> test-smokes.yml = 3 levels, + # well within GitHub's reusable-workflow nesting limit. + workflow_call: + inputs: + extra-r-packages: + description: "extra R package to install for the runs (like a dev version of one of the deps) - comma separated, passed to renv::install" + required: false + type: string + default: "" + quarto-install: + description: "Which quarto to test (forwarded to test-smokes.yml): 'dev' (source tree, default), 'release', or 'artifact'" + required: false + type: string + default: "dev" + quarto-version: + description: "Version to install when quarto-install is 'release' (forwarded to test-smokes.yml)" + required: false + type: string + default: "" + quarto-artifact-name: + description: "Workflow artifact name containing the built quarto tarball/zip when quarto-install is 'artifact' (forwarded to test-smokes.yml)" + required: false + type: string + default: "" + quarto-artifact-run-id: + description: "Workflow run id to download the artifact from (forwarded to test-smokes.yml; empty = the current run)" + required: false + type: string + default: "" + ref: + description: "Git ref to check out (forwarded to test-smokes.yml; empty means the default checkout behavior)" + required: false + type: string + default: "" + runners: + description: "JSON list of runner labels for the OS matrix (forwarded to test-smokes.yml)" + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' workflow_dispatch: inputs: extra-r-packages: @@ -19,6 +64,7 @@ on: - ".github/workflows/performance-check.yml" - ".github/workflows/stale-needs-repro.yml" - ".github/workflows/test-bundle.yml" + - ".github/workflows/test-smokes-built.yml" - ".github/workflows/test-smokes-parallel.yml" - ".github/workflows/test-install.yml" - ".github/workflows/test-quarto-latexmk.yml" @@ -33,12 +79,21 @@ on: - ".github/workflows/stale-needs-repro.yml" - ".github/workflows/test-bundle.yml" - ".github/workflows/test-install.yml" + - ".github/workflows/test-smokes-built.yml" - ".github/workflows/test-smokes-parallel.yml" - ".github/workflows/test-quarto-latexmk.yml" - ".github/workflows/update-test-timing.yml" concurrency: - group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }} + # The suffix matters under workflow_call (the built-version ff-matrix + # legs): a called workflow's top-level concurrency evaluates in the + # CALLER's context (github.workflow/ref/run_id are the caller run's), so + # without a per-call component every ff-matrix leg of one + # test-smokes-built.yml run would share a single cancel-in-progress group + # and could cancel a sibling leg. inputs.runners is only ever set on + # workflow_call, and differs per leg; dev triggers (inputs empty) get the + # constant '-dev' suffix, keeping today's dedup groups. + group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}-${{ inputs.runners && format('call-{0}-{1}', github.run_id, inputs.runners) || 'dev' }} cancel-in-progress: true jobs: @@ -46,5 +101,16 @@ jobs: name: Run feature-format matrix uses: ./.github/workflows/test-smokes.yml with: + # the ff-matrix bucket glob is defined ONLY here (DRY) - built-version + # callers reuse it through the workflow_call interface above buckets: '[ "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" ]' extra-r-packages: ${{ inputs.extra-r-packages }} + # inputs.* is empty on schedule/push/PR triggers; the fallbacks keep + # those paths on today's dev defaults (test-smokes.yml is also + # empty-string robust for the inputs passed through without fallback) + quarto-install: ${{ inputs.quarto-install || 'dev' }} + quarto-version: ${{ inputs.quarto-version }} + quarto-artifact-name: ${{ inputs.quarto-artifact-name }} + quarto-artifact-run-id: ${{ inputs.quarto-artifact-run-id }} + ref: ${{ inputs.ref }} + runners: ${{ inputs.runners || '["ubuntu-latest", "windows-latest"]' }} diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml new file mode 100644 index 00000000000..40dbdcaac33 --- /dev/null +++ b/.github/workflows/test-smokes-built.yml @@ -0,0 +1,430 @@ +# Runs the test suites against a BUILT quarto instead of the dev source tree. +# Three sources for the quarto under test: +# - nightly (the workflow_run path - runs after every nightly create-release +# build; also dispatchable): reuse the SIGNED artifacts of a completed +# create-release run (its nightly schedule builds without publishing) - +# the linux 'Deb Zip' tarball, the signed 'Windows Zip' (the real +# quarto.exe), AND the signed+notarized 'Mac Zip' - and run the smokes at +# that run's commit. No duplicate build, three-OS coverage for free +# (the ONLY macOS smoke coverage in CI). Each OS leg is gated on its +# artifact existing in the run, so a smoke-artifacts-only build (no Mac +# Zip) gets linux+windows tested instead of a red mac job. To test a +# signed build of a specific commit on demand: dispatch create-release +# with publish-release=false (+ smoke-artifacts-only for a cheaper +# linux+windows build) - the workflow_run trigger then tests it +# automatically; dispatching this workflow with source=nightly and a +# run-id is only needed to re-test an older run. +# - build (dispatch default): build a linux-amd64 dist from THIS ref with the +# same recipe as create-release (shared build-dist-tarball action) and test +# it. The only mode that works for arbitrary refs/PR branches and on forks +# (no signing secrets or create-release runs required). +# - release: install a published (pre-)release with quarto-actions/setup and +# run the smokes at the matching refs/tags/v. +# nightly/release only work when the target commit's harness has binary-mode +# support (tests/quarto-cmd.ts) - see llm-docs/built-version-testing-architecture.md (D10). +# +# Mode resolution (repeated in each job's `if`): workflow_run => nightly; +# dispatch => the source input (default build). +# +# THREE TEST LEGS per source mode, each an independent caller job so one red +# suite never cancels the others: +# - smoke leg: the run-smokes-* jobs below. Doubles as the general bucket +# runner - its bucket is the `buckets` dispatch input (empty = the +# binary-mode default, smoke/). +# - playwright leg: bucket ["integration/playwright-tests.test.ts"] - +# renders the playwright corpus with the built quarto, then runs browser +# assertions over the served output. Linux in every mode, plus macOS on +# nightly; NEVER windows: the browser assertions are hard-ignored on +# Windows CI (playwright-tests.test.ts `ignore:` gate), so a windows leg +# would render-then-skip and report a misleading green. Revisit the gate +# before adding windows here. +# - ff-matrix leg: calls the reusable test-ff-matrix.yml, which owns the +# feature-format bucket glob. Linux in every mode, plus windows on +# nightly/release (matching the dev ff-matrix OS scope); no macOS - its +# Julia/TeX toolchain there is unproven. +# When the `buckets` dispatch input is set, only the smoke-slot leg runs +# (with that bucket); playwright + ff-matrix legs skip. Per-leg OS scope is +# tuned HERE (the runners inputs below), in one place. +name: Smoke Tests (Built Version) +on: + # Fires on EVERY completed create-release run - the nightly schedule + # (daily built-version testing, right after each build, no stale-artifact + # risk) but also manual create-release dispatches, whose builds get tested + # too (partial smoke-artifacts-only builds only run the legs whose + # artifacts exist). The workflow file must be on the default branch for + # workflow_run to fire; the triggering build may be on any branch. + workflow_run: + workflows: ["Build Installers"] + types: [completed] + workflow_dispatch: + inputs: + source: + description: "Where the quarto under test comes from: build = build from this ref now; nightly = artifacts of a create-release run; release = published (pre-)release. The workflow_run trigger (after each nightly build) always uses nightly." + required: false + type: choice + options: + - build + - nightly + - release + default: build + buckets: + description: "Optional JSON list of test buckets/globs for a partial run (e.g. the ff-matrix bucket); empty = full smoke run" + required: false + type: string + default: "" + version: + description: "For source=release: 'pre-release' (latest prerelease), 'release' (latest stable), or a literal version like 1.9.10" + required: false + type: string + default: "pre-release" + run-id: + description: "For source=nightly: create-release run id to take artifacts from (empty = latest successful run)" + required: false + type: string + default: "" + +# required by the called test-smokes.yml (julia cache cleanup) +permissions: + actions: write + contents: read + +jobs: + # source: build (dispatch default) - build from THIS ref + build-artifact: + name: Build quarto dist (linux-amd64) + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.rec.outputs.sha }} + steps: + - name: Checkout Repo + uses: actions/checkout@v6 + + - name: Record commit under test + id: rec + shell: bash + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Compute version marker + id: version + shell: bash + # BUILD METADATA version marker (+test.): semver-valid and + # range-transparent for every 'quarto-required' gate, while still + # distinguishable from the dev sentinel 99.9.9. NEVER use a '-suffix' + # (prereleases fail plain ranges) or a 4th dot component (not semver). + run: echo "version=$(cat version.txt)+test.$(date +%Y%m%d)" >> "$GITHUB_OUTPUT" + + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball + with: + version: ${{ steps.version.outputs.version }} + arch: amd64 + tarball-name: built-quarto-linux-amd64.tar.gz + artifact-name: built-quarto-linux-amd64 + + run-smokes-artifact: + name: Smoke tests against built artifact + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + # harness commit == binary commit: no version skew by construction + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-playwright-artifact: + name: Playwright tests against built artifact + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + && github.event.inputs.buckets == '' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-ff-matrix-artifact: + name: Feature-format matrix against built artifact + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + && github.event.inputs.buckets == '' + needs: [build-artifact] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + + # source: release (workflow_dispatch only) + resolve-release: + name: Resolve release version + if: github.event.inputs.source == 'release' + runs-on: ubuntu-latest + outputs: + version: ${{ steps.r.outputs.version }} + steps: + - name: Resolve version + id: r + shell: bash + run: | + input="${{ github.event.inputs.version }}" + case "$input" in + pre-release | prerelease | "") + version="$(curl -fsSL https://quarto.org/docs/download/_prerelease.json | jq -r '.version')" + ;; + release) + version="$(curl -fsSL https://quarto.org/docs/download/_download.json | jq -r '.version')" + ;; + *) + version="$input" + ;; + esac + if [ -z "$version" ] || [ "$version" = "null" ]; then + echo "::error::could not resolve a quarto version from input '$input'" + exit 1 + fi + echo "Resolved version: $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Preflight - tag must have binary-mode harness support + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.r.outputs.version }} + # releases cut before the QUARTO_TEST_BIN harness support merged cannot be + # tested in this mode (the harness at the tag lacks tests/quarto-cmd.ts); + # test-smokes.yml's "Pin and verify" step re-checks this in the checkout. + run: | + if ! gh api "repos/${GITHUB_REPOSITORY}/contents/tests/quarto-cmd.ts?ref=refs/tags/v${VERSION}" --silent 2>/dev/null; then + echo "::error::tests/quarto-cmd.ts not found at refs/tags/v${VERSION}: either the tag does not exist or the release predates binary-mode harness support - it cannot be tested in release mode" + exit 1 + fi + echo "refs/tags/v${VERSION} has binary-mode harness support" + + run-smokes-release: + name: Smoke tests against published release + if: github.event.inputs.source == 'release' + needs: [resolve-release] + uses: ./.github/workflows/test-smokes.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + runners: '["ubuntu-latest", "windows-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-playwright-release: + name: Playwright tests against published release + if: github.event.inputs.source == 'release' && github.event.inputs.buckets == '' + needs: [resolve-release] + uses: ./.github/workflows/test-smokes.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + # linux only: browser assertions are ignored on Windows CI (see header) + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-ff-matrix-release: + name: Feature-format matrix against published release + if: github.event.inputs.source == 'release' && github.event.inputs.buckets == '' + needs: [resolve-release] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + # ubuntu + windows, matching the dev ff-matrix scope + runners: '["ubuntu-latest", "windows-latest"]' + + # source: nightly (the workflow_run path, daily after each build; also + # dispatchable) - reuse the signed artifacts of a completed no-publish + # create-release run (linux tarball + signed Windows zip + notarized Mac zip) + resolve-nightly: + name: Resolve create-release run + # on workflow_run: only test successful builds (a skipped day is visible, + # a stale artifact is not) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') + runs-on: ubuntu-latest + outputs: + run-id: ${{ steps.r.outputs.run-id }} + sha: ${{ steps.r.outputs.sha }} + has-linux: ${{ steps.r.outputs.has-linux }} + has-windows: ${{ steps.r.outputs.has-windows }} + has-mac: ${{ steps.r.outputs.has-mac }} + steps: + - name: Resolve run id, commit and artifacts + id: r + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # workflow_run: the triggering build IS the run to test + run_id="${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.event.inputs.run-id }}" + if [ -z "$run_id" ]; then + run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow create-release.yml \ + --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')" + fi + if [ -z "$run_id" ]; then + echo "::error::no successful create-release run found to take artifacts from" + exit 1 + fi + sha="$(gh run view "$run_id" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha // empty')" + if [ -z "$sha" ]; then + echo "::error::could not resolve head_sha for run ${run_id}" + exit 1 + fi + # A smoke-artifacts-only build has no 'Mac Zip' (and workflow_run + # fires on EVERY completed create-release run, not only nightly + # schedules) - gate each OS leg on its artifact actually existing + # instead of failing red at download time. + # exclude expired artifacts: a source=nightly re-test dispatch of an + # old run-id would otherwise pass the has-* gates and fail at + # download time + names="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts" \ + --paginate --jq '.artifacts[] | select(.expired | not) | .name')" + has() { grep -Fxq "$1" <<<"$names" && echo true || echo false; } + has_linux="$(has 'Deb Zip')" + has_windows="$(has 'Windows Zip')" + has_mac="$(has 'Mac Zip')" + if [ "$has_linux" = false ] && [ "$has_windows" = false ] && [ "$has_mac" = false ]; then + echo "::error::run ${run_id} produced none of the expected artifacts (Deb Zip / Windows Zip / Mac Zip)" + exit 1 + fi + echo "Using create-release run ${run_id} at commit ${sha} (linux=${has_linux} windows=${has_windows} mac=${has_mac})" + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "has-linux=$has_linux" >> "$GITHUB_OUTPUT" + echo "has-windows=$has_windows" >> "$GITHUB_OUTPUT" + echo "has-mac=$has_mac" >> "$GITHUB_OUTPUT" + + run-smokes-nightly-linux: + name: Smoke tests against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + # harness commit == the commit create-release built from + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-windows: + name: Smoke tests against nightly build (windows) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-windows == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # the SIGNED Windows layout (quarto.exe) produced by make-installer-win + quarto-artifact-name: Windows Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["windows-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-mac: + name: Smoke tests against nightly build (macOS) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-mac == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # the SIGNED + notarized Mac layout produced by make-installer-mac - + # the only macOS smoke coverage in CI + quarto-artifact-name: Mac Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["macos-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + # playwright + ff-matrix nightly legs: each OS is its own caller job (a + # workflow_call passes ONE artifact name, and the signed nightly artifacts + # are per-OS downloads), gated on the same has-* flags as the smoke legs so + # a partial smoke-artifacts-only build skips instead of failing red. + run-playwright-nightly-linux: + name: Playwright tests against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-playwright-nightly-mac: + name: Playwright tests against nightly build (macOS) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-mac == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Mac Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["macos-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + # no windows playwright leg: browser assertions are ignored on Windows CI + # (see header) - a leg here would be a misleading green + + run-ff-matrix-nightly-linux: + name: Feature-format matrix against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + + run-ff-matrix-nightly-windows: + name: Feature-format matrix against nightly build (windows) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-windows == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Windows Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["windows-latest"]' + + # no macOS ff-matrix leg: the dev ff-matrix has never run on macOS (its + # Julia/TeX toolchain there is unproven) - one-line add here once proven diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 652807eddd7..7e2acbc4194 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -5,9 +5,10 @@ on: workflow_call: inputs: buckets: - description: "JSON string for buckets of tests to run in loop. Array of grouped tests." - required: true + description: "JSON string for buckets of tests to run in loop. Array of grouped tests. Empty means a full run." + required: false type: string + default: "" time-test: description: "Should we run tests to produce test file" required: false @@ -18,6 +19,36 @@ on: required: false type: string default: "" + quarto-install: + description: "Which quarto to test: 'dev' (source tree, default), 'release' (published release via quarto-actions/setup), or 'artifact' (built tarball from a workflow artifact)" + required: false + type: string + default: "dev" + quarto-version: + description: "Version to install when quarto-install is 'release' (passed to quarto-dev/quarto-actions/setup)" + required: false + type: string + default: "" + quarto-artifact-name: + description: "Workflow artifact name containing the built quarto tarball/zip when quarto-install is 'artifact'" + required: false + type: string + default: "" + quarto-artifact-run-id: + description: "Workflow run id to download the artifact from (empty = the current run); lets a caller test artifacts built by another workflow run, e.g. a no-publish create-release build" + required: false + type: string + default: "" + ref: + description: "Git ref to check out (empty means the default checkout behavior)" + required: false + type: string + default: "" + runners: + description: "JSON list of runner labels for the OS matrix. Policy: macos-latest is reserved for scheduled/built-version runs (test-smokes-built.yml nightly mode) - do NOT add it to per-commit callers (test-smokes-parallel.yml), which must stay fast." + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' workflow_dispatch: inputs: buckets: @@ -50,7 +81,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + # inputs is empty on workflow_dispatch/schedule triggers; fall back to the historical default + os: ${{ fromJSON(inputs.runners || '["ubuntu-latest", "windows-latest"]') }} time-test: - ${{ inputs.time-test }} exclude: @@ -63,6 +95,9 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v6 + with: + # empty (the default) keeps the standard checkout behavior + ref: ${{ inputs.ref }} - name: Fix temp dir to use runner one (windows) if: runner.os == 'Windows' @@ -86,7 +121,7 @@ jobs: node-version: 22 - name: Cache multiplex server node_modules - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} uses: actions/cache@v5 with: path: tests/integration/playwright/multiplex-server/node_modules @@ -95,13 +130,13 @@ jobs: ${{ runner.os }}-multiplex-server- - name: Install node dependencies - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} run: npm install --loglevel=error --no-audit working-directory: ./tests/integration/playwright shell: bash - name: Get Playwright version - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} run: | VERSION=$(node -p "require('@playwright/test/package.json').version") echo "PLAYWRIGHT_VERSION=$VERSION" >> "$GITHUB_ENV" @@ -109,7 +144,7 @@ jobs: shell: bash - name: Cache Playwright browsers - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} uses: actions/cache@v5 with: path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} @@ -118,13 +153,13 @@ jobs: ${{ runner.os }}-playwright- - name: Install Playwright system dependencies - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} timeout-minutes: 15 run: npx playwright install-deps working-directory: ./tests/integration/playwright - name: Install Playwright Browsers - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' || (inputs.quarto-install != 'dev' && inputs.quarto-install != '') }} timeout-minutes: 15 run: npx playwright install working-directory: ./tests/integration/playwright @@ -173,6 +208,14 @@ jobs: sudo apt-get update -y sudo apt-get install -y librsvg2-bin + - name: Install missing system deps (macOS) + if: runner.os == 'macOS' + # macos runners preinstall most toolchain deps; poppler (pdftotext/ + # pdftoppm) and librsvg (rsvg-convert) are the ones PDF tests need. + # Tests requiring other optional tools prereq-skip gracefully. + run: | + brew install poppler librsvg + - name: Restore R packages working-directory: tests run: | @@ -227,8 +270,103 @@ jobs: run: | uv sync --frozen + # ALWAYS runs, in every quarto-install mode: it provisions the Deno runtime, + # deno_cache and dev tree that the test harness itself needs. In non-dev modes + # the quarto under test is layered on top via a PATH prepend + QUARTO_TEST_BIN. - uses: ./.github/workflows/actions/quarto-dev + - name: Set up release quarto + if: inputs.quarto-install == 'release' + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: ${{ inputs.quarto-version }} + + - name: Download built quarto artifact + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id == '' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + + - name: Download built quarto artifact (from another run) + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id != '' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + run-id: ${{ inputs.quarto-artifact-run-id }} + github-token: ${{ github.token }} + + - name: Install built quarto outside the checkout + if: inputs.quarto-install == 'artifact' + shell: bash + # extraction must happen OUTSIDE the git checkout: the launcher detects a + # sibling src/quarto.ts and would silently run dev mode (dev-mode trap) + run: | + mkdir -p "$RUNNER_TEMP/quarto-under-test" + shopt -s nullglob + artifacts=("$RUNNER_TEMP"/quarto-artifact/*.tar.gz "$RUNNER_TEMP"/quarto-artifact/*.zip) + if [ "${#artifacts[@]}" -ne 1 ]; then + echo "::error::expected exactly one .tar.gz or .zip in the quarto artifact, found: ${artifacts[*]:-none}" + exit 1 + fi + # layouts differ per artifact: the linux tarball has a + # quarto-/ top dir; the Mac tarball and Windows Zip are flat + # ({bin,share} at the archive root). Extract, then flatten if needed. + # bsdtar (windows/mac runners) also handles .zip via tar -xf. + tar -xf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" + if [ ! -d "$RUNNER_TEMP/quarto-under-test/bin" ]; then + top="$(find "$RUNNER_TEMP/quarto-under-test" -mindepth 1 -maxdepth 1 -type d | head -1)" + if [ -z "$top" ] || [ ! -d "$top/bin" ]; then + echo "::error::could not locate bin/ in the extracted quarto artifact" + exit 1 + fi + mv "$top"/* "$RUNNER_TEMP/quarto-under-test"/ + fi + echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" + + - name: Pin and verify test target + if: inputs.quarto-install != 'dev' && inputs.quarto-install != '' + shell: bash + run: | + v="$(quarto --version)" + echo "quarto under test: $v ($(command -v quarto))" + if [ "$v" = "99.9.9" ]; then + echo "::error::quarto on PATH reports the dev sentinel version 99.9.9 - the built/released quarto was not picked up (dev-mode trap)" + exit 1 + fi + # version-marker guard: version must be plain semver, optionally with build + # metadata (+...). A '-prerelease' suffix or a 4th component would fail + # every 'quarto-required' range check in the corpus (semver range gates). + if ! echo "$v" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(\+[0-9A-Za-z.-]+)?$'; then + echo "::error::quarto version '$v' is not range-transparent semver (only build metadata '+...' is allowed; never a '-suffix' or a 4th component)" + exit 1 + fi + # preflight: the checked-out harness must support binary mode; older tags + # (before tests/quarto-cmd.ts existed) cannot be tested in this mode + if [ ! -f tests/quarto-cmd.ts ]; then + echo "::error::this checkout's test harness lacks binary-mode support (tests/quarto-cmd.ts is missing); release/artifact mode only works for refs that include the QUARTO_TEST_BIN harness support" + exit 1 + fi + # resolve the executable path in a form Deno.Command can spawn: + # on Windows, `command -v quarto` yields an extensionless bash-style + # path (/c/...) that neither Test-Path nor Deno.Command can use — + # prefer the real .exe/.cmd and convert to a native Windows path + if [ "$RUNNER_OS" = "Windows" ]; then + bin="$(command -v quarto.exe || command -v quarto.cmd || command -v quarto)" + bin="$(cygpath -w "$bin")" + else + bin="$(command -v quarto)" + fi + echo "QUARTO_TEST_BIN=$bin" >> "$GITHUB_ENV" + # when the caller pinned a concrete version, have the harness verify + # every spawn targets exactly that version + if [ -n "${{ inputs.quarto-version }}" ] \ + && [ "${{ inputs.quarto-version }}" != "pre-release" ] \ + && [ "${{ inputs.quarto-version }}" != "release" ]; then + echo "QUARTO_TEST_EXPECTED_VERSION=${{ inputs.quarto-version }}" >> "$GITHUB_ENV" + fi + - name: Install Tinytex env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -400,9 +538,13 @@ jobs: key: ${{ steps.cache-renv-packages-restore.outputs.cache-primary-key }} - uses: actions/upload-artifact@v7 - # Upload pLaywright test report if they exists (playwright is only running on Linux for now) + # Upload playwright test report if it exists (the browser assertions + # are ignored on Windows CI - see playwright-tests.test.ts). The name + # includes the runner OS: several calls of this workflow can share one + # workflow run (e.g. the per-OS playwright legs of + # test-smokes-built.yml) and duplicate artifact names fail the upload. if: ${{ !cancelled() && runner.os != 'Windows' && hashFiles('tests/integration/playwright/playwright-report/**/*') != '' }} with: - name: playwright-report + name: playwright-report-${{ runner.os }} path: ./tests/integration/playwright/playwright-report/ retention-days: 30 diff --git a/dev-docs/checklist-make-a-new-quarto-prerelease.md b/dev-docs/checklist-make-a-new-quarto-prerelease.md index f2b27a0d2fd..9256942fa15 100644 --- a/dev-docs/checklist-make-a-new-quarto-prerelease.md +++ b/dev-docs/checklist-make-a-new-quarto-prerelease.md @@ -11,5 +11,6 @@ - New release prerelease should be on Github at - A new tag should be on main for the new prerelease version - `version.txt` on main should have been updated by the workflow to the pre-release version just released: https://github.com/quarto-dev/quarto-cli/blob/main/version.txt +- (optional) validate the published prerelease: Actions -> "Smoke Tests (Built Version)" -> "Run Workflow" with source `release` (version defaults to `pre-release`) to run the smoke suite against the installed release binaries instead of the dev tree Note: Cloudsmith publishing is skipped for prereleases (only runs for stable releases). diff --git a/dev-docs/ci-julia-engine-binary-mode-followup.md b/dev-docs/ci-julia-engine-binary-mode-followup.md new file mode 100644 index 00000000000..881ed52a9c6 --- /dev/null +++ b/dev-docs/ci-julia-engine-binary-mode-followup.md @@ -0,0 +1,144 @@ +# Follow-up: run the julia-engine subtree tests in binary mode + +Status: **planned follow-up** (upstream change required). Companion to +`dev-docs/ci-test-log-grouping-design.md` (where the gap was found) and +`llm-docs/built-version-testing-architecture.md` (the binary-mode +architecture and its D3/D4 env-leak trap). + +## Current state: gated out of binary mode + +`.github/actions/merge-extension-tests` skips copying the julia-engine +subtree tests into `tests/` when `QUARTO_TEST_BIN` is set (with a +`::notice::` in the CI log). Dev-mode runs are unaffected — the tests still +run in every dev shard. Remove that gate as the last step of this plan. + +## The bug being avoided + +`smoke/julia-engine/{julia,render}.test.ts` come from the +`PumasAI/quarto-julia-engine` subtree +(`src/resources/extension-subtrees/julia-engine/tests/`) and are +self-contained: `jsr:` imports only, and they spawn quarto with raw +`new Deno.Command("quarto"| "quarto.cmd")` — PATH lookup plus the **fully +inherited environment**, bypassing the harness's `runQuarto()` dispatch. + +In binary mode the workflow prepends the built quarto to PATH, so the right +binary runs — but it inherits the dev env that `run-tests.[sh|ps1]` exports +for the harness process: + +- `QUARTO_DEBUG=true` → the built quarto takes the dev-only + `checkReconfiguration()` branch (`src/quarto.ts`) → + `readSourceDevConfig()` reads `/configuration`, which does not exist + outside the checkout → `NotFound: readfile '/home/runner/work/configuration'` + (the failure observed in the first long built-version runs, 2026-07-20). +- Even without the crash, leaked `QUARTO_SHARE_PATH`/`DENO_DIR`/... would + make the built quarto silently use dev-tree resources — the exact D3/D4 + trap `llm-docs/built-version-testing-architecture.md` documents, already + fixed for the playwright wrapper via `quartoSpawnEnvOptions()`. + +## Why there is no clean quarto-cli-side fix + +Evaluated and rejected: + +1. **Edit the copied tests here** — forbidden: subtree files are pulled from + the upstream repo (single source of truth); local edits are lost or + conflict on the next `pull-git-subtree` (`.claude/rules/extension-subtrees.md`). +2. **Patch the files during the `merge-extension-tests` copy** — a sed-style + fork of upstream code inside a composite action; drift-prone and worse + than the gate. +3. **Stop exporting the dev vars in binary mode (`run-tests.[sh|ps1]`)** — + relitigates settled decision D4: the *harness* process needs the dev-tree + env in all modes; only spawned children must be sanitized. +4. **A `quarto` shim earlier in PATH that `env -u`-strips and execs the real + binary** — needs a `.cmd` twin for Windows, hides the real defect, and + makes PATH resolution in CI even harder to reason about. + +The spawn site is the only correct place to sanitize, and the spawn site +lives upstream. + +## Upstream fix (PumasAI/quarto-julia-engine) + +The tests cannot import `tests/quarto-cmd.ts` (they must stay +self-contained), so they get a small local helper, e.g. `tests/spawn-env.ts` +in the subtree repo: + +```ts +// Env vars exported by quarto-cli's run-tests.[sh|ps1] for the dev-tree +// harness. They must not reach a spawned quarto under test: QUARTO_DEBUG +// triggers dev-only reconfiguration checks and the path vars make an +// installed quarto silently use dev-tree resources. +// Mirrors kStripEnvVars in quarto-cli tests/quarto-cmd.ts — keep in sync. +const kStripEnvVars = [ + "QUARTO_SHARE_PATH", + "QUARTO_BIN_PATH", + "QUARTO_DEBUG", + "DENO_DIR", + "QUARTO_DENO", + "QUARTO_DENO_DOM", + "QUARTO_ROOT", + "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION", + "QUARTO_VERSION_REQUIREMENT", + "QUARTO_PROJECT_DIR", + "QUARTO_PROFILE", + "QUARTO_LOG", + "QUARTO_LOG_LEVEL", + "QUARTO_LOG_FORMAT", + "RSTUDIO", +]; + +export function quartoSpawnOptions(): { env: Record; clearEnv: boolean } { + const env = { ...Deno.env.toObject() }; + for (const name of kStripEnvVars) delete env[name]; + return { env, clearEnv: true }; +} +``` + +Then every `new Deno.Command(quartoCmd(), { ... })` in +`tests/smoke/julia-engine/*.test.ts` adds `...quartoSpawnOptions()` to its +options. `clearEnv: true` + explicit env matches `quartoSpawnEnvOptions()` +semantics in quarto-cli (the sanitized env is authoritative). Stripping +unconditionally is fine — in dev mode the dev quarto re-derives these from +its own launcher, which is how a user shell invokes it anyway. + +Sync rule: the list mirrors `kStripEnvVars` in quarto-cli +`tests/quarto-cmd.ts`. If a var is added there, add it upstream too (the +comment in both files points each way). + +Manual sync is not enough — after the subtree pull, **both files live in +this repo** (`tests/quarto-cmd.ts` and +`src/resources/extension-subtrees/julia-engine/tests/spawn-env.ts`), so +drift is mechanically checkable here: add a quarto-cli unit test +(`tests/unit/strip-env-sync.test.ts`) that imports `kStripEnvVars` and +parses the subtree helper's list (regex over the file text — the subtree +file is `jsr:`-only and must not import harness modules), asserting set +equality. A future addition to `kStripEnvVars` then fails CI until the +upstream helper is updated and re-pulled, instead of silently re-opening +the env leak. + +## Procedure + +1. Open the upstream PR in `PumasAI/quarto-julia-engine` (helper + spawn + sites; their CI runs against a pinned quarto-cli rev and stays green — + the change is a no-op in dev mode). +2. After upstream merge, in quarto-cli: + `quarto dev-call pull-git-subtree julia-engine` (adds two commits; never + edit or rebase them — `.claude/rules/extension-subtrees.md`). +3. Remove the `QUARTO_TEST_BIN` gate (and its TODO comment) from + `.github/actions/merge-extension-tests/action.yml`. +4. Add the strip-list drift unit test + (`tests/unit/strip-env-sync.test.ts`, see above) in the same quarto-cli + PR that pulls the subtree. +5. Verify: dispatch `test-smokes-built.yml` with `source=build`; the smoke + leg must now run `smoke/julia-engine/*` and pass. Also confirm one dev + shard still runs them (no regression from the helper in dev mode). + +## Acceptance criteria + +- Binary-mode smoke leg runs the julia-engine tests green (no + `checkReconfiguration` crash, no dev-tree resource use — spot-check the + child's reported share path if in doubt). +- The strip-list drift unit test is in place and fails when the two lists + diverge. +- Dev shards unchanged. +- Gate and TODO removed; this doc updated to Status: done (or deleted, with + the sync rule moved to `llm-docs/built-version-testing-architecture.md`). diff --git a/dev-docs/debugging-flaky-tests.md b/dev-docs/debugging-flaky-tests.md index 67e30dc33e4..2534b0f08bd 100644 --- a/dev-docs/debugging-flaky-tests.md +++ b/dev-docs/debugging-flaky-tests.md @@ -184,8 +184,17 @@ quarto install tinytex for test in test1.ts test2.ts test3.ts; do ./run-tests.sh $test || break done + +# Run against a built quarto instead of the in-process dev sources +# (binary mode; see llm-docs/built-version-testing-architecture.md) +QUARTO_TEST_BIN=/path/to/installed/quarto ./run-tests.sh path/to/test.ts ``` +By default the harness invokes quarto in-process; with `QUARTO_TEST_BIN` set, +each invocation is spawned as a subprocess of a built distribution. A flake +that disappears in binary mode points at in-process state pollution in the +dev harness rather than a product bug. + ### Package/Dependency Comparison ```bash diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md new file mode 100644 index 00000000000..49d8f1e821a --- /dev/null +++ b/llm-docs/built-version-testing-architecture.md @@ -0,0 +1,337 @@ +--- +main_commit: 2e6695811 +analyzed_date: 2026-07-20 +key_files: + - tests/quarto-cmd.ts + - tests/test.ts + - tests/run-tests.sh + - tests/run-tests.ps1 + - tests/integration/playwright-tests.test.ts + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-built.yml + - .github/workflows/test-ff-matrix.yml + - .github/workflows/create-release.yml + - .github/actions/build-dist-tarball/action.yml +--- + +# Built-Version Testing Architecture + +How the smoke test suite runs against a **built** quarto distribution +(binary mode) in addition to the dev source tree, and — most importantly — +**why the design is the way it is**. This is the decision record; consult it +before changing the harness seam or the CI wiring, so settled trade-offs are +not accidentally relitigated. + +Document map: + +- **This doc** — architecture summary + design decisions (the durable "why"). +- `tests/README.md` → "Binary mode" and "How tests run (diagrams)" — the + operational "how" (local recipe, authoring rules, four mermaid diagrams). +- `llm-docs/testing-patterns.md` → "Dev Mode vs Binary Mode" — authoring + patterns for tests that must work in both modes. + +## Architecture in one paragraph + +Every `testQuartoCmd()`-based test invokes quarto through a single dispatch +point, `runQuarto()` in `tests/quarto-cmd.ts`. In **dev mode** (default) it +calls the in-process `quarto()` entry point from `src/quarto.ts`, exactly as +the harness always has. In **binary mode** (`QUARTO_TEST_BIN` set to a built +quarto extracted *outside* the checkout) it spawns that binary as a +subprocess with `--log --log-format json-stream`, merges the child's +log into the test's log file, and the verifiers run unchanged — they only +ever see log records and rendered outputs. CI-side, the reusable +`test-smokes.yml` gained `quarto-install: dev | release | artifact` inputs +(dev callers are untouched), and `test-smokes-built.yml` orchestrates three +sources for the binary under test: `build` (build from this ref, dispatch +only), `nightly` (reuse the signed artifacts of a nightly `create-release` +build — fires automatically via `workflow_run` after each one), and +`release` (install a published (pre-)release at its tag) — fanning each +source out to three test legs: smoke, playwright, and the feature-format +matrix (see "Built-mode test legs"). + +## When to use which mode + +| Mode | Quarto under test | Trigger | Suites (legs) | Question answered | +|---|---|---|---|---| +| dev (`test-smokes.yml`) | in-process TS sources (99.9.9) | every PR/push + daily cron | everything (sharded per-commit; ff-matrix via its own cron/push/PR) | did this code change break behavior? | +| nightly | signed nightly artifacts (Linux tarball, signed `quarto.exe`, notarized Mac zip) | automatic, after each nightly build | smoke (linux+windows+mac) + playwright (linux+mac) + ff-matrix (linux+windows) | does what we *ship* work? (bundling/packaging/launcher bugs; only macOS smoke coverage in CI) | +| build | fresh linux-amd64 dist from the current ref (unsigned) | manual dispatch | smoke + playwright + ff-matrix (all linux) | will *this branch* survive packaging? (works on forks/PR branches) | +| release | published (pre-)release via quarto-actions/setup, harness at its `v` tag | manual dispatch | smoke (linux+windows) + playwright (linux) + ff-matrix (linux+windows) | is the version users download healthy? (curative, post-publish) | + +Dev mode and built modes are complementary, not redundant: dev uniquely +covers `unit/`, `QUARTO_DEBUG` paths, the `quarto check` dev branch, and +in-process races; built modes cover the packaged product dev mode never +executes. The playwright suite (`integration/playwright-tests.test.ts`) is +no longer dev-only either: every built-mode source runs three suites +("legs") — smoke, playwright, and the feature-format matrix — see +"Built-mode test legs" below. The two other `tests/integration/` tests +(`guess-chunk-options-format-document.test.ts`, +`mermaid/github-issue-1340.test.ts`) still run only in the dev shards. + +In practice: + +- **Mostly, trigger nothing.** `nightly` fires itself after each nightly + build and is the preventive workhorse: it tests the exact signed + binaries the release pipeline produces, on all three OSes, *before* + anything is published. Red here means we caught it before users did. +- **Dispatch `build`** when a branch touches packaging or the harness + itself (`prepare-dist`, `configure`, `tests/quarto-cmd.ts`, ...) and you + want built-version feedback on *that ref* now. Trades coverage + (linux-only, unsigned) for immediacy and fork-friendliness. To get + *signed* Windows binaries for a branch instead, dispatch + `create-release` with `publish-release=false` + + `smoke-artifacts-only=true` — the `workflow_run` trigger then tests the + build automatically (only the legs whose artifacts exist); a manual + `source=nightly` dispatch with a `run-id` is just for re-testing an + older run (see D7). +- **Dispatch `release`** after publishing, as post-publish verification — + e.g. the optional step in + `dev-docs/checklist-make-a-new-quarto-prerelease.md`. Curative by + nature: it can only detect a broken published version, never prevent + one. Only works for releases cut after the harness support merged (D10). + +## Built-mode test legs (scheduler layout) + +`test-smokes-built.yml` = the mode **resolvers** (build-artifact / +resolve-nightly / resolve-release, unchanged) + a **scheduler**: per-leg +caller jobs fanning out to the reusable workflows. Three legs per source +mode, each an independent job (one red suite never cancels the others): + +| leg | goes through | bucket | OS scope | +|---|---|---|---| +| smoke | `test-smokes.yml` | `inputs.buckets` (empty = binary-mode `smoke/` default) | build: linux; nightly: linux+windows+mac (`has-*` gated); release: linux+windows | +| playwright | `test-smokes.yml` | `["integration/playwright-tests.test.ts"]` | linux (+ mac on nightly) — **never windows**, see below | +| ff-matrix | `test-ff-matrix.yml` (reusable) | owned by `test-ff-matrix.yml` | linux (+ windows on nightly/release) — no macOS (Julia/TeX toolchain unproven there) | + +Key points: + +- **The smoke leg doubles as the general bucket runner.** A manual dispatch + with the `buckets` input set runs only the smoke-slot jobs with that + bucket; the playwright + ff-matrix legs carry + `github.event.inputs.buckets == ''` in their `if:` and skip. +- **No windows playwright leg, deliberately.** The browser assertions are + hard-ignored on Windows CI (`playwright-tests.test.ts` + `ignore: gha.isGitHubActions() && isWindows`) — a windows leg would render + the corpus, skip every assertion, and report a misleading green. Rework + that gate (plus the `runner.os != 'Windows'` report-upload gate in + `test-smokes.yml`) before adding windows to the leg. +- **The playwright render wrapper needs the sanitized spawn env.** + `playwright-tests.test.ts` renders via `execProcess` + + `quartoDevCmd()` and MUST pass `quartoSpawnEnvOptions()`: without it the + built quarto inherits the dev-tree env (`QUARTO_SHARE_PATH`, ...) exported + by `run-tests.[sh|ps1]` for the harness and silently renders with + dev-tree resources (the D3/D4 dev-mode trap). +- **Playwright report artifacts are named per OS** + (`playwright-report-${{ runner.os }}`): several `test-smokes.yml` calls + share one workflow run in the fan-out, and duplicate artifact names make + `upload-artifact` fail even on green tests. +- **Per-leg OS scope is tuned in one place** — the `runners` inputs on the + scheduler jobs in `test-smokes-built.yml`. + +### `test-ff-matrix.yml` is reusable (`workflow_call`) + +The ff-matrix bucket glob +(`../dev-docs/feature-format-matrix/qmd-files/**/*.qmd`) is defined **only** +in `test-ff-matrix.yml`; built-mode callers reuse it through its +`workflow_call` trigger, which coexists with the dev triggers +(cron/push/PR/dispatch). Inputs `quarto-install`, `quarto-version`, +`quarto-artifact-name`, `quarto-artifact-run-id`, `ref`, `runners`, +`extra-r-packages` are forwarded verbatim to `test-smokes.yml`; the job uses +`${{ inputs.x || }}` fallbacks so the non-call triggers (where +`inputs.*` is empty) keep today's dev behavior. Nesting depth +`test-smokes-built.yml → test-ff-matrix.yml → test-smokes.yml` is 3, well +within GitHub's reusable-workflow nesting limit. CAUTION on +`test-ff-matrix.yml`'s top-level `concurrency`: a called workflow's +top-level concurrency evaluates in the CALLER's context +(`github.workflow`/`ref`/`run_id` are the caller run's), so the group +carries a per-call suffix derived from `inputs.runners` + `github.run_id` — +without it, every ff-matrix leg of one `test-smokes-built.yml` run would +share a single cancel-in-progress group and could cancel a sibling leg. +Dev triggers get a constant `-dev` suffix (dedup semantics unchanged). +`test-ff-matrix.yml` declares no `permissions`, so the caller's +`actions: write` (julia cache cleanup) flows through. + +## Design decisions + +Each entry: what was decided, why, and what would justify revisiting. + +### D1. Nightly wiring: `workflow_run`, not `workflow_call`/dispatch/inversion + +**Decision.** `test-smokes-built.yml` listens for completed "Build +Installers" (`create-release.yml`) runs via `workflow_run` and reuses their +artifacts cross-run (`quarto-artifact-run-id`). The release pipeline is not +modified for testing purposes. + +**Alternatives considered (2026-07, maintainer question):** + +- *create-release dispatches the test workflow at the end* — `workflow_run` + hand-rolled: needs `actions: write` + `gh workflow run` code inside the + release workflow, same default-branch constraint. Strictly dominated. +- *create-release `workflow_call`s `test-smokes.yml` after building* — + same-run artifacts (no run-id resolution) and an explicit DAG, but smoke + results would redden `Build Installers` runs including real publishes; + gating to schedule-only moves testing configuration (buckets, runner + policy) into the release workflow permanently. +- *Inversion: `test-smokes-built` owns the daily schedule and calls + `create-release` via `workflow_call`* — feasible (create-release uses the + `inputs.` context exclusively, which works under `workflow_call`; its + `github.event_name == 'schedule'` guard still behaves because a called + workflow sees the caller's event). Rejected because: (a) + `smoke-artifacts-only` skips `make-installer-mac`, so the daily run would + need the full build anyway — zero compute saved (there is no double + build today: one nightly build, one test pass reusing its artifacts); + (b) it reverses the dependency — the nightly build is also a + release-pipeline canary (signing certs, notarization, installer tooling) + and must not die when the test workflow is broken or paused; (c) nightly + builds would disappear from the "Build Installers" run history. + +**Why `workflow_run` wins:** zero risk to the most sensitive workflow in +the repo, and clean failure attribution — "Build Installers" red = the +pipeline broke; "Smoke Tests (Built Version)" red = the product broke. + +**Known weaknesses (accepted):** the trigger couples on the workflow +*display name* string (`workflows: ["Build Installers"]`; renaming +create-release's `name:` silently stops the trigger), and "trigger never +fired" is silent (mitigated by daily cadence — an absent run is visible). +Note the trigger fires on EVERY completed create-release run, not only +nightly schedules — manual dispatches (including partial +`smoke-artifacts-only` builds) get tested too, which is why each nightly +OS leg is gated on its artifact actually existing in the resolved run. + +**Revisit when:** the system has a green track record and maintainers want +one atomic nightly build-and-test signal — then the inversion is the +principled consolidation, done as a deliberate follow-up. + +### D2. Version marker: semver *build metadata* (`X.Y.Z+test.YYYYMMDD`) + +Built test dists are stamped `$(cat version.txt)+test.$(date +%Y%m%d)`. +Never a `-suffix` (prerelease versions fail every plain `>=X.Y` +`quarto-required` range — the vendored semver has no `includePrerelease` +anywhere) and never a 4th dot component (not semver; the vendored +`deno.land/x/semver@1.4.0` throws). Build metadata is range-transparent for +every gate while still distinguishable from the `99.9.9` dev sentinel. + +### D3. Dist outside the checkout + `99.9.9` sentinel refusal + +Installed launchers detect dev mode via a sibling `src/quarto.ts`: an +in-repo `package/dist/bin/quarto` silently runs the TS sources instead of +the built code. Therefore the dist under test must be extracted *outside* +the repo, and both `run-tests.[sh|ps1]` and `assertTestBinary()` refuse a +binary reporting `99.9.9` (`kLocalDevelopment`). CI extracts artifacts to +`RUNNER_TEMP`. + +### D4. Child env: inherit ambient + strip dev vars (not clearEnv+allowlist) + +Binary-mode spawns inherit the ambient environment minus a strip list +(`QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, `DENO_DIR`, `QUARTO_DEBUG`, +`QUARTO_FORCE_VERSION`, ...), with `TestContext.env` overlaid last. A +clearEnv+allowlist was rejected: the Windows system-variable surface +(`SystemRoot`, `PATHEXT`, ...) is unenumerable in practice. The dev-tree +exports in `run-tests.[sh|ps1]` are kept in all modes — the *harness* +process still needs them; only the *child* is sanitized. + +### D5. Silent-green guard: synthetic ERROR records + +"Non-zero exit ⇒ ERROR record in the log" is NOT an invariant +(pre-logger-init failures, `quarto add/remove` commandFailed, pandoc/typst +passthroughs), and ~23% of smoke-all docs are verified only via the log. So +`runQuarto()` appends a synthetic `{level: 40}` record (exit code + stderr +tail) when a non-zero exit leaves the child log record-free, and a timeout +record when the process-tree kill fires (`pgrep -P` walk on POSIX — +portable to macOS, unlike `ps --ppid`; `taskkill /T` on Windows). + +### D6. `QUARTO_TEST_BIN` is set at runtime, never declared statically + +The "Pin and verify test target" step in `test-smokes.yml` resolves the +installed binary, verifies it (sentinel refusal, semver shape, optional +`QUARTO_TEST_EXPECTED_VERSION` match), and exports it via `$GITHUB_ENV` so +every later step — including the unchanged `run-tests.sh` invocation — +sees it. With `quarto-install: dev` (all pre-existing callers) these steps +are skipped and nothing changes. + +### D7. `smoke-artifacts-only` is for cheap *branch* builds, not the daily path + +The `create-release.yml` input skips source/arm64 tarballs and the Mac +installer, yielding a fast signed Linux+Windows build for on-demand testing +of a branch: dispatch create-release with `publish-release=false` + +`smoke-artifacts-only=true` and the `workflow_run` trigger tests the build +automatically (mac leg skipped via the artifact-existence gate — one +dispatch total). Guards: `configure` fails fast if `publish-release` (which +defaults to true) is combined with `smoke-artifacts-only` — otherwise the +version commit+tag step would push an orphan tag — and such runs use a +per-run concurrency group so they never queue in the shared `prerelease` +group against a real release. It deliberately does NOT feed the daily +path: the daily needs the full build (Mac Zip = the only macOS smoke +coverage). + +### D8. macOS runners: scheduled/built runs only, never per-commit + +`test-smokes-parallel.yml` (per-commit) must stay fast, so it never passes +`runners` and keeps the `ubuntu-latest`/`windows-latest` default. The only +`macos-latest` smoke job is the nightly Mac leg in `test-smokes-built.yml`. +Encoded in the `runners` input description in `test-smokes.yml`. + +### D9. Built mode runs smoke + playwright + ff-matrix daily; the dev crons stay + +**Revised 2026-07-20** (originally: `integration/` stays dev-only with a +future dev daily job — that deferral is resolved the other way). + +Built mode is not smoke-only: the nightly path (and every other source +mode) runs three legs — smoke, playwright +(`integration/playwright-tests.test.ts`), and the feature-format matrix — +so packaging/launcher/signing regressions surface in browser behavior and +the full ff corpus too, not just the smoke suite (see "Built-mode test +legs"). Both suites were only ever excluded from binary mode by the +`run-tests.[sh|ps1]` default, never hard-blocked; the harness prerequisites +(the `quartoSpawnEnvOptions()` render-env fix, playwright provisioning in +non-dev CI modes) are in place. + +What stays dev-only: `unit/` (in-process by definition), the +non-playwright `integration/` tests +(`guess-chunk-options-format-document.test.ts`, +`mermaid/github-issue-1340.test.ts` — dev shards only), `QUARTO_DEBUG` +paths, the `quarto check` dev branch, and in-process races. Also +*temporarily* dev-only: the julia-engine subtree tests +(`smoke/julia-engine/`, copied in by `merge-extension-tests`) — their raw +`Deno.Command("quarto")` spawns inherit the harness dev env (the D3/D4 +trap: `QUARTO_DEBUG` crashes the built quarto in `checkReconfiguration`), +so the merge action skips them when `QUARTO_TEST_BIN` is set until the +spawns are sanitized upstream; plan in +`dev-docs/ci-julia-engine-binary-mode-followup.md`. Residual gaps +no suite exercises against the built quarto (not covered anywhere in CI +today, recorded so they read as known boundaries rather than oversights): +`quarto preview`/serve interactive paths (the playwright render glob +excludes `docs/playwright/(serve|shiny)`), `quarto publish` flows +(credentials), the actual installer packages (`.deb`/`.msi`/`.pkg` — the +legs test the tarball/zip layouts, never install-time behavior like PATH or +registry), the linux-arm64 tarball, and playwright visual snapshots +(`--ignore-snapshots`). Windows browser behavior and macOS ff-matrix are +also uncovered but deliberate, with revisit conditions in "Built-mode test +legs". The daily dev +crons also stay — dev ff-matrix catches source regressions, built +ff-matrix catches packaging regressions; complementary, not redundant. +Nothing is throttled initially (all legs daily): per-leg OS scope lives in +the scheduler jobs as the tuning knob once real CI spend is observed. Known +cost blind spot, accepted: the `workflow_run` trigger fires on EVERY +completed create-release run (D1), so manual builds also get the full +fan-out; gate the heavy legs on +`github.event.workflow_run.event == 'schedule'` if that ever needs +trimming. + +### D10. Release mode only works for post-harness tags + +Release mode checks out the tag, so the harness at that tag must already +contain `tests/quarto-cmd.ts` — a preflight fails clearly for older +releases. True backfill (main-branch harness testing an older binary) +would require harness/binary decoupling (plan §6 Phase 4, not implemented). + +The same skew applies per-suite: nightly/release legs run the harness at +the *target* ref, so a ref that has `tests/quarto-cmd.ts` but predates the +`quartoSpawnEnvOptions()` render fix in `playwright-tests.test.ts` +(2026-07-20) runs the old env-leaking wrapper — its playwright leg renders +with dev-tree resources and its result (green or red) is not meaningful. +The existence preflight cannot detect this. Affected window: releases and +nightly build shas cut between the harness-support merge and the multi-leg +merge, including the first post-merge `workflow_run` firings on pre-merge +build commits. The smoke and ff-matrix legs are unaffected (their spawns go +through `runQuarto`, whose env sanitization is as old as `quarto-cmd.ts`). diff --git a/llm-docs/testing-patterns.md b/llm-docs/testing-patterns.md index fd36ab1ff6e..506cf14ef14 100644 --- a/llm-docs/testing-patterns.md +++ b/llm-docs/testing-patterns.md @@ -1,3 +1,13 @@ +--- +main_commit: 2e6695811 +analyzed_date: 2026-07-20 +key_files: + - tests/test.ts + - tests/quarto-cmd.ts + - tests/verify.ts + - tests/utils.ts +--- + # Quarto Test Patterns This document describes the standard patterns for writing smoke tests in the Quarto CLI test suite. @@ -7,9 +17,44 @@ This document describes the standard patterns for writing smoke tests in the Qua Quarto uses Deno for testing with custom verification helpers located in: - `tests/test.ts` - Core test runner (`testQuartoCmd`) +- `tests/quarto-cmd.ts` - Quarto invocation dispatch (`runQuarto`: in-process dev quarto vs built binary) - `tests/verify.ts` - Verification helpers (`fileExists`, `pathDoNotExists`, etc.) - `tests/utils.ts` - Utility functions (`docs()`, `outputForInput()`, etc.) +### Dev Mode vs Binary Mode + +`testQuartoCmd` does not call quarto directly — it goes through `runQuarto()` +in `tests/quarto-cmd.ts`, the single dispatch point for invoking the quarto +under test: + +- **Dev mode (default):** quarto runs in-process via the `quarto()` entry + point imported from `src/quarto.ts`, as the harness always has. +- **Binary mode:** when `QUARTO_TEST_BIN` points at an installed quarto (a + built distribution extracted *outside* this checkout), quarto is spawned as + a subprocess with `--log --log-format json-stream`, so the log-record + verifiers work unchanged. Dev-tree env vars (`QUARTO_SHARE_PATH`, + `QUARTO_DEBUG`, `DENO_DIR`, ...) are stripped from the child. + `run-tests.sh`/`.ps1` refuse a binary reporting the `99.9.9` dev sentinel + and default the selection to `smoke/` (`unit/` is dev-only; the playwright + suite and ff-matrix corpus are binary-compatible and run when passed + explicitly). Exercised by `.github/workflows/test-smokes-built.yml`, + which runs smoke + playwright + ff-matrix legs per source mode. + Architecture and design decisions: + `llm-docs/built-version-testing-architecture.md`. + +Consequences for writing smoke tests: + +- Do **not** import `src/quarto.ts` (or call `quarto()`) directly from + `tests/smoke/` — route invocations through `testQuartoCmd`/`runQuarto` so + the test works in both modes. +- Tests that spawn a quarto subprocess themselves should resolve the + executable via `quartoDevCmd()` (`tests/utils.ts`, honors + `QUARTO_TEST_BIN`) and pass `quartoSpawnEnvOptions()` from + `tests/quarto-cmd.ts` as spawn env options. +- A test that genuinely exercises quarto internals in-process can set + `TestContext.requiresDevQuarto: true`; it is ignored in binary mode. Use + sparingly — most such code belongs in `tests/unit/` instead. + ## Common Test Patterns ### Simple Render Tests @@ -110,7 +155,7 @@ testQuartoCmd("render", [projectDir], [noErrors /*, ... */], { **Key points:** - The budget is machine-dependent (post-fix render time must sit well under it, pre-fix hang well over it), so it is defense-in-depth. Pair it with a deterministic unit test on the actual fix mechanism as the primary guard. -- A timed-out render subprocess is not killed by the harness, so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. +- In dev (in-process) mode a timed-out render is not killed by the harness (the timeout only rejects), so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. In binary mode (`QUARTO_TEST_BIN`) the spawned process tree *is* killed on timeout, but the kill is best-effort — keep the same defensive teardown. ### Extension Template Tests @@ -382,16 +427,19 @@ Rscript -e "renv::install(); renv::snapshot()" | `./run-tests.sh` (default) | **Race condition** | Files run in parallel, share `Deno.env` | | `./run-parallel-tests.sh` | **None** | Separate OS processes | -**Existing bad pattern** - `tests/smoke/website/drafts-env.test.ts`: +**Preferred channel:** pass per-test env via `TestContext.env` — it reaches +the in-process `quarto()` call in dev mode and the spawned binary in binary +mode (`QUARTO_TEST_BIN`), without mutating process-global state. -```typescript -// BAD: Sets env var, never restores it -// Only "works" because no other test reads QUARTO_PROFILE -Deno.env.set("QUARTO_PROFILE", "drafts"); -testQuartoCmd("render", [renderDir], [...]); -``` +**Known justified exception** - `tests/smoke/website/drafts-env.test.ts` +still sets `QUARTO_PROFILE` at module load *in addition to* `context.env`: +`src/project/project-profile.ts` caches the base profile from the env on the +first render in the process (`baseQuartoProfile`), so in dev (in-process) +mode a per-render env override is ignored whenever another test rendered +first. The module-load set runs before any test and keeps the cache correct; +the `context.env` copy is what the spawned binary sees in binary mode. -**Alternatives:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. +**Alternatives for new tests:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. ## Testing File Exclusion diff --git a/tests/README.md b/tests/README.md index ef401eddaad..1035d2ce279 100644 --- a/tests/README.md +++ b/tests/README.md @@ -12,7 +12,7 @@ Tests are run in our CI workflow on GHA at each commit, and for each PR. ## How the tests are created and organized ? -Tests are running through `Deno.test()` framework, adapted for our Quarto project and all written in Typescript. Infrastructure are in `tests.ts`, `tests.deps.ts` `verify.ts` and `utils.ts` which contains the helper functions that can be used. +Tests are running through `Deno.test()` framework, adapted for our Quarto project and all written in Typescript. Infrastructure are in `test.ts`, `test-deps.ts`, `quarto-cmd.ts`, `verify.ts` and `utils.ts` which contains the helper functions that can be used. - `unit/` and `integration/`, `smoke/`contain some `.ts` script representing each tests. - `docs/` is a special folder containing of the necessary files and projects used for the tests. @@ -430,6 +430,162 @@ Don't do ./run-tests.sh smoke/extensions/extension-render-doc.test.ts smoke/smoke-all.test.ts -- ./docs/smoke-all/2023/01/04/issue-3847.qmd ``` +### Binary mode (`QUARTO_TEST_BIN`) + +By default, tests run quarto **in-process** from the dev sources: `runQuarto()` in `tests/quarto-cmd.ts` calls the `quarto()` entry point imported from `src/quarto.ts`. When the `QUARTO_TEST_BIN` environment variable points at an installed quarto, `runQuarto()` instead spawns that binary as a subprocess, with `--log --log-format json-stream` so the log-based verifiers keep working unchanged. This is used to run the test suites (smoke, playwright, ff-matrix legs) against a built distribution (see `llm-docs/built-version-testing-architecture.md` for the architecture and design decisions, and the `test-smokes-built.yml` CI workflow below). + +To run in binary mode locally: + +```bash +# 1. Build a distribution (after ./configure.sh) +cd package/src +./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" +cd ../.. + +# 2. Copy the built dist OUTSIDE the git checkout. An in-repo quarto +# (e.g. package/dist/bin/quarto) resolves to dev mode — the launcher runs +# the TS sources when it finds a sibling src/quarto.ts — and must NOT be +# used: run-tests.[sh|ps1] refuses a binary reporting the 99.9.9 dev +# version sentinel. +cp -r package/pkg-working ~/quarto-under-test + +# 3. Run the tests against it +cd tests +QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh +``` + +In binary mode: + +- With no test arguments, `run-tests.[sh|ps1]` defaults to `smoke/` only. `unit/` exercises quarto internals in-process and is dev-only by definition; `integration/playwright-tests.test.ts` and the feature-format matrix ARE binary-compatible but need extra toolchain (playwright browsers, the full R/Python/Julia/TeX corpus deps), so they run only when passed explicitly — in CI they are their own legs in `test-smokes-built.yml` (smoke + playwright + ff-matrix, daily against the nightly build): + + ```bash + # playwright suite against a built quarto + QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh integration/playwright-tests.test.ts + # feature-format matrix against a built quarto + QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" + ``` + +- The test environment is configured as usual; set `QUARTO_TESTS_NO_CONFIG` to skip that step as in dev mode. +- Tests with `requiresDevQuarto: true` in their `TestContext` are ignored (rare escape hatch for tests that must exercise quarto internals in-process). + +Authoring rules that keep tests working in both modes: + +- Never `import { quarto } from "../src/quarto.ts"` in tests — invoke quarto through `testQuartoCmd()` (`tests/test.ts`) or `runQuarto()` (`tests/quarto-cmd.ts`). +- Tests that spawn quarto as a subprocess themselves should resolve the executable with `quartoDevCmd()` (`tests/utils.ts`, honors `QUARTO_TEST_BIN`) — or `quartoDevBinCmd()` (`tests/quarto-cmd.ts`) when the test must pin the locally-built dev CLI — and pass `quartoSpawnEnvOptions()` as spawn env options so the dev-tree env vars don't leak into the built quarto. + +## How tests run (diagrams) + +### Test invocation: one seam, two modes + +Every `testQuartoCmd()`-based test goes through a single dispatch point, +`runQuarto()` in `tests/quarto-cmd.ts`. The verifiers never know which mode +ran — they only read the json-stream log file and the rendered outputs. + +```mermaid +flowchart TB + subgraph deno ["Deno test process (tests/ harness, always runs from the repo checkout)"] + TQC["testQuartoCmd / testRender / testSite / smoke-all driver"] + RQ{"runQuarto()
tests/quarto-cmd.ts"} + LOG[("json-stream log file
{msg, level, levelName} per line")] + OUT[("rendered output files")] + VER["verifiers (tests/verify.ts)
noErrors, printsMessage, ensureHtmlElements, ..."] + end + DEV["in-process quarto()
imported from src/quarto.ts
dev TS sources, version 99.9.9"] + BIN["spawned subprocess: built quarto
--log file --log-format json-stream
dev env vars stripped (QUARTO_SHARE_PATH, DENO_DIR, ...)"] + + TQC --> RQ + RQ -->|"dev mode (default:
QUARTO_TEST_BIN unset)"| DEV + RQ -->|"binary mode
(QUARTO_TEST_BIN set)"| BIN + DEV --> LOG + DEV --> OUT + BIN --> LOG + BIN --> OUT + LOG --> VER + OUT --> VER +``` + +### Binary mode: lifecycle of one test + +```mermaid +sequenceDiagram + participant T as test() (tests/test.ts) + participant R as runQuarto() + participant Q as built quarto (subprocess) + participant V as verifiers + + T->>T: create temp json-stream log file + Note over T: binary mode: harness logger NOT initialized
(the child owns log capture) + T->>R: execute(logFile) + R->>Q: spawn QUARTO_TEST_BIN render ...
--log (per-invocation temp) --log-format json-stream
env = ambient minus dev-tree vars, plus TestContext.env + Q->>Q: render, write log records + output files + Q-->>R: exit (code, stdout/stderr drained) + R->>T: merge child log into the test log file + alt exit != 0 and no ERROR record in child log + R->>T: append synthetic ERROR record
(exit code + stderr tail) — prevents silent green + end + alt timeout + R->>Q: kill process tree (launcher spawns deno and waits) + R->>T: append timeout ERROR record + end + T->>V: verify(log records) + verify(output files) +``` + +### CI: which workflow tests what + +```mermaid +flowchart LR + subgraph dev ["Dev mode (unchanged): quarto = in-process TS sources"] + PR["PR / push"] --> TSP["test-smokes-parallel.yml
sharded buckets"] + DAILY["daily schedule"] --> TSfull["full run"] + end + subgraph built ["Binary mode: quarto = built distribution (QUARTO_TEST_BIN)"] + TSB["test-smokes-built.yml
runs after every nightly build
(workflow_run) + manual dispatch
3 legs per source mode:
smoke + playwright + ff-matrix"] + BUILDM["source: build (dispatch default)
build linux-amd64 dist from this ref
(any ref, works on forks)"] + NIGHTM["source: nightly (daily via workflow_run)
reuse SIGNED artifacts of a create-release run:
linux + windows quarto.exe + macOS
(the only macOS smoke coverage)"] + RELM["source: release
install published (pre-)release,
checkout its v-tag"] + TSB -->|"dispatch"| BUILDM + TSB -->|"after each nightly build
+ dispatch"| NIGHTM + TSB -->|"dispatch"| RELM + end + CR["create-release.yml
nightly build (no publish),
dispatch = publish; smoke-artifacts-only
input = cheap signed branch builds"] + ACT[".github/actions/build-dist-tarball
(shared build recipe)"] + + FFM["test-ff-matrix.yml (reusable)
dev triggers: cron / push / PR / dispatch
owns the ff-matrix qmd bucket"] + TS["test-smokes.yml (reusable)
inputs: quarto-install, ref, runners,
buckets, quarto-artifact-*"] + TSP --> TS + DAILY --> TS + FFM --> TS + BUILDM -->|"smoke + playwright legs"| TS + NIGHTM -->|"smoke + playwright legs"| TS + RELM -->|"smoke + playwright legs"| TS + BUILDM -->|"ff-matrix leg"| FFM + NIGHTM -->|"ff-matrix leg"| FFM + RELM -->|"ff-matrix leg"| FFM + BUILDM -. uses .-> ACT + CR -. "make-tarball jobs use" .-> ACT + NIGHTM -. "downloads artifacts from" .-> CR +``` + +### Binary mode in CI: how `QUARTO_TEST_BIN` reaches the tests + +`QUARTO_TEST_BIN` is never declared statically — the "Pin and verify test +target" step in `test-smokes.yml` computes it at runtime and exports it via +`$GITHUB_ENV`, making it visible to every later step of the job. When +`quarto-install` is `dev` (all existing callers), these steps are skipped +and nothing changes. + +```mermaid +flowchart TB + IN["workflow input
quarto-install: artifact | release"] + QD["quarto-dev action (ALWAYS runs)
provisions the harness Deno runtime"] + INST["install quarto under test
artifact: extract to RUNNER_TEMP (outside checkout)
release: quarto-actions/setup"] + PIN["Pin and verify test target
refuse 99.9.9 dev sentinel; check semver shape;
echo QUARTO_TEST_BIN=path >> GITHUB_ENV"] + RTS["./run-tests.sh (unchanged invocation)
sees QUARTO_TEST_BIN: banner + guard,
default selection = smoke/ only"] + QC["tests/quarto-cmd.ts
Deno.env.get('QUARTO_TEST_BIN')
every runQuarto spawns the built binary"] + + IN --> QD --> INST --> PIN --> RTS --> QC +``` + ## Debugging within tests `.vscode/launch.json` has a `Run Quarto test` configuration that can be used to debug when running tests. One need to modify the `program` and `args` fields to match the test to run. @@ -519,3 +675,13 @@ Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tes - `test-smokes.yml` is the main CI workflow which configure the environment, and run the tests on Ubuntu and Windows. - If it was triggerred by `workflow_call`, then it will run each test in using `run-tests.[sh|ps1]` in a for-loop. - Scheduled tests are still run daily in their sequential version. + - It is parameterized (`quarto-install`, `quarto-version`, `quarto-artifact-name`, `ref`, `runners`, ...) so callers can run the suite against a built quarto instead of the dev source tree: the workflow installs the quarto under test outside the checkout and exports `QUARTO_TEST_BIN` (see "Binary mode" above). +- `test-smokes-built.yml` runs the test suites against a **built** quarto (daily, via `workflow_run` after each nightly `create-release.yml` build, plus `workflow_dispatch`). Per source mode it fans out to three independent legs: **smoke** (`test-smokes.yml`; also the general bucket runner when the `buckets` dispatch input is set — the other legs then skip), **playwright** (`test-smokes.yml` with the `integration/playwright-tests.test.ts` bucket; linux + macOS only — browser assertions are ignored on Windows CI), and **ff-matrix** (the reusable `test-ff-matrix.yml`, which owns the feature-format bucket glob; linux + windows). Which mode to trigger when: + + | Mode | Trigger | Use it to answer | + |---|---|---| + | `nightly` | automatic (after each completed create-release run, scheduled or dispatched); dispatchable with a `run-id` to re-test an older run | does what we ship work? Signed artifacts from a create-release run — Linux, Windows (`quarto.exe`), macOS (the only macOS smoke coverage in CI); each OS leg runs only if its artifact exists in the run | + | `build` | dispatch (default) | will *this ref* survive packaging? Builds a linux-amd64 dist from the checkout via the shared `.github/actions/build-dist-tarball` action (also used by `create-release.yml`); the only mode that works on forks/PR branches | + | `release` | dispatch | is the *published* (pre-)release healthy? Post-publish verification; only works for releases whose tag contains the binary-mode harness | + + Full rationale and design decisions: `llm-docs/built-version-testing-architecture.md`. diff --git a/tests/docs/convert/issue-12318.qmd b/tests/docs/convert/issue-12318.qmd new file mode 100644 index 00000000000..8859466ef00 --- /dev/null +++ b/tests/docs/convert/issue-12318.qmd @@ -0,0 +1,28 @@ +--- +title: Quarto Crossrefs +format: html +jupyter: python3 +categories: [plot, python, equation] +--- + +## Overview + +See @fig-simple in @sec-plot for a demonstration of a simple plot. + +See @eq-stddev to better understand standard deviation. + +## Plot {#sec-plot} + +```{python} +#| label: fig-simple +#| fig-cap: "Simple Plot" +import matplotlib.pyplot as plt +plt.plot([1,23,2,4]) +plt.show() +``` + +## Equation {#sec-equation} + +$$ +s = \sqrt{\frac{1}{N-1} \sum_{i=1}^N (x_i - \overline{x})^2} +$$ {#eq-stddev} diff --git a/tests/docs/crossrefs/editor-support/all.qmd b/tests/docs/crossrefs/editor-support/all.qmd new file mode 100644 index 00000000000..b027c14e316 --- /dev/null +++ b/tests/docs/crossrefs/editor-support/all.qmd @@ -0,0 +1,26 @@ +--- +title: All crossref types +--- + +## Introduction {#sec-intro} + +See @sec-intro for the overview, @fig-plot for the figure, @tbl-data for the +table, @eq-var for the equation, and @thm-line for the theorem. + +![A simple plot.](plot.png){#fig-plot} + +| Column A | Column B | +|----------|----------| +| 1 | 2 | + +: A small table. {#tbl-data} + +$$ +\sigma^2 = \frac{1}{N} \sum_{i=1}^N (x_i - \mu)^2 +$$ {#eq-var} + +::: {#thm-line} +## Line + +The equation of a straight line is $y = mx + b$. +::: diff --git a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml index eb86e8b6bf2..1a6f1d67ea7 100644 --- a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml +++ b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml @@ -1,7 +1,7 @@ title: Test author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: project: project: diff --git a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml index 2d5882388ab..66571285084 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml index 2d5882388ab..66571285084 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml index 859925eab79..c95827710de 100644 --- a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My Brand author: Quarto version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml index 106a96726ca..4bc546fbf56 100644 --- a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml +++ b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml @@ -1,7 +1,7 @@ title: Typst-brand-typography-example author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml index f11d93da35b..bbc62e89a3c 100644 --- a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml +++ b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml @@ -1,7 +1,7 @@ title: Font Provider Extension author: Test version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: formats: typst: diff --git a/tests/integration/playwright-tests.test.ts b/tests/integration/playwright-tests.test.ts index 4318d9836c0..8feb6d9a31b 100644 --- a/tests/integration/playwright-tests.test.ts +++ b/tests/integration/playwright-tests.test.ts @@ -1,5 +1,5 @@ /* - * smoke-all.test.ts + * playwright-tests.test.ts * * Copyright (C) 2022 Posit Software, PBC * @@ -13,6 +13,7 @@ import { } from "../../src/core/lib/yaml-validation/state.ts"; import { cleanoutput } from "../smoke/render/render.ts"; import { execProcess } from "../../src/core/process.ts"; +import { quartoSpawnEnvOptions } from "../quarto-cmd.ts"; import { quartoDevCmd } from "../utils.ts"; import { fail } from "testing/asserts"; import { isWindows } from "../../src/deno_ral/platform.ts"; @@ -72,11 +73,16 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { // mediabag inspection if we don't wait all renders // individually. This is very slow.. console.log(`Rendering ${input}...`); + // quartoSpawnEnvOptions: in binary mode the built quarto must not + // inherit the dev-tree env (QUARTO_SHARE_PATH etc.) exported by + // run-tests.[sh|ps1] for the harness — it would silently render with + // dev-tree resources instead of the packaged ones const result = await execProcess({ cmd: quartoDevCmd(), args: ["render", input, ...options], stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); if (!result.success) { @@ -91,8 +97,11 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { } Deno.test({ - name: "Playwright tests are passing", - // currently we run playwright tests only on Linux + name: "Playwright tests are passing", + // browser assertions are skipped on Windows CI (the renders above still + // run); Linux and macOS run them. This gate is why built-version CI + // (test-smokes-built.yml) has no windows playwright leg - rework it AND + // the report-upload gate in test-smokes.yml before adding one. ignore: gha.isGitHubActions() && isWindows, fn: async () => { try { diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts new file mode 100644 index 00000000000..1915473e30c --- /dev/null +++ b/tests/quarto-cmd.ts @@ -0,0 +1,486 @@ +/* +* quarto-cmd.ts +* +* Single dispatch point for invoking the quarto under test. +* +* By default (dev mode) quarto runs in-process by calling the `quarto()` +* entry point imported from ../src/quarto.ts, exactly as the harness always +* has. When QUARTO_TEST_BIN points at an installed quarto (a built +* distribution extracted OUTSIDE this checkout — the launcher enters dev +* mode when it finds a sibling src/quarto.ts), quarto is spawned as a +* subprocess instead, with `--log --log-format json-stream` so the +* existing log-record verifiers keep working unchanged. +* +* See llm-docs/built-version-testing-architecture.md for the design and rationale. +* +* Copyright (C) 2020-2026 Posit Software, PBC +* +*/ +import { quarto } from "../src/quarto.ts"; +import { kLocalDevelopment } from "../src/core/quarto.ts"; +import { isWindows } from "../src/deno_ral/platform.ts"; +import { join } from "../src/deno_ral/path.ts"; + +// Env vars that identify the dev tree (or stale per-render state) and must +// never leak into the spawned binary: the installed launchers inherit +// QUARTO_SHARE_PATH / QUARTO_DEBUG / QUARTO_DENO / QUARTO_DENO_DOM when +// already set, and never reset DENO_DIR outside dev mode. Logging vars are +// stripped because the seam owns logging via explicit CLI flags. Everything +// else in the ambient environment (PATH, HOME, toolchain locations like +// QUARTO_PYTHON/QUARTO_R, platform system vars) is inherited; per-test env +// is overlaid last so tests can deliberately set any of these. +const kStripEnvVars = [ + "QUARTO_SHARE_PATH", + "QUARTO_BIN_PATH", + "QUARTO_DEBUG", + "DENO_DIR", + "QUARTO_DENO", + "QUARTO_DENO_DOM", + "QUARTO_ROOT", + "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION", + "QUARTO_VERSION_REQUIREMENT", + "QUARTO_PROJECT_DIR", + "QUARTO_PROFILE", + "QUARTO_LOG", + "QUARTO_LOG_LEVEL", + "QUARTO_LOG_FORMAT", + "RSTUDIO", +]; + +// json-stream ERROR record shape expected by readExecuteOutput/verify.ts +// (level 40 == std/log LogLevels.ERROR) +const kErrorLevel = 40; + +// Default per-invocation render timeout (dev and binary mode alike). +const kDefaultRenderTimeoutMs = 600000; + +// Path of the built quarto under test, when binary mode is active. +export function quartoTestBin(): string | undefined { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + return bin && bin.length > 0 ? bin : undefined; +} + +// True when tests target an external built quarto (QUARTO_TEST_BIN) instead +// of the in-process dev sources. +export function isBinaryMode(): boolean { + return quartoTestBin() !== undefined; +} + +// Path to the quarto executable for tests that historically pinned the +// locally-built dev CLI (package/dist/bin/quarto) rather than PATH quarto — +// keeps them testing the dev tree in local runs where PATH may hold a stale +// system quarto. QUARTO_BIN_PATH is exported by run-tests.sh/.ps1. +export function quartoDevBinCmd(): string { + const bin = quartoTestBin(); + if (bin) { + return bin; + } + const binPath = Deno.env.get("QUARTO_BIN_PATH") ?? + join("..", "package", "dist", "bin"); + return join(binPath, isWindows ? "quarto.cmd" : "quarto"); +} + +export function buildBinaryEnv( + overlay?: Record, +): Record { + const env = Deno.env.toObject(); + for (const name of kStripEnvVars) { + delete env[name]; + } + return { ...env, ...(overlay ?? {}) }; +} + +// Env/clearEnv spawn options for tests that launch the quarto under test +// themselves (via execProcess or Deno.Command with quartoDevCmd()) instead +// of going through runQuarto. In binary mode: full ambient env with the +// dev-tree vars stripped (kStripEnvVars) and the per-test overlay merged +// last, plus clearEnv so the sanitized env is authoritative — otherwise the +// installed launcher would inherit QUARTO_SHARE_PATH/QUARTO_DEBUG/DENO_DIR +// from the harness (run-tests.sh exports them) and silently use dev-tree +// resources. In dev mode: today's behavior exactly — the overlay (if any) +// merges into the inherited ambient env. +export function quartoSpawnEnvOptions( + overlay?: Record, +): { env?: Record; clearEnv?: boolean } { + if (isBinaryMode()) { + return { env: buildBinaryEnv(overlay), clearEnv: true }; + } + return overlay !== undefined ? { env: overlay } : {}; +} + +// Appends a synthetic ERROR record to a json-stream log file. Only call +// after the child process has exited (single-writer at that point). +export function appendLogError(logFile: string, msg: string) { + const record = JSON.stringify({ + msg, + level: kErrorLevel, + levelName: "ERROR", + }); + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // file may not exist yet + } + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + record + "\n"); +} + +// A timeout kill can interrupt the child mid-write, leaving a torn (partial) +// final JSON line. Drop a trailing unparseable line so the merged log stays +// valid json-stream — readExecuteOutput is deliberately strict (a parse throw +// is how the logging tests detect malformed output), so the torn line must be +// removed at the source rather than tolerated by every reader. +function stripTornTrailingLine(content: string): string { + const lines = content.split("\n"); + let i = lines.length - 1; + while (i >= 0 && lines[i] === "") { + i--; + } + if (i < 0) { + return content; + } + try { + JSON.parse(lines[i]); + return content; + } catch { + lines.splice(i, 1); + return lines.join("\n"); + } +} + +function hasErrorRecordText(content: string): boolean { + for (const line of content.split("\n")) { + if (!line) continue; + try { + const record = JSON.parse(line); + if ( + typeof record?.levelName === "string" && + record.levelName.toLowerCase() === "error" + ) { + return true; + } + } catch { + // tolerate partial/corrupt lines + } + } + return false; +} + +// QUARTO_TEST_BIN must point at an installed layout, not the dev tree: the +// installed launcher enters dev mode (runs the TS sources) whenever a +// sibling src/quarto.ts exists, and reports the 99.9.9 dev sentinel. Fail +// loudly rather than silently testing the wrong thing. +let checkedBinary: string | undefined; +export function assertTestBinary(bin: string) { + if (checkedBinary === bin) { + return; + } + // Probe with the sanitized env used for test spawns: the installed + // launcher keeps an inherited QUARTO_SHARE_PATH (exported by + // run-tests.[sh|ps1] for the harness), under which --version reads the + // dev tree's nonexistent src/resources/version and reports empty. + const result = new Deno.Command(bin, { + args: ["--version"], + stdout: "piped", + stderr: "piped", + env: buildBinaryEnv(), + clearEnv: true, + }).outputSync(); + const version = new TextDecoder().decode(result.stdout).trim(); + if (result.code !== 0) { + const stderr = new TextDecoder().decode(result.stderr).trim(); + throw new Error( + `QUARTO_TEST_BIN (${bin}) failed to report a version (exit ${result.code}):\n${stderr}`, + ); + } + if (version.length === 0) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reported an empty version. ` + + `The distribution is likely incomplete (missing share/version).`, + ); + } + if (version === kLocalDevelopment) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports the dev version sentinel ${kLocalDevelopment}. ` + + `It is resolving to a dev-mode quarto (the launcher runs the TS sources when a ` + + `sibling src/quarto.ts exists). Point QUARTO_TEST_BIN at a built distribution ` + + `extracted outside the git checkout.`, + ); + } + const expected = Deno.env.get("QUARTO_TEST_EXPECTED_VERSION"); + if (expected && version !== expected) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports version ${version}, expected ${expected} ` + + `(QUARTO_TEST_EXPECTED_VERSION).`, + ); + } + console.log(`[binary mode] testing quarto ${version} at ${bin}`); + checkedBinary = bin; +} + +// QUARTO_TEST_BIN is a launcher that spawns deno and waits (it does not +// exec), so killing the direct child would orphan the actual renderer — +// which also still holds the --log file at its own offset. Kill the whole +// tree, deepest first. (Best effort: children spawned between collection +// and kill can escape.) +async function killProcessTree(pid: number) { + if (isWindows) { + try { + await new Deno.Command("taskkill", { + args: ["/PID", String(pid), "/T", "/F"], + stdout: "null", + stderr: "null", + }).output(); + } catch { + // taskkill unavailable (should not happen on a real Windows runner) or + // the tree already exited. Fall back to killing the direct child so the + // awaited child.output() can resolve instead of blocking on a still-live + // launcher; this cannot reach an orphaned grandchild renderer, but is + // strictly better than killing nothing. + try { + Deno.kill(pid, "SIGKILL"); + } catch { + // already exited + } + } + return; + } + const pids: number[] = []; + const stack = [pid]; + while (stack.length > 0) { + const current = stack.pop()!; + pids.push(current); + try { + // pgrep -P is portable across Linux and macOS/BSD (unlike the + // procps-only `ps --ppid` long option) + const result = new Deno.Command("pgrep", { + args: ["-P", String(current)], + stdout: "piped", + stderr: "null", + }).outputSync(); + const children = new TextDecoder() + .decode(result.stdout) + .split("\n") + .map((line) => parseInt(line.trim(), 10)) + .filter((child) => !isNaN(child)); + stack.push(...children); + } catch { + // pgrep unavailable; fall back to killing what we have + } + } + for (const target of pids.reverse()) { + try { + Deno.kill(target, "SIGKILL"); + } catch { + // already exited + } + } +} + +export interface RunQuartoOptions { + // per-test environment overlay (TestContext.env) + env?: Record; + // working directory for the spawned binary (binary mode only; dev mode + // call sites manage cwd via Deno.chdir as they always have) + cwd?: string; + // json-stream log target; when set (binary mode), --log/--log-format/ + // --log-level flags are appended so verifiers can read the records + logFile?: string; + logLevel?: string; + logFormat?: string; + timeoutMs?: number; + // Binary mode only: throw when the child exits non-zero or times out. + // Defaults to TRUE so direct call sites (module-level project + // pre-renders, context.setup pre-renders) keep today's fail-loudly + // semantics. testQuartoCmd passes false and relies on the log records + // (including the synthetic one below) reaching the verifiers. + // In dev mode failures always propagate as exceptions, exactly as today. + throwOnFailure?: boolean; +} + +export interface RunQuartoResult { + code: number; + timedOut: boolean; + stderrTail?: string; +} + +// Dispatches to the in-process dev sources or, when QUARTO_TEST_BIN is set, +// the built binary. The two modes have distinct process/logging/timeout +// mechanics, so each lives in its own helper below. +export async function runQuarto( + args: string[], + options: RunQuartoOptions = {}, +): Promise { + const bin = quartoTestBin(); + return bin + ? runBinaryQuarto(bin, args, options) + : runDevQuarto(args, options); +} + +// dev mode: in-process call, preserving existing semantics exactly +// (a timeout rejects but does not kill the in-process render) +async function runDevQuarto( + args: string[], + options: RunQuartoOptions, +): Promise { + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; + let timer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); + }); + try { + await Promise.race([quarto(args, undefined, options.env), timeout]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } + return { code: 0, timedOut: false }; +} + +// binary mode: spawn the built quarto, enforce the timeout by killing the +// process tree, reconcile the child's log into the caller's, and apply the +// failure policy. +async function runBinaryQuarto( + bin: string, + args: string[], + options: RunQuartoOptions, +): Promise { + assertTestBinary(bin); + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; + const throwOnFailure = options.throwOnFailure ?? true; + + // The binary's LogFileHandler opens --log in truncate mode, so two + // invocations sharing one log file would erase the first invocation's + // records (including any synthetic ERROR). Give each child its own temp + // log and merge it into the caller's log file after exit. + const spawnArgs = [...args]; + let childLog: string | undefined; + if (options.logFile) { + childLog = Deno.makeTempFileSync({ suffix: ".json" }); + spawnArgs.push( + "--log", + childLog, + "--log-format", + options.logFormat ?? "json-stream", + // per-test log intent must land in the flags: explicit flags beat + // QUARTO_LOG_LEVEL env in quarto's logOptions, so passing the env + // var through would be silently ignored + "--log-level", + options.logLevel ?? options.env?.QUARTO_LOG_LEVEL ?? "info", + ); + } + + const child = new Deno.Command(bin, { + args: spawnArgs, + cwd: options.cwd ?? Deno.cwd(), + env: buildBinaryEnv(options.env), + clearEnv: true, + stdout: "piped", + stderr: "piped", + }).spawn(); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + // fire-and-forget: child.output() below resolves once the kill lands. + // Swallow any rejection so it never surfaces as an unhandled rejection. + killProcessTree(child.pid).catch(() => {}); + }, timeoutMs); + + // output() drains both streams (undrained pipes deadlock at the 64KiB + // buffer on verbose renders) and resolves once the process exits — + // including after a kill + const output = await child.output(); + clearTimeout(timer); + + const stderrText = new TextDecoder().decode(output.stderr); + const stderrTail = stderrText.split("\n").slice(-25).join("\n").trim(); + const commandLine = `quarto ${args.join(" ")}`; + + if (options.logFile && childLog) { + mergeChildLog(options.logFile, childLog, { + timedOut, + code: output.code, + timeoutMs, + commandLine, + stderrTail, + }); + } + + if ((output.code !== 0 || timedOut) && throwOnFailure) { + throw new Error( + timedOut + ? `${commandLine} timed out after ${timeoutMs}ms` + : `${commandLine} exited with code ${output.code}\nstderr (tail):\n${stderrTail}`, + ); + } + + return { code: output.code, timedOut, stderrTail }; +} + +// Merges a binary invocation's temp log into the caller's log file and +// synthesizes an ERROR record when the child failed without logging one, so +// log-reading verifiers can see the failure. Deletes the temp log. +function mergeChildLog( + logFile: string, + childLog: string, + outcome: { + timedOut: boolean; + code: number; + timeoutMs: number; + commandLine: string; + stderrTail: string; + }, +) { + let childContent = ""; + try { + childContent = Deno.readTextFileSync(childLog); + } catch { + // child never wrote the log (e.g. failed before logger init) + } + try { + Deno.removeSync(childLog); + } catch { + // best effort + } + // Only a timeout kill can tear a line; a clean exit flushes whole records. + if (outcome.timedOut) { + childContent = stripTornTrailingLine(childContent); + } + // Always ensure logFile exists, even when the child logged nothing on a + // successful run: test.ts treats a MISSING logTarget as a hard failure + // ("test log file is missing"). A quiet successful command must leave an + // empty log (which verifiers read as an empty record array), not no file. + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // log file may not exist yet + } + if (childContent.length > 0) { + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + childContent); + } else if (existing.length === 0) { + Deno.writeTextFileSync(logFile, ""); + } + if (outcome.timedOut) { + appendLogError( + logFile, + `${outcome.commandLine} timed out after ${outcome.timeoutMs}ms and was killed`, + ); + } else if (outcome.code !== 0 && !hasErrorRecordText(childContent)) { + // A child can exit non-zero without any ERROR record: failures + // before logger init (deno startup, bundle load, missing share) and + // commandFailed paths (quarto add/remove). Without this synthetic + // record, log-only verifiers (noErrorsOrWarnings — the default + // smoke-all spec) would pass vacuously against an empty log. + appendLogError( + logFile, + `${outcome.commandLine} exited with code ${outcome.code} without logging an error\n` + + `stderr (tail):\n${outcome.stderrTail}`, + ); + } +} diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index f0f3ef77f7c..ac0d51f0908 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -66,6 +66,54 @@ If ($null -eq $Env:QUARTO_DENO_DIR) { $Env:DENO_DIR = $Env:QUARTO_DENO_DIR } +# BINARY MODE: when QUARTO_TEST_BIN is set, tests run against a built quarto +# (an installed distribution extracted OUTSIDE this checkout). The harness +# itself still runs from the dev tree — the dev env vars above stay — and +# tests/quarto-cmd.ts dispatches quarto invocations to the binary, stripping +# the dev env from the child process. See +# llm-docs/built-version-testing-architecture.md. +If ($null -ne $Env:QUARTO_TEST_BIN) { + If (-not (Test-Path $Env:QUARTO_TEST_BIN)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) does not exist" + Exit 1 + } + # Probe with the dev-tree env stripped (same list as tests/quarto-cmd.ts): + # the installed launcher keeps an inherited QUARTO_SHARE_PATH, and the dev + # exports above would make a healthy built quarto read the dev tree's + # (nonexistent) src/resources/version and report an EMPTY version. + $probeStrip = @( + "QUARTO_SHARE_PATH", "QUARTO_BIN_PATH", "QUARTO_DEBUG", "DENO_DIR", + "QUARTO_DENO", "QUARTO_DENO_DOM", "QUARTO_ROOT", "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION" + ) + $probeSaved = @{} + ForEach ($name in $probeStrip) { + $probeSaved[$name] = [Environment]::GetEnvironmentVariable($name) + Remove-Item "Env:$name" -ErrorAction SilentlyContinue + } + Try { + $QUARTO_TEST_BIN_VERSION = & $Env:QUARTO_TEST_BIN --version + } Finally { + ForEach ($name in $probeStrip) { + If ($null -ne $probeSaved[$name]) { + [Environment]::SetEnvironmentVariable($name, $probeSaved[$name]) + } + } + } + If ([string]::IsNullOrWhiteSpace($QUARTO_TEST_BIN_VERSION)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) did not report a version." + Write-Host -ForegroundColor red "The distribution is likely incomplete (missing share/version)." + Exit 1 + } + If ($QUARTO_TEST_BIN_VERSION -eq "99.9.9") { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + Write-Host -ForegroundColor red "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." + Write-Host -ForegroundColor red "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + Exit 1 + } + Write-Host "> BINARY MODE: testing built quarto $QUARTO_TEST_BIN_VERSION at $($Env:QUARTO_TEST_BIN)" +} + # Preparing running Deno with default arguments $QUARTO_IMPORT_MAP_ARG="--importmap=$(Join-Path $QUARTO_SRC_DIR "import_map.json")" @@ -161,6 +209,16 @@ If ($customArgs[0] -notlike "*smoke-all.test.ts") { $TESTS_TO_RUN=$customArgs } +# Binary-mode default selection: smoke tests only. tests/unit/ exercises +# quarto internals in-process (dev-only by definition); +# tests/integration/playwright-tests.test.ts IS binary-compatible but needs +# the playwright toolchain, so it only runs when asked for explicitly (CI +# runs it as its own leg in test-smokes-built.yml). +If ($null -ne $Env:QUARTO_TEST_BIN -and $TESTS_TO_RUN.count -eq 0 -and $customArgs.count -eq 0) { + $TESTS_TO_RUN = @("smoke/") + Write-Host "> BINARY MODE: defaulting to smoke/ tests (pass a path explicitly to run others, e.g. integration/playwright-tests.test.ts)" +} + # ---- Running tests with Deno ------- $DENO_ARGS = @() diff --git a/tests/run-tests.sh b/tests/run-tests.sh index a2c134fd392..8d3777ee0ef 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -62,6 +62,41 @@ export QUARTO_DEBUG=true QUARTO_DENO_OPTIONS="--config test-conf.json --v8-flags=--enable-experimental-regexp-engine,--max-old-space-size=8192,--max-heap-size=8192 --unstable-kv --unstable-ffi --no-lock --allow-all" +# BINARY MODE: when QUARTO_TEST_BIN is set, tests run against a built quarto +# (an installed distribution extracted OUTSIDE this checkout). The harness +# itself still runs from the dev tree — the dev env exports above stay — and +# tests/quarto-cmd.ts dispatches quarto invocations to the binary, stripping +# the dev env from the child process. See +# llm-docs/built-version-testing-architecture.md. +if [[ -n "$QUARTO_TEST_BIN" ]]; then + if [[ ! -x "$QUARTO_TEST_BIN" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) does not exist or is not executable" + exit 1 + fi + # Probe with the dev-tree env stripped. This is the subset of + # tests/quarto-cmd.ts's kStripEnvVars that affects `--version` resolution + # (share/root/deno paths) — the logging/profile vars in the full list don't + # change --version output, so they're omitted here. The installed launcher + # keeps an inherited QUARTO_SHARE_PATH, and the dev exports above would make + # a healthy built quarto read the dev tree's (nonexistent) + # src/resources/version and report an EMPTY version. + QUARTO_TEST_BIN_VERSION="$(env -u QUARTO_SHARE_PATH -u QUARTO_BIN_PATH \ + -u QUARTO_DEBUG -u DENO_DIR -u QUARTO_DENO -u QUARTO_DENO_DOM \ + -u QUARTO_ROOT -u QUARTO_SRC_PATH -u QUARTO_FORCE_VERSION \ + "$QUARTO_TEST_BIN" --version 2>/dev/null)" + if [[ -z "$QUARTO_TEST_BIN_VERSION" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) did not report a version." + echo "The distribution is likely incomplete (missing share/version)." + exit 1 + fi + if [[ "$QUARTO_TEST_BIN_VERSION" == "99.9.9" ]]; then + echo "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + echo "It resolves to a dev-mode quarto: the launcher runs the TS sources whenever a sibling src/quarto.ts exists." + echo "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + exit 1 + fi + echo "> BINARY MODE: testing built quarto ${QUARTO_TEST_BIN_VERSION} at ${QUARTO_TEST_BIN}" +fi if [[ -z $GITHUB_ACTION ]] && [[ -z $QUARTO_TESTS_NO_CONFIG ]] then @@ -161,6 +196,15 @@ else TESTS_TO_RUN=("${SMOKE_ALL_TEST_FILE}" "--" "${SMOKE_ALL_FILES[@]}") fi fi + # Binary-mode default selection: smoke tests only. tests/unit/ exercises + # quarto internals in-process (dev-only by definition); + # tests/integration/playwright-tests.test.ts IS binary-compatible but needs + # the playwright toolchain, so it only runs when asked for explicitly (CI + # runs it as its own leg in test-smokes-built.yml). + if [[ -n "$QUARTO_TEST_BIN" && "${#TESTS_TO_RUN[@]}" -eq 0 && -z "$*" ]]; then + TESTS_TO_RUN=("smoke/") + echo "> BINARY MODE: defaulting to smoke/ tests (pass a path explicitly to run others, e.g. integration/playwright-tests.test.ts)" + fi # TESTS_TO_RUN is an array and quoted here on purpose: a bucket can be a # literal, unexpanded ** glob pattern (e.g. from the ff-matrix CI bucket), # and smoke-all.test.ts expands it itself via expandGlobSync. Expanding it diff --git a/tests/smoke/convert/issue-12318.test.ts b/tests/smoke/convert/issue-12318.test.ts index 13d95b9e06c..4d14adc85ed 100644 --- a/tests/smoke/convert/issue-12318.test.ts +++ b/tests/smoke/convert/issue-12318.test.ts @@ -11,32 +11,54 @@ import { test, } from "../../test.ts"; import { assert } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; (() => { const input = "docs/convert/issue-12318"; + const roundtrip = "issue-12318-2.qmd"; test({ // The name of the test name: "issue-12318", - + // Sets up the test context: { teardown: async () => { if (existsSync(input + '.ipynb')) { Deno.removeSync(input + '.ipynb'); } + if (existsSync(roundtrip)) { + Deno.removeSync(roundtrip); + } } }, - + // Executes the test - execute: async () => { - await quarto(["convert", "docs/convert/issue-12318.qmd"]); - await quarto(["convert", "docs/convert/issue-12318.ipynb", "--output", "issue-12318-2.qmd"]); - const txt = Deno.readTextFileSync("issue-12318-2.qmd"); - assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); + execute: async (logFile?: string) => { + await runQuarto(["convert", input + ".qmd"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["convert", input + ".ipynb", "--output", roundtrip], { + logFile, + throwOnFailure: false, + }); }, - - verify: [], + + // The harness catches execute() errors and turns them into log records, + // so assertions must live in verifiers (an empty verify list would pass + // silently on failure) - noErrors surfaces convert failures first. + verify: [ + noErrors, + { + name: "no triple backticks mid-line after roundtrip", + verify: (_outputs: ExecuteOutput[]) => { + const txt = Deno.readTextFileSync(roundtrip); + assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); + return Promise.resolve(); + }, + }, + ], type: "unit" }); })(); diff --git a/tests/smoke/create/create.test.ts b/tests/smoke/create/create.test.ts index dc685b9d492..cdec16994bc 100644 --- a/tests/smoke/create/create.test.ts +++ b/tests/smoke/create/create.test.ts @@ -11,6 +11,7 @@ import { walkSync } from "../../../src/deno_ral/fs.ts"; import { CreateResult } from "../../../src/command/create/cmd-types.ts"; import { assert } from "testing/asserts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; const kCreateTypes: Record = { "project": ["website", "default", "book", "website:blog"], @@ -54,6 +55,7 @@ for (const type of Object.keys(kCreateTypes)) { args: cmd.slice(1), stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }, stdIn); assert(process.success, process.stderr); if (process.stdout) { @@ -101,6 +103,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(buildProcess.success, buildProcess.stderr); } @@ -115,6 +118,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(process.success, process.stderr); } diff --git a/tests/smoke/crossref/syntax.test.ts b/tests/smoke/crossref/syntax.test.ts index 8d4f913d718..8217557d487 100644 --- a/tests/smoke/crossref/syntax.test.ts +++ b/tests/smoke/crossref/syntax.test.ts @@ -4,7 +4,7 @@ * Copyright (C) 2020-2022 Posit Software, PBC */ -import { ensureFileRegexMatches } from "../../verify.ts"; +import { ensureFileRegexMatches, noErrors } from "../../verify.ts"; import { testRender } from "../render/render.ts"; import { crossref } from "./utils.ts"; import { @@ -15,7 +15,8 @@ import { Verify, } from "../../test.ts"; import { assert, fail } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; +import { safeRemoveSync } from "../../../src/deno_ral/fs.ts"; const syntaxQmd = crossref("syntax.qmd", "html"); testRender(syntaxQmd.input, "html", false, [ @@ -58,22 +59,35 @@ const verify: Verify = { }; const context: TestContext = { teardown: () => { - Deno.removeSync(imgQmd.output.outputPath); - Deno.removeSync(imgQmd.output.supportPath, { recursive: true }); + // safeRemoveSync tolerates a missing path: when a render fails, noErrors + // fails first and these outputs never exist. Raw Deno.removeSync would then + // throw NotFound from this finally-phase teardown and mask the noErrors + // report with an unhelpful ENOENT. + safeRemoveSync(imgQmd.output.outputPath); + safeRemoveSync(imgQmd.output.supportPath, { recursive: true }); - Deno.removeSync(divQmd.output.outputPath); - Deno.removeSync(divQmd.output.supportPath, { recursive: true }); + safeRemoveSync(divQmd.output.outputPath); + safeRemoveSync(divQmd.output.supportPath, { recursive: true }); return Promise.resolve(); }, }; const testDesc: TestDescriptor = { // FIXME: why is this test flaky now? Ask @dragonstyle name: "test html produced by different figure syntax", context, - execute: async () => { - await quarto(["render", imgQmd.input]); - await quarto(["render", divQmd.input]); + execute: async (logFile?: string) => { + // render failures land in the log as ERROR records; noErrors below + // surfaces them (otherwise the comparison verify would just hit an + // unhelpful ENOENT on the missing output file) + await runQuarto(["render", imgQmd.input], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", divQmd.input], { + logFile, + throwOnFailure: false, + }); }, - verify: [verify], + verify: [noErrors, verify], type: "smoke", }; test(testDesc); diff --git a/tests/smoke/engine/invalid-engine-in-project.test.ts b/tests/smoke/engine/invalid-engine-in-project.test.ts index 4b7081e94f8..e4c810c8200 100644 --- a/tests/smoke/engine/invalid-engine-in-project.test.ts +++ b/tests/smoke/engine/invalid-engine-in-project.test.ts @@ -1,19 +1,21 @@ -import { assertRejects } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; +import { testQuartoCmd } from "../../test.ts"; +import { printsMessage } from "../../verify.ts"; -test( - { - name: "invalid engines option errors", - execute: async () => { - assertRejects( - async () => {await quarto(["render", "docs/engine/invalid-project/notebook.qmd"])}, - Error, - "'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine", - ) - }, - type: "smoke", - context: {}, - verify: [], - } -) \ No newline at end of file +// The previous version of this test wrapped an (un-awaited) assertRejects +// around an in-process quarto() call, so it passed vacuously. Asserting on +// the ERROR log record works in both dev and binary (QUARTO_TEST_BIN) modes: +// in binary mode the message reaches the log via the child's error logging +// (or the synthetic record carrying the stderr tail). +testQuartoCmd( + "render", + ["docs/engine/invalid-project/notebook.qmd"], + [ + printsMessage({ + level: "ERROR", + regex: + /'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine/, + }), + ], + {}, + "invalid engines option errors", +); diff --git a/tests/smoke/env/check.test.ts b/tests/smoke/env/check.test.ts index 9125b71d9c9..d0b41d09899 100644 --- a/tests/smoke/env/check.test.ts +++ b/tests/smoke/env/check.test.ts @@ -5,13 +5,20 @@ * */ import { testQuartoCmd } from "../../test.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { noErrorsOrWarnings, printsMessage } from "../../verify.ts"; +// Dev mode reports the 99.9.9 sentinel version; a built binary reports its +// real version, so only require a semver-shaped version line there. +const versionRegex = isBinaryMode() + ? /Version: \d+\.\d+\.\d+/ + : /Version: 99\.9\.9/; + testQuartoCmd( "check", [], [ noErrorsOrWarnings, - printsMessage({level: "INFO", regex: /Version: 99\.9\.9/}), + printsMessage({level: "INFO", regex: versionRegex}), ], ); diff --git a/tests/smoke/extensions/extension-render-journals.test.ts b/tests/smoke/extensions/extension-render-journals.test.ts index 0327048fb64..9137c9f9819 100644 --- a/tests/smoke/extensions/extension-render-journals.test.ts +++ b/tests/smoke/extensions/extension-render-journals.test.ts @@ -5,10 +5,11 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const journalRepos = [ // { repo: "acm", noSupporting: true }, TODO this format needs changes after this merge. @@ -42,15 +43,14 @@ for (const journalRepo of journalRepos) { // Sets up the test setup: async () => { console.log(`using quarto-journals/${journalRepo.repo}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await quarto([ - "use", - "template", - `quarto-journals/${journalRepo.repo}`, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + `quarto-journals/${journalRepo.repo}`, + "--no-prompt", + ]); + }); }, // Cleans up the test diff --git a/tests/smoke/extensions/extension-render-typst-templates.test.ts b/tests/smoke/extensions/extension-render-typst-templates.test.ts index 5acdbe91487..1d62194c5b7 100644 --- a/tests/smoke/extensions/extension-render-typst-templates.test.ts +++ b/tests/smoke/extensions/extension-render-typst-templates.test.ts @@ -5,10 +5,11 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const GITHUB_REPO = "quarto-ext/typst-templates"; @@ -40,15 +41,14 @@ for (const name of typstTemplates) { setup: async () => { const source = `${GITHUB_REPO}/${name}`; console.log(`using template: ${source}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await quarto([ - "use", - "template", - source, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + source, + "--no-prompt", + ]); + }); }, teardown: async () => { diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 77729c882e1..11be1abe0a6 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -5,20 +5,19 @@ */ import { docs } from "../../utils.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { assertEquals } from "testing/asserts"; -import { isWindows } from "../../../src/deno_ral/platform.ts"; async function runEditorSupportCrossref(doc: string) { - const cmdLine: string = isWindows ? - "../package/dist/bin/quarto.cmd" : - "../package/dist/bin/quarto"; - - const cmd = new Deno.Command(cmdLine, { + // pinned to the locally-built dev CLI (not PATH quarto) as before the + // binary-mode migration; resolves to QUARTO_TEST_BIN in binary mode + const cmd = new Deno.Command(quartoDevBinCmd(), { args: ["editor-support", "crossref"], stdin: "piped", stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); const child = cmd.spawn(); const writer = child.stdin.getWriter(); @@ -26,7 +25,11 @@ async function runEditorSupportCrossref(doc: string) { Deno.readTextFileSync(doc), ); await writer.write(buf); - writer.releaseLock(); + // close() flushes the write and closes the stream (sends EOF to the child); + // it does not release the writer's lock, but nothing reuses the stream so + // the held lock is inert and leaks no resource. Do NOT releaseLock() first — + // that detaches the writer and makes close() throw "A writable stream is not + // associated with the writer". await writer.close(); const outputBuf = await child.output(); const status = await child.status; @@ -36,15 +39,23 @@ async function runEditorSupportCrossref(doc: string) { return json; } +// The harness swallows errors thrown from execute() (they become log +// records nothing here reads), so the spawn + assertions must live in a +// verifier for these tests to be able to fail at all. test({ name: "editor-support:crossref:smoke-1", context: {}, - execute: async () => { - const json = await runEditorSupportCrossref(docs("crossrefs/sections.qmd")); - assertEquals(json.entries[0].key, "sec-introduction"); - assertEquals(json.entries[0].caption, "Introduction"); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref output", + verify: async (_outputs) => { + const json = await runEditorSupportCrossref( + docs("crossrefs/sections.qmd"), + ); + assertEquals(json.entries[0].key, "sec-introduction"); + assertEquals(json.entries[0].caption, "Introduction"); + }, + }], type: "smoke", }); @@ -52,10 +63,13 @@ function smokeTestCrossref(name: string, doc: string) { test({ name, context: {}, - execute: async () => { - await runEditorSupportCrossref(doc); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref runs cleanly", + verify: async (_outputs) => { + await runEditorSupportCrossref(doc); + }, + }], type: "smoke", }); } diff --git a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts index 2f26608179b..d0ae5e8041c 100644 --- a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts +++ b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts @@ -7,6 +7,7 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { _setIsRStudioForTest } from "../../../src/core/platform.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { ExecuteOutput, testQuartoCmd, @@ -14,8 +15,11 @@ import { import { assert, assertEquals } from "testing/asserts"; // Test: standalone file inspect with RStudio override should NOT emit project. -// Uses _setIsRStudioForTest to avoid Deno.env.set() race conditions in -// parallel tests (see #14218, PR #12621). +// Dev mode uses _setIsRStudioForTest to avoid Deno.env.set() race conditions +// in parallel tests (see #14218, PR #12621). In binary mode the in-process +// hook cannot reach the spawned quarto, so RSTUDIO=1 is passed via the +// test's env instead (isRStudio() checks the env var; buildBinaryEnv strips +// ambient RSTUDIO so the companion "not RStudio" test below stays clean). (() => { const input = "docs/inspect/standalone-hello.qmd"; const output = "docs/inspect/standalone-hello.json"; @@ -34,11 +38,16 @@ import { assert, assertEquals } from "testing/asserts"; } ], { + env: isBinaryMode() ? { RSTUDIO: "1" } : undefined, setup: async () => { - _setIsRStudioForTest(true); + if (!isBinaryMode()) { + _setIsRStudioForTest(true); + } }, teardown: async () => { - _setIsRStudioForTest(undefined); + if (!isBinaryMode()) { + _setIsRStudioForTest(undefined); + } if (existsSync(output)) { Deno.removeSync(output); } diff --git a/tests/smoke/issues/9133/9133.test.ts b/tests/smoke/issues/9133/9133.test.ts index d0b21db04a0..82b5b6c06af 100644 --- a/tests/smoke/issues/9133/9133.test.ts +++ b/tests/smoke/issues/9133/9133.test.ts @@ -1,23 +1,23 @@ -import { quarto } from "../../../../src/quarto.ts"; +import { runQuarto } from "../../../quarto-cmd.ts"; import { test } from "../../../test.ts"; if (Deno.build.os !== "windows") { test({ - name: "https://github.com/quarto-dev/quarto-cli/issues/9133", + name: "https://github.com/quarto-dev/quarto-cli/issues/9133", context: { setup: async () => { Deno.mkdirSync("smoke/issues/9133/oh'\"no", { recursive: true }); Deno.copyFileSync("smoke/issues/9133/jl", "smoke/issues/9133/oh'\"no/jl.qmd"); Deno.copyFileSync("smoke/issues/9133/py", "smoke/issues/9133/oh'\"no/py.qmd"); - const timeout = new Promise((_resolve, reject) => { - setTimeout(reject, 600000, "timed out after 10 minutes"); - }); - await Promise.race([ - Promise.all([ - quarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), - quarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), - ]), - timeout, + // concurrent renders reproduce the original intra-process race; + // in binary mode each render is a separate process, so the race + // may not reproduce there. runQuarto supplies the 10-minute + // timeout the explicit Promise.race used to provide, and failures + // throw (no logFile: two concurrent children would interleave a + // shared log file). + await Promise.all([ + runQuarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), + runQuarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), ]); } }, diff --git a/tests/smoke/jupyter/cache.test.ts b/tests/smoke/jupyter/cache.test.ts index 6a687778aab..63a347e6949 100644 --- a/tests/smoke/jupyter/cache.test.ts +++ b/tests/smoke/jupyter/cache.test.ts @@ -4,10 +4,10 @@ * Copyright (C) 2023 Posit Software, PBC */ import { dirname, join } from "path"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { docs } from "../../utils.ts"; -import { folderExists, printsMessage } from "../../verify.ts"; +import { folderExists, noErrors, printsMessage } from "../../verify.ts"; import { fileLoader } from "../../utils.ts"; import { safeExistsSync, safeRemoveSync } from "../../../src/core/path.ts"; @@ -16,14 +16,18 @@ const cacheFolder = join(dirname(testInput.input), ".jupyter_cache") test({ name: "Jupyter cache is working", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); + // repeated executions to trigger jupyter cache; failures reach the + // verifiers as log records, mirroring testQuartoCmd + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { @@ -39,6 +43,7 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder), // this will check only for the second render that should be read from cache printsMessage({ level: "INFO", regex: /Notebook read from cache/}) @@ -54,14 +59,18 @@ const cacheFolder2 = join(dirname(testInput2.input), ".cache/jupyter-cache") test({ name: "Jupyter cache folder can be change", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); + // repeated executions to trigger jupyter cache; failures reach the + // verifiers as log records, mirroring testQuartoCmd + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { @@ -80,6 +89,7 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder2), // this will check only for the second render that should be read from cache printsMessage({level: "INFO", regex: /Notebook read from cache/}) diff --git a/tests/smoke/jupyter/issue-10097.test.ts b/tests/smoke/jupyter/issue-10097.test.ts index 19586200e28..743e237a76f 100644 --- a/tests/smoke/jupyter/issue-10097.test.ts +++ b/tests/smoke/jupyter/issue-10097.test.ts @@ -1,26 +1,25 @@ /* * parameter-label-duplication.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/10097 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:parameter:label-duplication", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/10097 - await quarto(["render", - "docs/jupyter/parameters/issue-10097.qmd", - "--execute-param", 'datapath:"weird"', - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/10097 +testQuartoCmd( + "render", + [ + "docs/jupyter/parameters/issue-10097.qmd", + "--execute-param", + 'datapath:"weird"', + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:parameter:label-duplication", +); diff --git a/tests/smoke/jupyter/issue-12374.test.ts b/tests/smoke/jupyter/issue-12374.test.ts index 56c5ce67dc0..3007adb3499 100644 --- a/tests/smoke/jupyter/issue-12374.test.ts +++ b/tests/smoke/jupyter/issue-12374.test.ts @@ -1,25 +1,23 @@ /* * issue-12374.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/12374 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:issue-12374.test.ts", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/12374 - await quarto(["render", - "docs/jupyter/issue-12374.ipynb", - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/12374 +testQuartoCmd( + "render", + [ + "docs/jupyter/issue-12374.ipynb", + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:issue-12374.test.ts", +); diff --git a/tests/smoke/logging/log-level-and-formats.test.ts b/tests/smoke/logging/log-level-and-formats.test.ts index 9c2bbf703d0..1b7e7b05b72 100644 --- a/tests/smoke/logging/log-level-and-formats.test.ts +++ b/tests/smoke/logging/log-level-and-formats.test.ts @@ -10,6 +10,7 @@ import { execProcess } from "../../../src/core/process.ts"; import { md5HashSync } from "../../../src/core/hash.ts"; import { safeRemoveIfExists } from "../../../src/core/path.ts"; import { quartoDevCmd, outputForInput } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { assert } from "testing/asserts"; import { LogFormat } from "../../../src/core/log.ts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; @@ -90,7 +91,8 @@ function testLogDirectly(options: { cmd: quartoDevCmd(), args: args, stdout: "piped", - stderr: "piped" + stderr: "piped", + ...quartoSpawnEnvOptions(), }); // Get stdout/stderr with fallback to empty string @@ -137,35 +139,34 @@ function testLogDirectly(options: { } - // If JSON format is specified, verify the output is valid JSON + // If JSON format is specified, verify the log file parses and the + // record levels match expectations. NOTE: keep the parse try/catch + // NARROW - a previous version wrapped the level assertions in a + // catch-all that silently swallowed every failure. if (logFile && options.format === "json-stream") { assert(existsSync(logFile), "Log file should exist"); - let foundValidJson = false; + let outputs; try { - const outputs = readExecuteOutput(logFile); - foundValidJson = true; - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldNotContainLevel?.includes(out.levelName)).forEach( - (out) => { - assert(false, `JSON output should not contain level ${out.levelName}, but found: ${out.msg}`); - } + outputs = readExecuteOutput(logFile); + } catch { + outputs = undefined; + } + assert(outputs !== undefined, "JSON format should produce valid JSON output"); + const records = outputs!.filter((out) => out.msg !== ""); + const levels = new Set(records.map((out) => out.levelName)); + for (const lvl of options.expectedOutputs?.shouldNotContainLevel ?? []) { + const offending = records.find((out) => out.levelName === lvl); + assert( + offending === undefined, + `JSON log should not contain level ${lvl}, but found: ${offending?.msg}` ); - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldContainLevel?.includes(out.levelName)).forEach( - (out) => { - let json = undefined; - try { - json = JSON.parse(out.msg); - } catch { - assert(false, "Error parsing JSON returned by quarto meta"); - } - assert( - Object.keys(json).length > 0, - "JSON returned by quarto meta seems invalid", - ); - } + } + for (const lvl of options.expectedOutputs?.shouldContainLevel ?? []) { + assert( + levels.has(lvl), + `JSON log should contain at least one ${lvl} record; found levels: ${[...levels].join(", ") || "(none)"}` ); - - } catch (e) {} - assert(foundValidJson, "JSON format should produce valid JSON output"); + } } } finally { // Clean up log file if it exists diff --git a/tests/smoke/lua-unit/lua-unit.test.ts b/tests/smoke/lua-unit/lua-unit.test.ts index 1553052f60f..77dcf429fa9 100644 --- a/tests/smoke/lua-unit/lua-unit.test.ts +++ b/tests/smoke/lua-unit/lua-unit.test.ts @@ -21,6 +21,7 @@ import { fromFileUrl, join } from "../../../src/deno_ral/path.ts"; import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; // Explicit list, relative to tests/unit-lua/. Keep alphabetized. @@ -50,7 +51,7 @@ for (const relPath of LUA_TESTS) { { cmd: quartoDevCmd(), args: ["run", luaScript], - env: { LUA_PATH }, + ...quartoSpawnEnvOptions({ LUA_PATH }), }, undefined, undefined, diff --git a/tests/smoke/project/project-prepost.test.ts b/tests/smoke/project/project-prepost.test.ts index b7895f4a5f1..370195ca03f 100644 --- a/tests/smoke/project/project-prepost.test.ts +++ b/tests/smoke/project/project-prepost.test.ts @@ -63,18 +63,26 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/extension")], - [{ - name: "i-exist.txt exists", + // assertions live in verify (teardown assertions skip cleanup on failure); + // noErrors keeps the purely-negative i-exist check from passing vacuously + // on a failed render + [noErrors, { + name: "prepost extension file effects", verify: async () => { - const path = join(docs("project/prepost/extension"), "i-exist.txt"); - verifyNoPath(path); + verifyNoPath(join(docs("project/prepost/extension"), "i-exist.txt")); + verifyPath(join(docs("project/prepost/extension"), "i-was-created.txt")); } }], { + // remove artifacts a prior crashed run may have left behind, so verifyPath + // proves this render created i-was-created.txt (not a stale copy) and + // pre-render's i-exist.txt guard starts from a clean slate + setup: async () => { + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-exist.txt")); + }, teardown: async () => { - const path = join(docs("project/prepost/extension"), "i-was-created.txt"); - verifyPath(path); - safeRemoveIfExists(path); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); const siteDir = join(docs("project/prepost/extension"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); @@ -85,19 +93,29 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/issue-10828")], - [], + // assertions live in verify, not teardown (an empty verify list + // asserted nothing about the render itself) + [noErrors, { + name: "project input/output files written", + verify: async () => { + verifyPath(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + verifyPath(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + } + }], { env: { "QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES": normalizePath(docs("project/prepost/issue-10828/input-files.txt")), "QUARTO_USE_FILE_FOR_PROJECT_OUTPUT_FILES": normalizePath(docs("project/prepost/issue-10828/output-files.txt")) }, + // remove artifacts a prior crashed run may have left behind, so verifyPath + // proves this render wrote input-files.txt/output-files.txt, not a stale copy + setup: async () => { + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + }, teardown: async () => { - const inputPath = normalizePath(docs("project/prepost/issue-10828/input-files.txt")); - const outputPath = normalizePath(docs("project/prepost/issue-10828/output-files.txt")); - verifyPath(inputPath); - safeRemoveIfExists(inputPath); - verifyPath(outputPath); - safeRemoveIfExists(outputPath); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); const siteDir = join(docs("project/prepost/issue-10828"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); diff --git a/tests/smoke/project/project-stdout.test.ts b/tests/smoke/project/project-stdout.test.ts index dfa38d86299..b0f2aec8fe2 100644 --- a/tests/smoke/project/project-stdout.test.ts +++ b/tests/smoke/project/project-stdout.test.ts @@ -14,6 +14,7 @@ import { docs } from "../../utils.ts"; import { directoryEmptyButFor, fileExists, + noErrors, verifyYamlFile, } from "../../verify.ts"; @@ -30,7 +31,8 @@ const siteOutDir = join(siteProjDir, outDir); testQuartoCmd( "render", [siteProjDir, "-o", "-"], - [], + // an empty verify list asserted nothing - a failed stdout render passed + [noErrors], { teardown: async () => { if (existsSync(siteOutDir)) { diff --git a/tests/smoke/render/render-format-extension.test.ts b/tests/smoke/render/render-format-extension.test.ts index f82374cf32f..09eaa5ab758 100644 --- a/tests/smoke/render/render-format-extension.test.ts +++ b/tests/smoke/render/render-format-extension.test.ts @@ -12,8 +12,8 @@ // Both files serve different purposes and should remain separate. import { safeRemoveSync } from "../../../src/core/path.ts"; -import { docs } from "../../utils.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { docs, withCwd } from "../../utils.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { testRender } from "./render.ts"; @@ -28,19 +28,16 @@ import { testRender } from "./render.ts"; const updateExtensions = async () => { try { console.log("Updating quarto-journals extensions to latest versions..."); - const wd = Deno.cwd(); - Deno.chdir(docs("extensions/format/academic")); - - for (const repo of ["acs", "elsevier"]) { - await quarto([ - "update", - "extension", - `quarto-journals/${repo}`, - "--no-prompt", - ]); - } - - Deno.chdir(wd); + await withCwd(docs("extensions/format/academic"), async () => { + for (const repo of ["acs", "elsevier"]) { + await runQuarto([ + "update", + "extension", + `quarto-journals/${repo}`, + "--no-prompt", + ]); + } + }); console.log("Extensions updated successfully"); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/tests/smoke/render/render-freeze.test.ts b/tests/smoke/render/render-freeze.test.ts index 7291e24d798..2298d4f41cd 100644 --- a/tests/smoke/render/render-freeze.test.ts +++ b/tests/smoke/render/render-freeze.test.ts @@ -10,9 +10,9 @@ import { assert } from "testing/asserts"; import { Metadata } from "../../../src/config/types.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ExecuteOutput, Verify } from "../../test.ts"; -import { outputCreated } from "../../verify.ts"; +import { noErrors, outputCreated } from "../../verify.ts"; import { testRender } from "./render.ts"; const regex = /output file: .*\.knit\.md/m; @@ -67,9 +67,10 @@ const ignoreFrozen = { const projectOutputExists: Verify = { name: "Make sure project output exists", - verify: (_output: ExecuteOutput[]) => { - outputCreated(path, "html"); - return Promise.resolve(); + verify: (output: ExecuteOutput[]) => { + // delegate to outputCreated's verify - a previous version constructed + // the Verify and discarded it, asserting nothing + return outputCreated(path, "html").verify(output); }, }; @@ -107,7 +108,7 @@ function testFileContext( markdown, ); - await quarto(["render", path]); + await runQuarto(["render", path]); }, teardown: async () => { // Clean up the test file @@ -130,7 +131,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "clean fzr - auto", ...testContext, @@ -142,7 +143,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, ignoreFrozen], + [noErrors, projectOutputExists, ignoreFrozen], { name: "dirty fzr - auto", setup: async () => { @@ -167,7 +168,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "dirty fzr - freeze", setup: async () => { diff --git a/tests/smoke/render/render-output-file-collision.test.ts b/tests/smoke/render/render-output-file-collision.test.ts index f360ed0d54e..0b802f536f8 100644 --- a/tests/smoke/render/render-output-file-collision.test.ts +++ b/tests/smoke/render/render-output-file-collision.test.ts @@ -14,7 +14,7 @@ */ import { existsSync, safeRemoveSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { docs } from "../../utils.ts"; import { testQuartoCmd } from "../../test.ts"; import { @@ -103,7 +103,7 @@ testQuartoCmd( { setup: async () => { await cleanup(defaultDir, [...defaultOutputs, ".quarto"])(); - await quarto(["render", defaultDir]); + await runQuarto(["render", defaultDir]); }, teardown: cleanup(defaultDir, [...defaultOutputs, ".quarto"]), }, diff --git a/tests/smoke/run/command-passthrough.test.ts b/tests/smoke/run/command-passthrough.test.ts index 6db94e984e9..462580d896c 100644 --- a/tests/smoke/run/command-passthrough.test.ts +++ b/tests/smoke/run/command-passthrough.test.ts @@ -1,16 +1,18 @@ import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; const testPassthroughCmd = (name: string, command: string, args: string[]) => { unitTest(name, async () => { const result = await execProcess({ - cmd: quartoDevCmd(), + cmd: quartoDevCmd(), args: [ command, ...args, - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }); diff --git a/tests/smoke/run/run-script.test.ts b/tests/smoke/run/run-script.test.ts index ee6fb0f16fe..308a465ffce 100644 --- a/tests/smoke/run/run-script.test.ts +++ b/tests/smoke/run/run-script.test.ts @@ -3,6 +3,7 @@ import { ensureDirSync } from "../../../src/deno_ral/fs.ts"; import { assert, assertEquals } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; import { EOL } from "fs/eol"; import { lines } from "../../../src/core/text.ts"; @@ -20,9 +21,9 @@ const ensureStreams = (name: string, script: string, stdout: string, stderr: str basename(script), ], // disable logging here to allow for checking the output - env: { + ...quartoSpawnEnvOptions({ "QUARTO_LOG_LEVEL": "CRITICAL", - } + }), }, undefined, undefined, @@ -55,7 +56,8 @@ const testRunCmd = (name: string, script: string) => { args: [ "run", basename(script), - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }, diff --git a/tests/smoke/run/stdlib-run-version.test.ts b/tests/smoke/run/stdlib-run-version.test.ts index bbe7dadf726..eef54b05648 100644 --- a/tests/smoke/run/stdlib-run-version.test.ts +++ b/tests/smoke/run/stdlib-run-version.test.ts @@ -8,15 +8,18 @@ import { execProcess } from "../../../src/core/process.ts"; import { assert } from "testing/asserts"; import { unitTest } from "../../test.ts"; +import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { isWindows } from "../../../src/deno_ral/platform.ts"; unitTest("stdlib-run-version", async () => { const result = await execProcess({ - cmd: "quarto", + cmd: quartoDevCmd(), args: [ "run", "docs/run/test-stdlib.ts", ], + ...quartoSpawnEnvOptions(), }); console.log({result}) assert(result.success); diff --git a/tests/smoke/self-contained/stdout.test.ts b/tests/smoke/self-contained/stdout.test.ts index f03a8e6c730..5254c0a397a 100644 --- a/tests/smoke/self-contained/stdout.test.ts +++ b/tests/smoke/self-contained/stdout.test.ts @@ -1,11 +1,13 @@ -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; test({ name: "https://github.com/quarto-dev/quarto-cli/issues/11068", context: { setup: async() => { - await quarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); + // only asserts the render to stdout succeeds (failure throws); in + // binary mode the child's stdout is drained and discarded + await runQuarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); } }, execute: async () => {}, diff --git a/tests/smoke/smoke-all.test.ts b/tests/smoke/smoke-all.test.ts index 1172d2f13e4..df4791a496b 100644 --- a/tests/smoke/smoke-all.test.ts +++ b/tests/smoke/smoke-all.test.ts @@ -57,7 +57,7 @@ import { findProjectDir, findProjectOutputDir, outputForInput } from "../utils.t import { jupyterNotebookToMarkdown } from "../../src/command/convert/jupyter.ts"; import { basename, dirname, join, relative } from "../../src/deno_ral/path.ts"; import { WalkEntry } from "../../src/deno_ral/fs.ts"; -import { quarto } from "../../src/quarto.ts"; +import { runQuarto } from "../quarto-cmd.ts"; import { safeExistsSync, safeRemoveSync } from "../../src/core/path.ts"; import { runningInCI } from "../../src/core/ci-info.ts"; @@ -431,7 +431,9 @@ for (const { path: fileName } of files) { projectPath && !renderedProjects.has(projectPath) ) { - await quarto(["render", projectPath]); + // fail-loudly pre-render (throwOnFailure defaults to true); + // dispatches to the built binary when QUARTO_TEST_BIN is set + await runQuarto(["render", projectPath]); renderedProjects.add(projectPath); } diff --git a/tests/smoke/typst-gather/typst-gather.test.ts b/tests/smoke/typst-gather/typst-gather.test.ts index 7920a63158c..791128eab8c 100644 --- a/tests/smoke/typst-gather/typst-gather.test.ts +++ b/tests/smoke/typst-gather/typst-gather.test.ts @@ -4,6 +4,19 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { execProcess } from "../../../src/core/process.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; + +// The cached typst/ dirs are gitignored and SURVIVE across local runs - a +// stale cache would keep the existence verifiers green even if typst-gather +// broke entirely. Each test removes its cache first so verification is +// always against a fresh gather. +const freshCache = (cacheDir: string) => async () => { + if (existsSync(cacheDir)) { + Deno.removeSync(cacheDir, { recursive: true }); + } +}; + // Test 1: Auto-detection from _extension.yml const verifyPackagesCreated: Verify = { name: "Verify typst/packages directory was created", @@ -38,9 +51,10 @@ const verifyExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyPackagesCreated, verifyExamplePackageCached], + [noErrors, verifyPackagesCreated, verifyExamplePackageCached], { cwd: () => "smoke/typst-gather", + setup: freshCache("_extensions/test-format/typst"), }, "typst-gather caches preview packages from extension templates", ); @@ -78,9 +92,10 @@ const verifyConfigExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyConfigPackagesCreated, verifyConfigExamplePackageCached], + [noErrors, verifyConfigPackagesCreated, verifyConfigExamplePackageCached], { cwd: () => "smoke/typst-gather/with-config", + setup: freshCache("_extensions/config-format/typst"), }, "typst-gather uses rootdir from config file", ); @@ -246,7 +261,9 @@ const verifyNoPackagesStaged: Verify = { testQuartoCmd( "render", [join(noPackagesProjectDir, "index.qmd"), "--to", "typst"], - [verifyNoPackagesStaged], + // noErrors: a render failing before staging would trivially satisfy the + // purely-negative "nothing staged" assertions + [noErrors, verifyNoPackagesStaged], { teardown: async () => { try { @@ -268,20 +285,18 @@ async function runQuarto( cwd: string, env?: Record, ): Promise<{ success: boolean; stdout: string; stderr: string }> { - const quartoCmd = Deno.build.os === "windows" ? "quarto.cmd" : "quarto"; - const quartoPath = join( - Deno.cwd(), - "..", - "package/dist/bin", - quartoCmd, - ); const result = await execProcess({ - cmd: quartoPath, + // pinned to the locally-built dev CLI (not PATH quarto) as before the + // binary-mode migration; resolves to QUARTO_TEST_BIN in binary mode + cmd: quartoDevBinCmd(), args, cwd, stdout: "piped", stderr: "piped", - env: env ? { ...Deno.env.toObject(), ...env } : undefined, + // dev mode: overlay merges into the inherited ambient env (same child + // env as the previous explicit { ...Deno.env.toObject(), ...env }); + // binary mode: sanitized ambient env + overlay with clearEnv + ...quartoSpawnEnvOptions(env), }); return { success: result.success, diff --git a/tests/smoke/verify/pdf-metadata.test.ts b/tests/smoke/verify/pdf-metadata.test.ts index 31ad2c775cb..bf694d8a73e 100644 --- a/tests/smoke/verify/pdf-metadata.test.ts +++ b/tests/smoke/verify/pdf-metadata.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfMetadata } from "../../verify-pdf-metadata.ts"; import { assert } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,14 +42,26 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +// Test: Render fixture and run assertions. The assertions live in a +// verifier (not teardown): teardown assertions skip cleanup on failure, +// leaving a stale fixture.pdf that would satisfy the next run even if its +// render failed. setup also removes any stale pdf from a crashed run. +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf metadata assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/smoke/verify/pdf-text-position.test.ts b/tests/smoke/verify/pdf-text-position.test.ts index 2f40fa0e26d..0ea195b7f77 100644 --- a/tests/smoke/verify/pdf-text-position.test.ts +++ b/tests/smoke/verify/pdf-text-position.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfTextPositions, PdfTextPositionAssertion } from "../../verify-pdf-text-position.ts"; import { assert, AssertionError } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,21 +42,33 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +// Test: Render fixture and run assertions. The assertions live in a +// verifier (not teardown): teardown assertions skip cleanup on failure, +// leaving a stale fixture.pdf that would satisfy the next run even if its +// render failed. setup also removes any stale pdf from a crashed run. +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf text position assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + await runSemanticTagTests(); + await runPageRoleTests(); + await runEdgeOverrideTests(); + await runDistanceConstraintTests(); + await runDistanceConstraintErrorTests(); + await runPageRoleWithEdgeTests(); + await runCenterEdgeTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - await runSemanticTagTests(); - await runPageRoleTests(); - await runEdgeOverrideTests(); - await runDistanceConstraintTests(); - await runDistanceConstraintErrorTests(); - await runPageRoleWithEdgeTests(); - await runCenterEdgeTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/smoke/website/drafts-env.test.ts b/tests/smoke/website/drafts-env.test.ts index 7e7584d4d8c..73347ba1bdc 100644 --- a/tests/smoke/website/drafts-env.test.ts +++ b/tests/smoke/website/drafts-env.test.ts @@ -16,6 +16,13 @@ const renderDir = docs("websites/drafts/drafts-env"); const dir = join(Deno.cwd(), renderDir); const outDir = join(dir, "_site"); +// The process-global set is required in dev (in-process) mode: +// src/project/project-profile.ts caches the base profile from the env on the +// FIRST render in the process (`baseQuartoProfile`), so a per-render env +// override is ignored whenever another test rendered first. Setting it at +// module load (before any test runs) preserves the pre-existing behavior. +// The context.env below is what reaches the spawned binary in binary mode +// (a fresh process per render, so the cache concern doesn't apply there). Deno.env.set("QUARTO_PROFILE", "drafts"); testQuartoCmd( @@ -23,6 +30,7 @@ testQuartoCmd( [renderDir], [noErrorsOrWarnings, ...[doesntHaveContentLinksToDrafts, doesntHaveEnvelopeLinksToDrafts, draftPostIsEmpty, searchDoesntHaveDraft, siteMapDoesntHaveDraft].map((ver) => { return ver(outDir)})], { + env: { QUARTO_PROFILE: "drafts" }, teardown: async () => { if (existsSync(outDir)) { await Deno.remove(outDir, { recursive: true }); diff --git a/tests/test.ts b/tests/test.ts index 729568e41f1..31a37fddfd2 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -10,7 +10,7 @@ import { warning } from "../src/deno_ral/log.ts"; import { initDenoDom } from "../src/core/deno-dom.ts"; import { cleanupLogger, initializeLogger, flushLoggers, logError, LogLevel, LogFormat } from "../src/core/log.ts"; -import { quarto } from "../src/quarto.ts"; +import { appendLogError, isBinaryMode, runQuarto } from "./quarto-cmd.ts"; import { join } from "../src/deno_ral/path.ts"; import * as colors from "fmt/colors"; import { runningInCI } from "../src/core/ci-info.ts"; @@ -36,8 +36,9 @@ export interface TestDescriptor { // Sets up the test context: TestContext; - // Executes the test - execute: () => Promise; + // Executes the test. In binary mode (QUARTO_TEST_BIN) the harness passes + // the json-stream log file path so the spawned quarto can write it. + execute: (logFile?: string) => Promise; // Used to verify the outcome of the test verify: Verify[]; @@ -90,6 +91,12 @@ export interface TestContext { // Defaults to 600000 (10 minutes). Lower it to assert a performance budget // (e.g. a render that must not regress into a hang). timeout?: number; + + // Marks a test that exercises quarto internals in-process and therefore + // cannot run against an external built binary. Such tests are ignored + // when QUARTO_TEST_BIN is set. Use sparingly — most tests should go + // through testQuartoCmd/runQuarto and work in both modes. + requiresDevQuarto?: boolean; } // Allow to merge test contexts in Tests helpers @@ -127,6 +134,9 @@ export function mergeTestContexts(baseContext: TestContext, additionalContext?: }, // override ignore if provided ignore: additionalContext.ignore ?? baseContext.ignore, + // override requiresDevQuarto if provided + requiresDevQuarto: additionalContext.requiresDevQuarto ?? + baseContext.requiresDevQuarto, // merge env with additional context taking precedence env: { ...baseContext.env, ...additionalContext.env }, // override timeout if provided @@ -147,19 +157,17 @@ export function testQuartoCmd( } test({ name, - execute: async () => { - const timeoutMs = context?.timeout ?? 600000; - const timeout = new Promise((_resolve, reject) => { - setTimeout( - reject, - timeoutMs, - `timed out after ${timeoutMs}ms`, - ); + execute: async (logFile?: string) => { + await runQuarto([cmd, ...args], { + env: context?.env, + logFile, + logLevel: logConfig?.level, + logFormat: logConfig?.format, + timeoutMs: context?.timeout, + // failures must reach the verifiers as log records, not exceptions + // (mirrors the historical catch-and-log behavior in test()) + throwOnFailure: false, }); - await Promise.race([ - quarto([cmd, ...args], undefined, context?.env), - timeout, - ]); }, verify, context: context || {}, @@ -213,7 +221,9 @@ export function test(test: TestDescriptor) { const sanitizeResources = test.context.sanitize?.resources; const sanitizeOps = test.context.sanitize?.ops; const sanitizeExit = test.context.sanitize?.exit; - const ignore = test.context.ignore; + // dev-only tests are ignored when targeting an external built binary + const ignore = test.context.ignore || + (isBinaryMode() && test.context.requiresDevQuarto); const userSession = !runningInCI(); const args: Deno.TestDefinition = { @@ -231,9 +241,15 @@ export function test(test: TestDescriptor) { await test.context.setup(); } + // In binary mode the spawned quarto owns the log file; the harness + // must not initialize (or later destroy) its own logger for + // capture — cleanupLogger() would permanently tear down the + // default handlers for subsequent tests in this process. + const binMode = isBinaryMode(); + let cleanedup = false; const cleanupLogOnce = async () => { - if (!cleanedup) { + if (!cleanedup && !binMode) { await cleanupLogger(); cleanedup = true; } @@ -241,8 +257,9 @@ export function test(test: TestDescriptor) { // Capture the output const log = Deno.makeTempFileSync({ suffix: ".json" }); - const handlers = await initializeLogger({ - log: test.logConfig?.log || log, + const logTarget = test.logConfig?.log || log; + const handlers = binMode ? undefined : await initializeLogger({ + log: logTarget, level: test.logConfig?.level || "INFO", format: test.logConfig?.format || "json-stream", quiet: true, @@ -260,19 +277,37 @@ export function test(test: TestDescriptor) { try { try { - await test.execute(); + await test.execute(logTarget); } catch (e) { - logError(e); + if (binMode) { + // no harness logger in binary mode — append the failure to + // the log file directly so verifiers (and the failure + // report) still see it + const message = e instanceof Error + ? `${e.message}\n${e.stack ?? ""}` + : String(e); + appendLogError(logTarget, message); + } else { + logError(e); + } } // Cleanup the output logging await cleanupLogOnce(); - flushLoggers(handlers); + if (handlers) { + flushLoggers(handlers); + } - // Read the output - const testOutput = logOutput(log); - if (testOutput) { + // Read the output. Verifiers must read logTarget - the harness + // logger and the binary-mode child both write there; reading the + // temp file would hand every log verifier an empty array whenever + // logConfig.log is set. And a missing log is a FAILURE - skipping + // verification would pass the test without checking anything. + const testOutput = logOutput(logTarget); + if (testOutput === undefined) { + fail(`test log file is missing: ${logTarget}`); + } else { for (const ver of test.verify) { lastVerify = ver; if (userSession) { @@ -320,7 +355,14 @@ export function test(test: TestDescriptor) { ? colors.brightGreen(verifyFailed) : verifyFailed; - const logMessages = logOutput(log); + // guarded: a corrupt/unparseable log must not clobber the + // assembled failure report with a secondary parse error + let logMessages: ExecuteOutput[] | undefined; + try { + logMessages = logOutput(logTarget); + } catch { + logMessages = undefined; + } // Create distinctive failure marker for easy log navigation // This helps users find the failure when clicking GitHub Actions annotations @@ -383,6 +425,10 @@ export function test(test: TestDescriptor) { Deno.test(args); } +// Strict on purpose: a JSON.parse throw is how log-level-and-formats.test.ts +// detects that quarto emitted malformed JSON-stream output. A timeout-killed +// built quarto can leave a torn final line, but that is stripped at the source +// in mergeChildLog (tests/quarto-cmd.ts) so the merged log stays valid here. export function readExecuteOutput(log: string) { const jsonStream = Deno.readTextFileSync(log); const lines = jsonStream.split("\n").filter((line) => !!line); diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence.test.ts diff --git a/tests/utils.ts b/tests/utils.ts index eabd8344be7..4766aa62121 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -32,6 +32,22 @@ export async function withTempDir( } } +// Runs fn with the process cwd changed to dir, restoring the original cwd +// even if fn throws. Bare Deno.chdir(dir) ... Deno.chdir(wd) leaks the cwd +// into later tests when fn rejects. +export async function withCwd( + dir: string, + fn: () => T | Promise, +): Promise { + const wd = Deno.cwd(); + Deno.chdir(dir); + try { + return await fn(); + } finally { + Deno.chdir(wd); + } +} + // Find a _quarto.yaml file in the directory hierarchy of the input file export function findProjectDir(input: string, until?: RegExp | undefined): string | undefined { let dir = dirname(input); @@ -240,8 +256,15 @@ export function fileLoader(...path: string[]) { }; } +// Resolves the quarto executable for tests that spawn a real subprocess. +// Honors QUARTO_TEST_BIN (binary mode) so these tests target the built +// quarto under test; otherwise the dev quarto from PATH. // On Windows, `quarto.cmd` needs to be explicit in `execProcess()` export function quartoDevCmd(): string { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + if (bin && bin.length > 0) { + return bin; + } return isWindows ? "quarto.cmd" : "quarto"; }