From fc5932184ceebf2f7a1c6585f88a6770c56d870c Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Tue, 1 Sep 2026 22:05:25 +0200 Subject: [PATCH 1/5] Share the qemu ssh argv between the shell lanes qemu-runner.sh and test-matrix.sh each spelled the ssh option list for the test VM, and the copies had drifted: the runner kept the peer alive for 10 s x 6 and the matrix for 15 s x 4. tests/lib/qemu-ssh.sh now holds the one list. qemu_ssh_opts fills QEMU_SSH_OPTS at call time so a caller can still wrap its ssh in timeout(1), which cannot wrap a shell function. Both lanes keep the 60 s dead-peer budget. --- tests/lib/qemu-ssh.sh | 24 ++++++++++++++++++++++++ tests/qemu-runner.sh | 15 +++++---------- tests/test-matrix.sh | 13 ++++--------- 3 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 tests/lib/qemu-ssh.sh diff --git a/tests/lib/qemu-ssh.sh b/tests/lib/qemu-ssh.sh new file mode 100644 index 00000000..9ea7a40b --- /dev/null +++ b/tests/lib/qemu-ssh.sh @@ -0,0 +1,24 @@ +# Shared ssh argv for the qemu test VM. +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash +# timeout(1) cannot wrap a shell function, so what the callers share is the +# argv: qemu_ssh_opts fills QEMU_SSH_OPTS from QEMU_SSH_KEY and QEMU_PORT at +# call time, and each caller builds its own ssh command line around it. + +# shellcheck disable=SC2034 # Consumed by the sourcing script. +qemu_ssh_opts() +{ + QEMU_SSH_OPTS=( + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o ConnectTimeout=10 + -o ServerAliveInterval=10 + -o ServerAliveCountMax=6 + -i "$QEMU_SSH_KEY" + -p "$QEMU_PORT" + ) +} diff --git a/tests/qemu-runner.sh b/tests/qemu-runner.sh index e5837fe4..097a7c98 100755 --- a/tests/qemu-runner.sh +++ b/tests/qemu-runner.sh @@ -19,6 +19,9 @@ _QR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)" _QR_FIX="${_QR_DIR}/externals/test-fixtures" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${_QR_DIR}/tests/lib/qemu-ssh.sh" + QEMU_BIN="${QEMU_BIN:-qemu-system-aarch64}" QEMU_PORT="${QEMU_PORT:-2222}" QEMU_MEM="${QEMU_MEM:-2048}" @@ -174,16 +177,8 @@ qemu_start() # the suite's tolerance. _qemu_ssh_raw() { - ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=10 \ - -o ServerAliveCountMax=6 \ - -i "$QEMU_SSH_KEY" \ - -p "$QEMU_PORT" \ - root@127.0.0.1 "$@" + qemu_ssh_opts + ssh "${QEMU_SSH_OPTS[@]}" root@127.0.0.1 "$@" } # Run a command in the VM. Any argument that is an absolute path under the host diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 4e62777d..0fb8bfcc 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -73,6 +73,8 @@ source "${REPO_ROOT}/tests/test-config.sh" TEST_LABEL_WIDTH=45 # shellcheck source=tests/lib/test-runner.sh source "${REPO_ROOT}/tests/lib/test-runner.sh" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${REPO_ROOT}/tests/lib/qemu-ssh.sh" # Globals (test-runner.sh seeds pass/fail/skip; test-matrix.sh resets them per # mode and tracks no extra counters). @@ -192,15 +194,8 @@ run_qemu() if [ "${#args[@]}" -gt 0 ]; then printf -v quoted '%q ' "${args[@]}" fi - timeout 60 ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=15 \ - -o ServerAliveCountMax=4 \ - -i "$QEMU_SSH_KEY" -p "$QEMU_PORT" \ + qemu_ssh_opts + timeout 60 ssh "${QEMU_SSH_OPTS[@]}" \ root@127.0.0.1 "cd /mnt/host && ${quoted}" 2> /dev/null } From 8604b95872b02a9690b71ec0d7d803ef387d4fb1 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Tue, 1 Sep 2026 22:06:43 +0200 Subject: [PATCH 2/5] Define the conformance command interface The parser owns one capability-first command tree used by Make and CI. Suite registration cannot add commands. run writes results.json before the backend stops, so a failing teardown cannot lose a completed lane, and records the argv it parsed. report is red for a red lane in every format. Exit 77 is for a prerequisite reported absent before anything starts; a backend that fails to start is red. qemu-runner.sh stores VM state across start and stop commands. A command that starts a VM owns its cleanup trap, preserving the trap installed by test-matrix.sh. Start rejects an unusable guest /tmp, and reports the serial console and qemu's own output before the run directory holding them goes, since that text is all the caller sees of a VM that did not boot. Stop verifies the recorded process still names its pidfile before sending a signal. tests/test-qemu-runner.sh pins both against stand-ins, so no VM boots: a recycled pid, a process whose argv names the pidfile, and a start that never opens the port. --- .github/workflows/build.yml | 1 + .github/workflows/conformance.yml | 122 +++++ Makefile | 1 + mk/conformance.mk | 53 +++ mk/tests.mk | 8 +- scripts/conformance | 13 + scripts/proof-scope.py | 1 + tests/conformance/__init__.py | 4 + tests/conformance/backends/__init__.py | 18 + tests/conformance/backends/base.py | 81 ++++ tests/conformance/backends/elfuse.py | 92 ++++ tests/conformance/backends/proc.py | 73 +++ tests/conformance/backends/qemu.py | 121 +++++ tests/conformance/backends/ssh.py | 110 +++++ tests/conformance/cli.py | 447 ++++++++++++++++++ tests/conformance/elfcheck.py | 90 ++++ tests/conformance/expectations.py | 214 +++++++++ tests/conformance/ids.py | 57 +++ tests/conformance/jsonc.py | 66 +++ tests/conformance/judge.py | 50 ++ tests/conformance/model.py | 117 +++++ tests/conformance/payload.py | 233 +++++++++ tests/conformance/providers/__init__.py | 29 ++ tests/conformance/providers/base.py | 88 ++++ tests/conformance/report.py | 121 +++++ tests/conformance/runner.py | 114 +++++ tests/conformance/seed.py | 111 +++++ tests/conformance/selection.py | 130 +++++ tests/conformance/selftest/__init__.py | 2 + tests/conformance/selftest/fixture.py | 123 +++++ tests/conformance/selftest/test_backends.py | 297 ++++++++++++ tests/conformance/selftest/test_cli.py | 150 ++++++ tests/conformance/selftest/test_elfcheck.py | 89 ++++ .../conformance/selftest/test_expectations.py | 134 ++++++ tests/conformance/selftest/test_ids.py | 44 ++ tests/conformance/selftest/test_jsonc.py | 49 ++ tests/conformance/selftest/test_judge.py | 63 +++ tests/conformance/selftest/test_model.py | 53 +++ tests/conformance/selftest/test_payload.py | 231 +++++++++ tests/conformance/selftest/test_report.py | 90 ++++ tests/conformance/selftest/test_runner.py | 72 +++ tests/conformance/selftest/test_seed.py | 98 ++++ tests/conformance/selftest/test_selection.py | 68 +++ tests/lib/qemu-ssh.sh | 3 +- tests/qemu-runner.sh | 99 +++- tests/test-qemu-runner.sh | 128 +++++ 46 files changed, 4343 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/conformance.yml create mode 100644 mk/conformance.mk create mode 100755 scripts/conformance create mode 100644 tests/conformance/__init__.py create mode 100644 tests/conformance/backends/__init__.py create mode 100644 tests/conformance/backends/base.py create mode 100644 tests/conformance/backends/elfuse.py create mode 100644 tests/conformance/backends/proc.py create mode 100644 tests/conformance/backends/qemu.py create mode 100644 tests/conformance/backends/ssh.py create mode 100644 tests/conformance/cli.py create mode 100644 tests/conformance/elfcheck.py create mode 100644 tests/conformance/expectations.py create mode 100644 tests/conformance/ids.py create mode 100644 tests/conformance/jsonc.py create mode 100644 tests/conformance/judge.py create mode 100644 tests/conformance/model.py create mode 100644 tests/conformance/payload.py create mode 100644 tests/conformance/providers/__init__.py create mode 100644 tests/conformance/providers/base.py create mode 100644 tests/conformance/report.py create mode 100644 tests/conformance/runner.py create mode 100644 tests/conformance/seed.py create mode 100644 tests/conformance/selection.py create mode 100644 tests/conformance/selftest/__init__.py create mode 100644 tests/conformance/selftest/fixture.py create mode 100644 tests/conformance/selftest/test_backends.py create mode 100644 tests/conformance/selftest/test_cli.py create mode 100644 tests/conformance/selftest/test_elfcheck.py create mode 100644 tests/conformance/selftest/test_expectations.py create mode 100644 tests/conformance/selftest/test_ids.py create mode 100644 tests/conformance/selftest/test_jsonc.py create mode 100644 tests/conformance/selftest/test_judge.py create mode 100644 tests/conformance/selftest/test_model.py create mode 100644 tests/conformance/selftest/test_payload.py create mode 100644 tests/conformance/selftest/test_report.py create mode 100644 tests/conformance/selftest/test_runner.py create mode 100644 tests/conformance/selftest/test_seed.py create mode 100644 tests/conformance/selftest/test_selection.py create mode 100755 tests/test-qemu-runner.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7208d643..fe0a2cc9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,6 +51,7 @@ on: - '.editorconfig' - '.clang-format' - 'frama-c-stubs/**' + - '.github/workflows/conformance.yml' - '.github/workflows/lint.yml' - '.github/workflows/static-analysis.yml' - '.github/workflows/verify.yml' diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..8a5d72b4 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,122 @@ +name: Conformance + +on: + push: + branches: [main] + pull_request: + branches: [main] + merge_group: + schedule: + - cron: '17 3 * * *' + workflow_dispatch: + inputs: + scope: + type: choice + options: [pr, full] + default: pr + update_check: + type: boolean + default: false + +concurrency: + # A schedule event resolves github.ref to the default branch, so without the + # event split the nightly full run and a main push share one group and a + # second push cancels the queued nightly. + group: ${{ github.workflow }}-${{ github.event_name == 'schedule' && 'nightly' || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + CONF_SCOPE: ${{ (github.event_name == 'schedule' || inputs.scope == 'full') && 'full' || 'pr' }} + CONF_REQUIRE: 1 + +jobs: + discover: + runs-on: ubuntu-24.04 + outputs: + suites: ${{ steps.suites.outputs.names }} + steps: + - uses: actions/checkout@v7 + - id: suites + run: | + names=$(python3 scripts/conformance suites --format json | + python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin)["suites"]))') + echo "names=$names" >> "$GITHUB_OUTPUT" + + harness: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - run: python3 scripts/conformance selftest + + payload: + needs: discover + if: needs.discover.outputs.suites != '[]' + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v7 + - run: make conformance-payloads + - run: tar -C externals -cf conformance-payloads.tar payloads + - uses: actions/upload-artifact@v7 + with: + name: conformance-payloads + path: conformance-payloads.tar + if-no-files-found: error + + qemu: + needs: [discover, payload] + if: needs.discover.outputs.suites != '[]' + runs-on: [self-hosted, macOS, ARM64] + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: conformance-payloads + - run: mkdir -p externals && tar -C externals -xf conformance-payloads.tar + - run: bash tests/fetch-fixtures.sh + - run: make test-conformance BACKEND=qemu + + elfuse: + needs: [discover, payload, qemu] + if: needs.discover.outputs.suites != '[]' + runs-on: [self-hosted, macOS, ARM64] + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: conformance-payloads + - run: mkdir -p externals && tar -C externals -xf conformance-payloads.tar + - run: bash tests/fetch-fixtures.sh + - run: make elfuse + - run: make test-conformance BACKEND=elfuse + + conformance: + name: Conformance (make test-conformance) + needs: [discover, harness, payload, qemu, elfuse] + if: always() + runs-on: ubuntu-24.04 + steps: + - env: + SUITES: ${{ needs.discover.outputs.suites }} + DISCOVER: ${{ needs.discover.result }} + HARNESS: ${{ needs.harness.result }} + PAYLOAD: ${{ needs.payload.result }} + QEMU: ${{ needs.qemu.result }} + ELFUSE: ${{ needs.elfuse.result }} + run: | + [ "$DISCOVER" = success ] + [ "$HARNESS" = success ] + if [ "$SUITES" = '[]' ]; then + [ "$PAYLOAD $QEMU $ELFUSE" = 'skipped skipped skipped' ] + else + [ "$PAYLOAD $QEMU $ELFUSE" = 'success success success' ] + fi + + update-check: + if: github.event_name == 'workflow_dispatch' && inputs.update_check + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - run: python3 scripts/conformance pins check diff --git a/Makefile b/Makefile index d7a55b47..731e714c 100644 --- a/Makefile +++ b/Makefile @@ -691,6 +691,7 @@ $(BUILD_DIR)/probe: tests/fixtures/sharun/probe.c \ endif include mk/tests.mk +include mk/conformance.mk include mk/lint.mk include mk/verify.mk include mk/format.mk diff --git a/mk/conformance.mk b/mk/conformance.mk new file mode 100644 index 00000000..af30828c --- /dev/null +++ b/mk/conformance.mk @@ -0,0 +1,53 @@ +.PHONY: test-conformance-harness test-conformance test-conformance-full \ + conformance-payloads clean-payloads update-pins + +CONFORMANCE := python3 scripts/conformance +# The suite registry lives in tests/conformance/providers/__init__.py. On a +# failed discovery the marker fails every consumer instead of skipping. +CONF_SUITES ?= $(shell $(CONFORMANCE) suites || echo suite-discovery-failed) +BACKEND ?= elfuse +TEST ?= +CONF_JOBS ?= 4 +CONF_RESULTS ?= $(BUILD_DIR)/conformance +CONF_RUN = $(CONFORMANCE) run +CONF_SCOPE ?= pr +CONF_SELECT = $(if $(TEST),$(foreach id,$(TEST),--case '$(id)'),--scope $(CONF_SCOPE)) +CONF_NO_SUITES = $(if $(CONF_SUITES),,@printf "$(YELLOW)SKIP$(RESET) no conformance suites registered\n") +# foreach inserts spaces, but RUN_OPTIONAL_SKIP77 expands as a recipe line. +define conf-newline + + +endef +define conf-lane +$(foreach s,$(CONF_SUITES),$(call RUN_OPTIONAL_SKIP77,$(CONF_RUN) $(s) $(1) --backend $(BACKEND) --jobs $(CONF_JOBS) --results $(CONF_RESULTS),test-$(s)$(2))$(conf-newline)) +endef + +## Run the conformance harness selftests (hermetic) +test-conformance-harness: + @$(CONFORMANCE) selftest + +## Run every suite's CONF_SCOPE subset, or TEST=ID... (BACKEND=elfuse|qemu|all) +test-conformance: + $(CONF_NO_SUITES) + $(call conf-lane,$(CONF_SELECT),) + +## Run every suite in full, the nightly shape +test-conformance-full: + $(CONF_NO_SUITES) + $(call conf-lane,--scope full,-full) + +## Build every conformance payload under externals/payloads/ +conformance-payloads: + $(CONF_NO_SUITES) + $(foreach s,$(CONF_SUITES),$(CONFORMANCE) payload build $(s) &&) true + +## Remove the conformance payloads (they survive clean and distclean) +clean-payloads: + rm -rf externals/payloads + +UPDATE_CHECK ?= + +## Refresh the conformance pins from upstream (UPDATE_CHECK=1 to report only) +update-pins: + $(CONF_NO_SUITES) + $(foreach s,$(CONF_SUITES),$(CONFORMANCE) pins $(if $(filter 1,$(UPDATE_CHECK)),check,update) $(s) $(if $(CONF_REF_$(s)),--ref $(CONF_REF_$(s))) &&) true diff --git a/mk/tests.mk b/mk/tests.mk index 724bcf03..3c1c52ed 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -55,7 +55,7 @@ ELFUSE_HOST_NOFILE_MIN ?= $(shell bash "$(CURDIR)/tests/test-config.sh" --host-n test-sysroot-pathmax test-sysroot-corpus \ test-sysroot-name-soak check-soak \ check-name-caseexact test-sysroot-path-matrix \ - test-usage-synopsis \ + test-usage-synopsis test-qemu-runner \ probe-volume-naming perf ## Build and run the assembly hello world test @@ -344,6 +344,8 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage check-eintr-contract ch $(call run-lane,test-rosetta-cli,rosetta CLI gating) $(call run-lane,test-bench-guardrail,hot-syscall guardrail) $(call run-lane,test-sharun,sharun launcher and probe) + $(call run-lane,test-conformance-harness,conformance harness selftests) + $(call run-lane,test-qemu-runner,qemu-runner start and stop checks) ## Hot-syscall performance guardrail: ensure getpid, libc clock_gettime, ## and 1-byte /dev/urandom reads stay under their TODO ns/op ceilings. @@ -1170,6 +1172,10 @@ test-usage-synopsis: $(ELFUSE_BIN) test-gdbstub-host: $(BUILD_DIR)/test-gdbstub-host $(BUILD_DIR)/test-gdbstub-host +## Check qemu-runner.sh start reporting and stop identity, against stand-ins +test-qemu-runner: + @bash tests/test-qemu-runner.sh + ## Run GDB stub integration tests (LLDB <-> elfuse gdbstub) test-gdbstub: $(ELFUSE_BIN) $(TEST_DIR)/test-hello $(BUILD_DIR)/test-gdbstub-host $(call run-host-unit,test-gdbstub-host,buffered GDB session regression) diff --git a/scripts/conformance b/scripts/conformance new file mode 100755 index 00000000..073574a6 --- /dev/null +++ b/scripts/conformance @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tests")) + +from conformance.cli import main # noqa: E402 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/proof-scope.py b/scripts/proof-scope.py index be67c36f..23fe4d66 100755 --- a/scripts/proof-scope.py +++ b/scripts/proof-scope.py @@ -349,6 +349,7 @@ def inert_name(name): MAKEFILE_INERT_INCLUDES = { "mk/shim.mk", "mk/tests.mk", + "mk/conformance.mk", "mk/lint.mk", "mk/format.mk", "mk/help.mk", diff --git a/tests/conformance/__init__.py b/tests/conformance/__init__.py new file mode 100644 index 00000000..62860527 --- /dev/null +++ b/tests/conformance/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +EXIT_OK, EXIT_RED, EXIT_USAGE, EXIT_DRIFT, EXIT_SKIP = 0, 1, 2, 3, 77 diff --git a/tests/conformance/backends/__init__.py b/tests/conformance/backends/__init__.py new file mode 100644 index 00000000..e5077194 --- /dev/null +++ b/tests/conformance/backends/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from conformance.backends.base import Backend, BackendError + +def make(name: str, repo_root: Path, **options: Any) -> Backend: + if name == "elfuse": + from conformance.backends.elfuse import ElfuseBackend as cls + elif name == "qemu": + from conformance.backends.qemu import QemuBackend as cls + else: + raise BackendError("unknown backend %r" % (name,)) + return cls(repo_root, **options) diff --git a/tests/conformance/backends/base.py b/tests/conformance/backends/base.py new file mode 100644 index 00000000..0db0c50b --- /dev/null +++ b/tests/conformance/backends/base.py @@ -0,0 +1,81 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import contextlib +import errno +import fcntl +import os +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional + +from conformance.model import Invocation + + +class BackendError(RuntimeError): + pass + + +FIXED_ENV = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "TZ": "UTC"} +SCRATCH_NAMES = ("HOME", "TMPDIR", "TEST_TMPDIR") +KILL_WAIT_S = 30 + + +def guest_environment(scratch: str, env: Optional[Dict[str, str]] = None) -> Dict[str, str]: + scratch = os.path.abspath(scratch) + return {**FIXED_ENV, **{name: scratch for name in SCRATCH_NAMES}, **(env or {})} + + +class Backend: + name = "" + max_jobs = 0 # 0: no cap on --jobs + lock_file: Optional[Path] = None # held by serialize() when set + + def prerequisites(self) -> Optional[str]: + """Return why the backend cannot run, or None when it can.""" + return None + + def start(self) -> None: + pass + + def stop(self) -> None: + pass + + def run( + self, + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + fetch: Iterable[str] = (), + ) -> Invocation: + """Run argv in a fresh guest cwd and return artifacts in scratch.""" + raise NotImplementedError + + def guest_path(self, host_path: Path) -> str: + return str(host_path) + + @contextlib.contextmanager + def serialize(self) -> Iterator[None]: + """Take a non-blocking flock when lock_file is set.""" + if self.lock_file is None: + yield + return + try: + # The lock lives in a shared namespace, so another uid may own it. + self.lock_file.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(self.lock_file, os.O_RDWR | os.O_CREAT, 0o600) + except OSError as e: + raise BackendError("cannot open %s: %s" % (self.lock_file, e)) from None + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as e: + if e.errno not in (errno.EWOULDBLOCK, errno.EAGAIN): + raise + raise BackendError("another %s conformance session holds %s" + % (self.name, self.lock_file)) from None + yield + finally: + os.close(fd) diff --git a/tests/conformance/backends/elfuse.py b/tests/conformance/backends/elfuse.py new file mode 100644 index 00000000..b9cfb20c --- /dev/null +++ b/tests/conformance/backends/elfuse.py @@ -0,0 +1,92 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import signal +import subprocess +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +from conformance.backends import base, proc +from conformance.model import Invocation + + +# Linux programs expect an 8 MiB initial stack. +WRAPPER = ["/bin/sh", "-c", 'ulimit -c 0; ulimit -s 8192; exec "$@"', "--"] + + +def orphan_pids(ps_output: str, binary: str, group: int) -> list: + """Limit orphan cleanup to fork children from one case process group.""" + out = [] + for line in ps_output.splitlines(): + fields = line.split(None, 3) + if len(fields) < 4: + continue + pid, ppid, pgid, command = fields + if (ppid == "1" and pgid == str(group) and command.startswith(binary + " ") + and "--fork-child" in command): + out.append(int(pid)) + return out + + +class ElfuseBackend(base.Backend): + name = "elfuse" + + def __init__(self, repo_root: Path, sysroot: Optional[Path] = None, + binary: Optional[Path] = None): + self.repo_root = repo_root + self.binary = binary or repo_root / "build" / "elfuse" + self.sysroot = sysroot + # One session per user: guest /dev/shm is a per-uid host directory. + self.lock_file = Path("/tmp/elfuse-conformance-%d.lock" % os.getuid()) + + def prerequisites(self) -> Optional[str]: + if not os.access(self.binary, os.X_OK): + return "%s is absent; run: make elfuse" % self.binary + if self.sysroot is not None and not self.sysroot.is_dir(): + return "sysroot %s is absent" % self.sysroot + return None + + def argv(self, guest_argv: List[str]) -> List[str]: + out = [str(self.binary), "--timeout", "0"] + if self.sysroot is not None: + out += ["--sysroot", str(self.sysroot)] + return out + list(guest_argv) + + def run( + self, + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + fetch: Iterable[str] = (), + ) -> Invocation: + full = base.guest_environment(str(scratch), env) + inv = proc.run_local(WRAPPER + self.argv(argv), timeout_s, scratch, env=full) + if inv.pid is not None: + self.reap_orphans(inv.pid) + return inv + + def reap_orphans(self, group: int) -> None: + """Release VMs held by fork children that outlived their case.""" + try: + # The case ran in its own session, so its pid is the pgid; an + # empty group means no fork child survived and ps can be skipped. + os.killpg(group, 0) + except (ProcessLookupError, PermissionError): + return + try: + listing = subprocess.run( + ["ps", "-eo", "pid=,ppid=,pgid=,command="], + capture_output=True, + text=True, + ) + except OSError: + return + for pid in orphan_pids(listing.stdout, str(self.binary), group): + try: + os.kill(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass diff --git a/tests/conformance/backends/proc.py b/tests/conformance/backends/proc.py new file mode 100644 index 00000000..ad79366a --- /dev/null +++ b/tests/conformance/backends/proc.py @@ -0,0 +1,73 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import signal +import subprocess +import time +from pathlib import Path +from typing import Dict, List, Optional + +from conformance.backends.base import KILL_WAIT_S +from conformance.model import Invocation + + +def classify(rc: int, timed_out: bool, wall_us: int, stdout: str, stderr: str) -> Invocation: + """Interpret negative Popen return codes as signal deaths.""" + if timed_out: + return Invocation(execution="timeout", wall_us=wall_us, stdout=stdout, stderr=stderr) + if rc < 0: + return Invocation(execution="signal", wall_us=wall_us, signal=-rc, stdout=stdout, stderr=stderr) + return Invocation(execution="normal", wall_us=wall_us, exit_code=rc, stdout=stdout, stderr=stderr) + + +def _kill_and_reap(proc: subprocess.Popen) -> Optional[int]: + """Bound the wait for a guest stuck in an uninterruptible state.""" + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + return proc.wait(timeout=KILL_WAIT_S) + except subprocess.TimeoutExpired: + return None + + +def run_local( + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + stdout_name: str = "stdout", +) -> Invocation: + """Run argv in a new session so timeout cleanup reaches its group.""" + scratch.mkdir(parents=True, exist_ok=True) + out_path, err_path = scratch / stdout_name, scratch / "stderr" + started = time.monotonic() + with open(out_path, "wb") as out, open(err_path, "wb") as err, \ + open(os.devnull, "rb") as feed: + try: + proc = subprocess.Popen(argv, cwd=str(scratch), env=env, stdin=feed, + stdout=out, stderr=err, start_new_session=True) + except OSError as e: + err.write(("cannot spawn %s: %s\n" % (argv[0], e)).encode()) + return Invocation(execution="transport", wall_us=int((time.monotonic() - started) * 1_000_000), + stdout=str(out_path), stderr=str(err_path)) + timed_out = False + try: + rc = proc.wait(timeout=timeout_s) + except subprocess.TimeoutExpired: + timed_out = True + rc = _kill_and_reap(proc) + if rc is None: + err.write(("process group %d was not reaped after SIGKILL\n" % proc.pid).encode()) + wall_us = int((time.monotonic() - started) * 1_000_000) + if rc is None: + inv = Invocation(execution="transport", wall_us=wall_us, + stdout=str(out_path), stderr=str(err_path)) + else: + inv = classify(rc, timed_out, wall_us, str(out_path), str(err_path)) + inv.pid = proc.pid # start_new_session makes pid the process-group id + return inv diff --git a/tests/conformance/backends/qemu.py b/tests/conformance/backends/qemu.py new file mode 100644 index 00000000..b16423e6 --- /dev/null +++ b/tests/conformance/backends/qemu.py @@ -0,0 +1,121 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +from conformance.backends import base +from conformance.backends.ssh import SshSession +from conformance.model import Invocation + +FIXTURES = ("kernel/vmlinuz-virt", "initramfs.cpio.gz", "keys/ssh_key") + +RUNNER_TIMEOUT_S = 600 +# Match the environment passed to qemu-runner.sh. +RUNNER_PATH = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" + + +def parse_state(text: str) -> Dict[str, str]: + out = {} + for line in text.splitlines(): + key, sep, value = line.partition("=") + if sep: + out[key.strip()] = value.strip() + return out + + +class QemuBackend(base.Backend): + name = "qemu" + max_jobs = 1 # the reference accepts one command at a time + + def __init__(self, repo_root: Path, mem_mib: int = 2048, + runner: Optional[Path] = None, state_dir: Optional[Path] = None): + self.repo_root = repo_root + self.mem_mib = mem_mib + self.runner = runner or repo_root / "tests" / "qemu-runner.sh" + self.state_file = (state_dir or repo_root / "build" / "conformance") / "qemu.state" + # One session per checkout: start() reaps whatever the state file names. + self.lock_file = self.state_file.with_suffix(".lock") + self.session: Optional[SshSession] = None + + def prerequisites(self) -> Optional[str]: + fixtures = self.repo_root / "externals" / "test-fixtures" + missing = [f for f in FIXTURES if not (fixtures / f).is_file()] + if missing: + return "QEMU fixtures missing (%s); run: bash tests/fetch-fixtures.sh" % ", ".join(missing) + if shutil.which("qemu-system-aarch64", path=RUNNER_PATH) is None: + return ("qemu-system-aarch64 is not on the runner PATH (%s); " + "run: brew install qemu" % RUNNER_PATH) + return None + + def _runner(self, verb: str) -> subprocess.CompletedProcess: + self.state_file.parent.mkdir(parents=True, exist_ok=True) + with subprocess.Popen( + ["bash", str(self.runner), verb, "--state-file", str(self.state_file)], + cwd=str(self.repo_root), + env={**{k: v for k, v in os.environ.items() if k.startswith("QEMU_") or k == "TMPDIR"}, + "PATH": RUNNER_PATH, "QEMU_MEM": str(self.mem_mib), "HOME": str(Path.home())}, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) as run: + try: + out, _ = run.communicate(timeout=RUNNER_TIMEOUT_S) + except subprocess.TimeoutExpired: + # SIGTERM, not SIGKILL, so the runner's EXIT trap reaps the VM. + run.terminate() + try: + run.communicate(timeout=base.KILL_WAIT_S) + except subprocess.TimeoutExpired: + # Popen.__exit__ waits unconditionally; kill or hang here. + run.kill() + run.communicate() + raise base.BackendError("qemu-runner %s did not return in %ds" % (verb, RUNNER_TIMEOUT_S)) from None + return subprocess.CompletedProcess(run.args, run.returncode, out) + + def start(self) -> None: + # A state file from an interrupted run names a VM still up; reap it + # first or this start would overwrite the only record of it. + self.stop() + done = self._runner("start") + if done.returncode != 0: + raise base.BackendError("qemu-runner start failed:\n%s" % done.stdout) + try: + state = parse_state(self.state_file.read_text()) if self.state_file.exists() else {} + if "port" not in state or "key" not in state: + raise base.BackendError("qemu-runner wrote no port/key to %s" % self.state_file) + self.session = SshSession(int(state["port"]), Path(state["key"])) + except (base.BackendError, ValueError, OSError) as e: + self.stop() + raise base.BackendError(str(e)) from None + + def stop(self) -> None: + if self.session is not None or self.state_file.exists(): + done = self._runner("stop") + if done.returncode != 0: + raise base.BackendError("qemu-runner stop failed:\n%s" % done.stdout) + self.session = None + + def guest_path(self, host_path: Path) -> str: + try: + rel = Path(host_path).resolve().relative_to(self.repo_root.resolve()) + except ValueError: + raise base.BackendError( + "%s is outside the repo root, unreachable over the 9p share" % host_path + ) from None + return "/mnt/host/%s" % rel.as_posix() + + def run( + self, + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + fetch: Iterable[str] = (), + ) -> Invocation: + if self.session is None: + raise base.BackendError("qemu backend is not started") + return self.session.run(argv, timeout_s, scratch, env=env, fetch=fetch) diff --git a/tests/conformance/backends/ssh.py b/tests/conformance/backends/ssh.py new file mode 100644 index 00000000..65db5447 --- /dev/null +++ b/tests/conformance/backends/ssh.py @@ -0,0 +1,110 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# Alpine's initramfs has no sftp-server, so files return through ssh cat. + +from __future__ import annotations + +import os +import shlex +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +from conformance.backends import base, proc +from conformance.model import Invocation + +SENTINEL = "__CONF_RC=" +DIR_MARK = " __CONF_DIR=" +TRANSPORT_SLACK_S = 15 + + +class SshSession: + def __init__(self, port: int, key: Path, host: str = "127.0.0.1", user: str = "root", + ssh: str = "ssh"): + self.port, self.key, self.host, self.user = port, key, host, user + self.ssh = ssh + + def options(self) -> List[str]: + # The shell lanes spell the same list in tests/lib/qemu-ssh.sh. + return [ + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=10", + "-o", "ServerAliveInterval=10", + "-o", "ServerAliveCountMax=6", + "-i", str(self.key), + ] + + def ssh_argv(self, script: str) -> List[str]: + return [self.ssh] + self.options() + ["-p", str(self.port), + "%s@%s" % (self.user, self.host), script] + + @staticmethod + def remote_script(argv: List[str], timeout_s: int, env: Dict[str, str], + cwd: Optional[str], cleanup: bool = False) -> str: + """Build the guest command with an isolated cwd and fixed environment.""" + exports = "export %s;" % " ".join('%s="$PWD"' % n for n in base.SCRATCH_NAMES) + "".join( + " export %s=%s;" % (k, shlex.quote(v)) for k, v in sorted({**base.FIXED_ENV, **env}.items())) + enter = "cd %s" % shlex.quote(cwd) if cwd else 'd=$(mktemp -d /tmp/conf.XXXXXX) && cd "$d"' + if cleanup: + enter += " && trap 'cd / && rm -rf \"$d\"' EXIT" + return ( + "%s && { %s /usr/bin/timeout -s KILL %d %s; rc=$?; " + 'printf "\\n%s%%s%s%%s\\n" "$rc" "$PWD"; }' + % (enter, exports, timeout_s, shlex.join(argv), SENTINEL, DIR_MARK) + ) + + @staticmethod + def parse_sentinel(text: str) -> Optional[tuple]: + # DIR_MARK terminates rc and detects a partial sentinel write. + for line in reversed(text.splitlines()): + if line.startswith(SENTINEL): + head, mark, tail = line.partition(DIR_MARK) + field = head[len(SENTINEL):].split() + if not mark or not field or not field[0].lstrip("-").isdigit(): + return None + return int(field[0]), tail + return None + + def run(self, argv: List[str], timeout_s: int, scratch: Path, + env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, + fetch: Iterable[str] = ()) -> Invocation: + cleanup = not cwd and not fetch + script = self.remote_script(argv, timeout_s, env or {}, cwd, cleanup) + inv = proc.run_local(self.ssh_argv(script), timeout_s + TRANSPORT_SLACK_S, scratch, + stdout_name="stdout.raw") + raw_path, out_path = Path(inv.stdout), scratch / "stdout" + raw = raw_path.read_bytes() + parsed = self.parse_sentinel(raw.decode("utf-8", "replace")) + # A clean ssh exit vouches for the sentinel; a lost tail can leave a + # guest-printed lookalike as the last line. + if parsed is None or inv.execution != "normal" or inv.exit_code != 0: + raw_path.replace(out_path) + # An uninterruptible guest can outlive both timeout and SIGKILL. + return Invocation(execution="timeout" if inv.execution == "timeout" else "transport", + wall_us=inv.wall_us, + stdout=str(out_path), stderr=inv.stderr) + rc, guest_dir = parsed + os.truncate(raw_path, max(raw.rfind(SENTINEL.encode()) - 1, 0)) + raw_path.replace(out_path) + if guest_dir: + lost = [name for name in fetch + if not self.copy_from("%s/%s" % (guest_dir, name), scratch / name)] + if not cwd and fetch: + proc.run_local(self.ssh_argv("rm -rf %s" % shlex.quote(guest_dir)), 30, + scratch / ".cleanup") + if lost: + return Invocation(execution="transport", wall_us=inv.wall_us, + stdout=str(out_path), stderr=inv.stderr) + timed_out = rc == 137 and inv.wall_us >= timeout_s * 1_000_000 + # The shell status cannot distinguish signal death from exit(128+n). + return proc.classify(rc, timed_out, inv.wall_us, str(out_path), inv.stderr) + + def copy_from(self, remote: str, local: Path) -> bool: + inv = proc.run_local(self.ssh_argv("cat %s" % shlex.quote(remote)), 120, + local.parent / ".fetch", stdout_name=local.name + ".part") + if inv.exit_code != 0: + return False + Path(inv.stdout).replace(local) + return True diff --git a/tests/conformance/cli.py b/tests/conformance/cli.py new file mode 100644 index 00000000..0b7d326d --- /dev/null +++ b/tests/conformance/cli.py @@ -0,0 +1,447 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import contextlib +import datetime +import json +import os +import sys +import time +import unittest +from pathlib import Path +from typing import Callable, Iterator, List, Optional + +from conformance import EXIT_DRIFT, EXIT_OK, EXIT_RED, EXIT_SKIP, EXIT_USAGE +from conformance import backends, expectations, jsonc, payload, providers, report +from conformance import runner, seed, selection +from conformance.backends.base import BackendError +from conformance.model import Status +from conformance.providers.base import Provider, ProviderError + +REPO_ROOT = Path(__file__).resolve().parents[2] +BACKENDS = ("elfuse", "qemu", "all") + + +class Stop(Exception): + def __init__(self, code: int): + super().__init__(code) + self.code = code + + +class Cli: + def __init__(self, repo_root: Path, out: Callable[[str], None] = print, + err: Optional[Callable[[str], None]] = None): + self.repo_root = repo_root + self.out = out + self.err = err or out + + def fail(self, message: str) -> None: + self.err("conformance: " + message) + + def skip(self, args: argparse.Namespace, message: str) -> int: + self.fail(message) + required = args.require or os.environ.get("CONF_REQUIRE") == "1" + return EXIT_USAGE if required else EXIT_SKIP + + def provider(self, name: str) -> Provider: + return providers.make(name, self.repo_root) + + @staticmethod + def backend_names(name: str) -> List[str]: + return ["qemu", "elfuse"] if name == "all" else [name] + + def make_backend(self, args: argparse.Namespace, provider: Provider, name: str, + verify_payload: bool = False) -> backends.Backend: + absent = provider.prerequisites(name) + if absent: + raise Stop(self.skip(args, absent)) + if verify_payload: + try: + payload.verify(provider.payload_root(), provider.fingerprint()) + except payload.PayloadError as e: + self.fail(str(e)) + raise Stop(EXIT_USAGE) from None + try: + backend = backends.make( + name, self.repo_root, **provider.backend_options(name) + ) + except BackendError as e: + self.fail(str(e)) + raise Stop(EXIT_USAGE) from None + absent = backend.prerequisites() + if absent: + raise Stop(self.skip(args, absent)) + return backend + + @contextlib.contextmanager + def started(self, backend: backends.Backend) -> Iterator[None]: + with backend.serialize(): + # A backend that cannot start is red: an absent prerequisite is + # what prerequisites() reports, before anything is started. + backend.start() + try: + yield + except BaseException: + # A failing stop must not replace the in-flight error. + try: + backend.stop() + except BackendError as e: + self.fail("backend stop failed: %s" % e) + raise + backend.stop() + + def emit_list(self, args: argparse.Namespace, kind: str, key: str, + items: List, lines: List[str], **extra) -> int: + if args.format == "json": + self.out(json.dumps({"schema_version": 1, "kind": kind, key: items, + **extra}, sort_keys=True)) + else: + for line in lines: + self.out(line) + return EXIT_OK + + def suites(self, args: argparse.Namespace) -> int: + names = sorted(providers.REGISTRY) + return self.emit_list(args, "suite-list", "suites", names, names) + + def list_cases(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + name = "elfuse" if args.backend == "all" else args.backend + try: + backend = self.make_backend(args, provider, name) + with self.started(backend): + cases = provider.enumerate( + backend, provider.selection.groups(args.scope) + ) + except Stop as e: + return e.code + except BackendError as e: + self.fail("backend error: %s" % e) + return EXIT_RED + return self.emit_list( + args, "case-list", "cases", + [{"id": c.id, "group": c.group, "scope": c.scope, + "timeout_s": c.timeout_s} for c in cases], + [c.id for c in cases], suite=args.suite, scope=args.scope) + + def run(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + codes = [self.run_one(args, provider, name) + for name in self.backend_names(args.backend)] + for code in (EXIT_RED, EXIT_USAGE, EXIT_SKIP): + if code in codes: + return code + return EXIT_OK + + def run_one(self, args: argparse.Namespace, provider: Provider, + backend_name: str) -> int: + try: + backend = self.make_backend( + args, provider, backend_name, verify_payload=True + ) + exps = expectations.load( + provider.name, backend_name, provider.expectations_dir + ) + except Stop as e: + return e.code + except (expectations.ExpectationError, jsonc.JsoncError) as e: + self.fail(str(e)) + return EXIT_USAGE + selected_scope = "full" if args.case else args.scope + result_scope = "cases" if args.case else args.scope + results_dir = self.results_dir(args, provider.name, backend_name) + started = time.monotonic() + try: + with self.started(backend): + cases = provider.enumerate( + backend, provider.selection.groups(selected_scope) + ) + if args.case: + chosen, errors = selection.resolve_ids( + args.case, [c.id for c in cases], provider.name + ) + for error in errors: + self.fail(error) + if errors: + return EXIT_USAGE + wanted = set(chosen) + cases = [case for case in cases if case.id in wanted] + if args.dry_run: + for case in cases: + self.out(case.id) + return EXIT_OK + log = self.err if args.verbose else (lambda _: None) + results = runner.run_lane( + provider, backend, cases, exps, results_dir, args.jobs, + not args.no_retry, args.bootstrap, log + ) + meta = { + "suite": provider.name, + "backend": backend_name, + "scope": result_scope, + "cases": list(args.case), + "started": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(timespec="seconds"), + "elapsed_s": round(time.monotonic() - started, 3), + "bootstrap": args.bootstrap, + "argv": args.argv, + } + # Written before stop(), so a failing teardown cannot lose the lane. + report.write(results_dir, meta, results) + for line in report.summary_lines(meta, results, results_dir): + self.out(line) + except Stop as e: + return e.code + except BackendError as e: + self.fail("backend error: %s" % e) + return EXIT_RED + if result_scope == "full" and not args.bootstrap: + stale = exps.stale([case.id for case in cases]) + for problem in stale: + self.fail("stale expectation, " + problem) + if stale: + return EXIT_USAGE + if args.bootstrap: + return EXIT_RED if any( + case.status is Status.ERROR for case in results + ) else EXIT_OK + return EXIT_OK if report.gate(results) == "green" else EXIT_RED + + @staticmethod + def results_dir(args: argparse.Namespace, suite: str, backend: str) -> Path: + stamp = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y%m%dT%H%M%SZ" + ) + return Path(args.results) / suite / backend / ( + "%s-%d" % (stamp, os.getpid()) + ) + + def payload_fingerprint(self, args: argparse.Namespace) -> int: + self.out(self.provider(args.suite).fingerprint()) + return EXIT_OK + + def payload_verify(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + try: + payload.verify( + provider.payload_root(), args.fingerprint or provider.fingerprint() + ) + except payload.PayloadError as e: + self.fail(str(e)) + return EXIT_RED + self.out("%s: verified" % provider.payload_root()) + return EXIT_OK + + def payload_build(self, args: argparse.Namespace) -> int: + try: + self.provider(args.suite).build_payload(force=args.force) + except (payload.PayloadError, ProviderError) as e: + self.fail(str(e)) + if isinstance(e, payload.PayloadError) and e.kind != "config": + return EXIT_RED + return EXIT_USAGE + return EXIT_OK + + def selection_sync(self, args: argparse.Namespace) -> int: + lines = self.provider(args.suite).regen_selection(check=args.check) + for line in lines: + (self.fail if args.check else self.out)(line) + if not lines: + self.out("%s: selection is current" % args.suite) + return EXIT_DRIFT if args.check and lines else EXIT_OK + + def expectations_check(self, args: argparse.Namespace) -> int: + names = [args.suite] if args.suite else sorted(providers.REGISTRY) + problems: List[str] = [] + for name in names: + root = self.provider(name).expectations_dir + if root.is_dir(): + problems.extend(expectations.lint(root)) + for problem in problems: + self.fail(problem) + if not problems: + self.out("expectations: valid") + return EXIT_USAGE if problems else EXIT_OK + + def expectations_seed(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + meta, cases = report.load(Path(args.results)) + if meta.get("suite") != provider.name: + self.fail("%s does not contain %s results" % ( + args.results, provider.name + )) + return EXIT_USAGE + reason = args.reason or seed.default_reason( + Path(args.results), str(meta.get("started", ""))[:10] + ) + actions = seed.propose( + cases, reason, bool(meta.get("bootstrap")), + whole_groups=meta.get("scope") != "cases" + ) + if not actions: + self.out("expectations: no changes") + return EXIT_OK + if not args.write: + self.out(seed.format_actions(actions).rstrip("\n")) + return EXIT_OK + backend = meta.get("backend") + if not backend: + self.fail("results name no backend") + return EXIT_USAGE + leaf = expectations.leaf_path( + provider.expectations_dir, provider.name, backend + ) + seed.append(leaf, actions) + problems = expectations.lint(provider.expectations_dir, seeded_ok=True) + for problem in problems: + self.fail(problem) + self.out("%s: appended %d actions" % (leaf, len(actions))) + return EXIT_USAGE if problems else EXIT_OK + + def pins(self, args: argparse.Namespace) -> int: + names = [args.suite] if args.suite else sorted(providers.REGISTRY) + codes = [payload.refresh(self.provider(name), args.ref, args.check, + self.out, self.fail) for name in names] + for code in (EXIT_USAGE, EXIT_DRIFT): + if code in codes: + return code + return EXIT_OK + + def report(self, args: argparse.Namespace) -> int: + root = Path(args.results) + if args.format == "markdown": + text, red = report.markdown(root) + self.out(text.rstrip("\n")) + return EXIT_RED if red else EXIT_OK + meta, cases = report.load(root) + if args.format == "json": + self.out(json.dumps(report.document(meta, cases), sort_keys=True)) + else: + for line in report.summary_lines(meta, cases, root): + self.out(line) + return EXIT_OK if report.gate(cases) == "green" else EXIT_RED + + def selftest(self, args: argparse.Namespace) -> int: + suite = unittest.defaultTestLoader.discover( + str(self.repo_root / "tests" / "conformance" / "selftest"), + top_level_dir=str(self.repo_root / "tests"), + ) + result = unittest.TextTestRunner(verbosity=1).run(suite) + return EXIT_OK if result.wasSuccessful() else EXIT_RED + + +def common_run(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--backend", choices=BACKENDS, default="elfuse") + parser.add_argument("--results", default="build/conformance") + parser.add_argument("--jobs", type=int, default=1) + parser.add_argument("--bootstrap", action="store_true") + parser.add_argument("--require", action="store_true") + parser.add_argument("--no-retry", action="store_true") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("-v", "--verbose", action="store_true") + + +def command(parent: argparse._SubParsersAction, name: str, handler: str): + parser = parent.add_parser(name) + parser.set_defaults(handler=handler) + return parser + + +def family(parent: argparse._SubParsersAction, name: str): + parser = parent.add_parser(name) + return parser.add_subparsers(dest=name + "_command", required=True) + + +def suite_arg(parser: argparse.ArgumentParser, suites: List[str]) -> None: + parser.add_argument("suite", choices=suites) + + +def build_parser(suites: List[str]) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="conformance", + description="Run registered Linux test suites on elfuse or QEMU.", + ) + top = parser.add_subparsers(dest="command", required=True) + p = command(top, "suites", "suites") + p.add_argument("--format", choices=("text", "json"), default="text") + + p = command(top, "list", "list_cases") + suite_arg(p, suites) + p.add_argument("--scope", choices=selection.SCOPES, default="full") + p.add_argument("--format", choices=("text", "json"), default="text") + p.add_argument("--backend", choices=BACKENDS, default="elfuse") + p.add_argument("--require", action="store_true") + + p = command(top, "run", "run") + suite_arg(p, suites) + p.add_argument("--scope", choices=selection.SCOPES, default="pr") + p.add_argument("--case", action="append", default=[]) + common_run(p) + + sub = family(top, "payload") + p = command(sub, "fingerprint", "payload_fingerprint") + suite_arg(p, suites) + p = command(sub, "build", "payload_build") + suite_arg(p, suites) + p.add_argument("--force", action="store_true") + p = command(sub, "verify", "payload_verify") + suite_arg(p, suites) + p.add_argument("--fingerprint") + + sub = family(top, "selection") + for name, check in (("check", True), ("update", False)): + p = command(sub, name, "selection_sync") + p.set_defaults(check=check) + suite_arg(p, suites) + + sub = family(top, "expectations") + p = command(sub, "check", "expectations_check") + p.add_argument("suite", nargs="?", choices=suites) + p = command(sub, "seed", "expectations_seed") + suite_arg(p, suites) + p.add_argument("results") + p.add_argument("--reason") + p.add_argument("--write", action="store_true") + + sub = family(top, "pins") + p = command(sub, "check", "pins") + p.set_defaults(check=True) + p.add_argument("suite", nargs="?", choices=suites) + p.add_argument("--ref") + p = command(sub, "update", "pins") + p.set_defaults(check=False) + suite_arg(p, suites) + p.add_argument("--ref") + + p = command(top, "report", "report") + p.add_argument("results") + p.add_argument("--format", choices=("text", "markdown", "json"), + default="text") + command(top, "selftest", "selftest") + return parser + + +def main(argv: Optional[List[str]] = None, repo_root: Path = REPO_ROOT, + out: Callable[[str], None] = print, + err: Optional[Callable[[str], None]] = None) -> int: + parser = build_parser(sorted(providers.REGISTRY)) + argv = sys.argv[1:] if argv is None else list(argv) + args = parser.parse_args(argv) + args.argv = argv + if err is None: + err = lambda message: print(message, file=sys.stderr) + cli = Cli(repo_root, out, err) + try: + return getattr(cli, args.handler)(args) + except (ProviderError, selection.SelectionError, payload.PinError, + jsonc.JsoncError, expectations.ExpectationError, report.ReportError, + seed.SeedError) as e: + cli.fail(str(e)) + return EXIT_USAGE + except NotImplementedError as e: + cli.fail(str(e) or "operation is not supported") + return EXIT_USAGE diff --git a/tests/conformance/elfcheck.py b/tests/conformance/elfcheck.py new file mode 100644 index 00000000..3c1ff2e3 --- /dev/null +++ b/tests/conformance/elfcheck.py @@ -0,0 +1,90 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import struct +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional, Tuple + +EM_AARCH64 = 183 +PT_LOAD, PT_DYNAMIC, PT_INTERP = 1, 2, 3 +DT_NULL, DT_NEEDED, DT_STRTAB = 0, 1, 5 + + +class ElfError(ValueError): + pass + + +@dataclass +class DynamicInfo: + machine: int + interp: Optional[str] = None + needed: List[str] = field(default_factory=list) + has_load: bool = False + + +def _headers(data: bytes, path: Path) -> Tuple[int, list]: + if len(data) < 64 or data[:4] != b"\x7fELF": + raise ElfError("%s: not an ELF file" % path) + if data[4] != 2 or data[5] != 1: + raise ElfError("%s: not ELF64 little-endian" % path) + machine = struct.unpack_from(" len(data): + raise ElfError("%s: program header table out of range" % path) + phdrs = [struct.unpack_from(" Optional[int]: + for p_type, _, p_offset, p_vaddr, _, p_filesz, _, _ in phdrs: + if p_type == PT_LOAD and p_vaddr <= vaddr < p_vaddr + p_filesz: + return p_offset + (vaddr - p_vaddr) + return None + + +def _cstring(data: bytes, offset: int, path: Path) -> str: + end = data.find(b"\0", offset) + if end < 0: + raise ElfError("%s: string out of range" % path) + return data[offset:end].decode("ascii", "replace") + + +def read_dynamic(path: Path) -> DynamicInfo: + data = path.read_bytes() + machine, phdrs = _headers(data, path) + info = DynamicInfo(machine) + for p_type, _, p_offset, _, _, p_filesz, _, _ in phdrs: + if p_type == PT_LOAD: + info.has_load = True + elif p_type == PT_INTERP: + info.interp = _cstring(data, p_offset, path) + elif p_type == PT_DYNAMIC: + if p_offset + p_filesz > len(data): + raise ElfError("%s: dynamic segment out of range" % path) + entries = [struct.unpack_from(" None: + info = read_dynamic(path) + if info.machine != EM_AARCH64: + raise ElfError("%s: machine %d is not AArch64" % (path, info.machine)) + if not info.has_load: + raise ElfError("%s: no PT_LOAD segment" % path) + if info.interp is not None: + raise ElfError("%s: has PT_INTERP %s, not static" % (path, info.interp)) + if info.needed: + raise ElfError("%s: needs %s, not static" % (path, ", ".join(info.needed))) diff --git a/tests/conformance/expectations.py b/tests/conformance/expectations.py new file mode 100644 index 00000000..d6f4d9d4 --- /dev/null +++ b/tests/conformance/expectations.py @@ -0,0 +1,214 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from conformance import ids, jsonc + +ACTION_TYPES = ("expect_pass", "expect_failure", "expect_conf", "skip", "quarantine") +_ACTION_KEYS = {"type", "matchers", "reason", "since", "tracking"} +_TRACKING_RE = re.compile(r"^(#\d+|https?://\S+)$") +_SINCE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +SEEDED_PREFIX = "seeded from " +FLAKY = "flaky.jsonc" + + +class ExpectationError(ValueError): + pass + + +@dataclass(frozen=True) +class Action: + type: str + matchers: tuple + reason: str + source: str + since: str = "" + tracking: str = "" + + +@dataclass(frozen=True) +class Resolution: + type: str + reason: str + source: str + matcher: str + quarantined: bool = False + + def to_dict(self) -> Dict[str, Any]: + return { + "type": self.type, + "reason": self.reason, + "source": self.source, + "matcher": self.matcher, + "quarantined": self.quarantined, + } + + +class Expectations: + def __init__(self, suite: str, actions: List[Action]): + self.suite = suite + self.actions = list(actions) + first = next((a for a in actions if a.type != "quarantine"), None) + if first is None or first.type != "expect_pass" or first.matchers != ("*",): + raise ExpectationError( + 'the first effective action must be expect_pass on "*"' + ) + + def resolve(self, test_id: str) -> Resolution: + chosen: Optional[Action] = None + chosen_matcher = "" + quarantined = False + for action in self.actions: + for m in action.matchers: + if ids.matches(m, test_id): + if action.type == "quarantine": + quarantined = True + else: + chosen, chosen_matcher = action, m + assert chosen is not None + return Resolution( + type=chosen.type, + reason=chosen.reason, + source=chosen.source, + matcher=chosen_matcher, + quarantined=quarantined, + ) + + def stale(self, known: Iterable[str]) -> List[str]: + universe = list(known) + out = [] + for action in self.actions: + for m in action.matchers: + if m == "*" or ids.suite_of(m) != self.suite: + continue + if not any(ids.matches(m, i) for i in universe): + out.append("%s: %r matches no test" % (action.source, m)) + return out + + +def _check_text(text: str, where: str) -> None: + if "\u2014" in text: + raise ExpectationError("%s: em dash in text" % where) + + +def _parse_action(doc: Any, where: str, suite: str) -> Action: + if not isinstance(doc, dict): + raise ExpectationError("%s: action is not an object" % where) + unknown = set(doc) - _ACTION_KEYS + if unknown: + raise ExpectationError("%s: unknown keys %s" % (where, sorted(unknown))) + kind = doc.get("type") + if kind not in ACTION_TYPES: + raise ExpectationError("%s: unknown action type %r" % (where, kind)) + matchers = doc.get("matchers") + if not isinstance(matchers, list) or not matchers: + raise ExpectationError("%s: matchers must be a non-empty list" % where) + if any(not isinstance(m, str) for m in matchers): + raise ExpectationError("%s: matchers must be strings" % where) + if matchers != sorted(matchers): + raise ExpectationError("%s: matchers are not sorted" % where) + if len(set(matchers)) != len(matchers): + raise ExpectationError("%s: duplicate matcher" % where) + for m in matchers: + if m == "*": + if kind != "expect_pass": + raise ExpectationError('%s: "*" is legal only on expect_pass' % where) + continue + if ids.suite_of(m) != suite: + raise ExpectationError("%s: matcher %r is not in suite %s" % (where, m, suite)) + if not ids.is_valid(m.replace("*", "x").replace("?", "x")): + raise ExpectationError("%s: matcher %r is not an id pattern" % (where, m)) + reason = doc.get("reason", "") + if not isinstance(reason, str): + raise ExpectationError("%s: reason must be a string" % where) + if kind != "expect_pass" and not reason.strip(): + raise ExpectationError("%s: %s needs a reason" % (where, kind)) + _check_text(reason, where) + since = doc.get("since", "") + if since and not (isinstance(since, str) and _SINCE_RE.match(since)): + raise ExpectationError("%s: since must be YYYY-MM-DD" % where) + tracking = doc.get("tracking", "") + if tracking and not (isinstance(tracking, str) and _TRACKING_RE.match(tracking)): + raise ExpectationError("%s: tracking must be #N or a URL" % where) + return Action(kind, tuple(matchers), reason, where, since, tracking) + + +def read_file(path: Path, suite: Optional[str] = None, + seen: Optional[List[Path]] = None) -> List[Action]: + """With no suite, each action's suite comes from its first matcher.""" + seen = list(seen or []) + if path in seen: + raise ExpectationError("%s: include cycle" % path) + if len(seen) > 8: + raise ExpectationError("%s: include chain too deep" % path) + seen.append(path) + out: List[Action] = [] + for where, entry in _entries(path): + if isinstance(entry, dict) and "include" in entry: + if suite is None: + raise ExpectationError("%s: include is not legal in %s" % (where, FLAKY)) + if set(entry) != {"include"} or not isinstance(entry["include"], str): + raise ExpectationError("%s: include takes only a file name" % where) + out.extend(read_file(path.parent / entry["include"], suite, seen)) + continue + if suite is None: + matchers = entry.get("matchers") if isinstance(entry, dict) else None + first = matchers[0] if isinstance(matchers, list) and matchers else "" + entry_suite = ids.suite_of(first) if isinstance(first, str) else "" + else: + entry_suite = suite + action = _parse_action(entry, where, entry_suite) + if action.type == "quarantine" and path.name != FLAKY: + raise ExpectationError("%s: quarantine is legal only in %s" % (where, FLAKY)) + if path.name == FLAKY and action.type != "quarantine": + raise ExpectationError("%s: %s holds only quarantine actions" % (where, FLAKY)) + out.append(action) + return out + + +def _entries(path: Path) -> List[Tuple[str, Any]]: + if not path.exists(): + raise ExpectationError("%s: no such file" % path) + doc = jsonc.load(path) + if not isinstance(doc, dict) or set(doc) != {"actions"}: + raise ExpectationError('%s: expected an object with only "actions"' % path) + if not isinstance(doc["actions"], list): + raise ExpectationError("%s: actions must be a list" % path) + return [("%s:%d" % (path.name, index), entry) for index, entry in enumerate(doc["actions"])] + + +def leaf_path(root: Path, suite: str, backend: str) -> Path: + return root / ("%s_%s.jsonc" % (suite, backend)) + + +def load(suite: str, backend: str, root: Path) -> Expectations: + actions = read_file(leaf_path(root, suite, backend), suite) + flaky = root / FLAKY + if flaky.exists(): + actions.extend(a for a in read_file(flaky) if ids.suite_of(a.matchers[0]) == suite) + return Expectations(suite, actions) + + +def lint(root: Path, seeded_ok: bool = False) -> List[str]: + problems: List[str] = [] + seeded: List[str] = [] + for path in sorted(root.glob("*.jsonc")): + try: + suite = None if path.name == FLAKY else path.stem.split("_", 1)[0] + actions = read_file(path, suite) + if suite and "_" in path.stem: + Expectations(suite, actions) + except (ExpectationError, jsonc.JsoncError) as e: + problems.append(str(e)) + continue + seeded.extend(a.source for a in actions if a.reason.startswith(SEEDED_PREFIX)) + if not seeded_ok: + problems += ["%s: still carries a seeded reason; triage it" % s for s in seeded] + # A base file is read again through every leaf that includes it. + return list(dict.fromkeys(problems)) diff --git a/tests/conformance/ids.py b/tests/conformance/ids.py new file mode 100644 index 00000000..b8951a78 --- /dev/null +++ b/tests/conformance/ids.py @@ -0,0 +1,57 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import fnmatch +import hashlib +import re +from typing import Iterable, List, Optional, Tuple + +_ID_RE = re.compile( + r"^(?P[a-z][a-z0-9]*):(?P[A-Za-z0-9_][A-Za-z0-9_.-]*)" + r"(?P(/[A-Za-z0-9_.-]+)*)\Z" +) +_SLUG_BASE_MAX = 200 + + +class IdError(ValueError): + pass + + +def parse(test_id: str) -> Tuple[str, str, Optional[str]]: + m = _ID_RE.match(test_id) + if not m: + raise IdError("not a canonical test id: %r" % (test_id,)) + case = m.group("case") + return m.group("suite"), m.group("group"), case[1:] if case else None + + +def is_valid(test_id: str) -> bool: + return _ID_RE.match(test_id) is not None + + +def suite_of(text: str) -> str: + head, sep, _ = text.partition(":") + return head if sep else "" + + +def group_of(test_id: str) -> str: + return parse(test_id)[1] + + +def matches(pattern: str, test_id: str) -> bool: + """Match wildcards across the full canonical id, including slashes.""" + return fnmatch.fnmatchcase(test_id, pattern) + + +def expand(patterns: Iterable[str], ids: Iterable[str]) -> List[str]: + pats = list(patterns) + return [i for i in ids if any(matches(p, i) for p in pats)] + + +def slug(test_id: str) -> str: + """Append a digest to a bounded, sanitized id.""" + digest = hashlib.sha256(test_id.encode()).hexdigest()[:8] + base = re.sub(r"[^A-Za-z0-9_.-]", "_", test_id)[:_SLUG_BASE_MAX] + return base + "-" + digest diff --git a/tests/conformance/jsonc.py b/tests/conformance/jsonc.py new file mode 100644 index 00000000..4bd7ead6 --- /dev/null +++ b/tests/conformance/jsonc.py @@ -0,0 +1,66 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +class JsoncError(ValueError): + pass + + +def strip(text: str) -> str: + out = [] + comma = None + i, n = 0, len(text) + while i < n: + c = text[i] + if c == '"': + j = i + 1 + while j < n and text[j] != '"': + j += 2 if text[j] == "\\" else 1 + out.append(text[i : j + 1]) + comma = None + i = j + 1 + elif text.startswith("//", i): + j = text.find("\n", i) + i = n if j < 0 else j + elif text.startswith("/*", i): + j = text.find("*/", i + 2) + if j < 0: + raise JsoncError("unterminated block comment") + # Preserve token separation when a block comment is removed. + out.append(" " + "\n" * text.count("\n", i, j)) + i = j + 2 + else: + if c == ",": + comma = len(out) + elif c in "]}" and comma is not None: + out[comma] = "" + comma = None + elif not c.isspace(): + comma = None + out.append(c) + i += 1 + return "".join(out) + + +def _reject(literal: str) -> Any: + raise JsoncError("%s is not JSON" % literal) + + +def loads(text: str) -> Any: + try: + return json.loads(strip(text), parse_constant=_reject) + except json.JSONDecodeError as e: + raise JsoncError("line %d: %s" % (e.lineno, e.msg)) from None + + +def load(path: Path) -> Any: + try: + return loads(path.read_text()) + except (JsoncError, OSError, UnicodeDecodeError) as e: + raise JsoncError("%s: %s" % (path, e)) from None diff --git a/tests/conformance/judge.py b/tests/conformance/judge.py new file mode 100644 index 00000000..9bedcbdc --- /dev/null +++ b/tests/conformance/judge.py @@ -0,0 +1,50 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Tuple + +from conformance.expectations import Resolution +from conformance.model import Status, Verdict + +MAX_ATTEMPTS = 3 + +_SATISFIES = { + "expect_pass": (Status.PASS, Status.WARN), + "expect_failure": (Status.FAIL, Status.BROK), + "expect_conf": (Status.CONF,), +} + + +def decide(test_id: str, status: Status, resolution: Resolution) -> Tuple[Verdict, str]: + if status is Status.ERROR: + return Verdict.ERROR, "%s: HARNESS ERROR, the case did not run" % test_id + if status is Status.SKIP: + return Verdict.FILTERED, "" + if status in (Status.TIMEOUT, Status.CRASH, Status.INCONSISTENT): + return ( + Verdict.UNEXPECTED_FAILURE, + "%s: %s, which no expectation can satisfy" % (test_id, status.value), + ) + if status in _SATISFIES[resolution.type]: + return Verdict.AS_EXPECTED, "" + if status in _SATISFIES["expect_pass"]: + return ( + Verdict.UNEXPECTED_PASS, + "%s: %s but %s expects %s (matcher %r); narrow or delete that " + "matcher in this same change" % ( + test_id, status.value, resolution.source, resolution.type, + resolution.matcher), + ) + return ( + Verdict.UNEXPECTED_FAILURE, + "%s: %s but %s expects %s (matcher %r); fix the regression or record " + "the divergence in the backend leaf" % ( + test_id, status.value, resolution.source, resolution.type, + resolution.matcher), + ) + + +def may_retry(resolution: Resolution, attempts: int) -> bool: + return resolution.quarantined and attempts < MAX_ATTEMPTS diff --git a/tests/conformance/model.py b/tests/conformance/model.py new file mode 100644 index 00000000..54b2b9df --- /dev/null +++ b/tests/conformance/model.py @@ -0,0 +1,117 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional + + +class Status(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + SKIP = "SKIP" + CONF = "CONF" + WARN = "WARN" + BROK = "BROK" + TIMEOUT = "TIMEOUT" + CRASH = "CRASH" + INCONSISTENT = "INCONSISTENT" + ERROR = "ERROR" + + +class Verdict(str, Enum): + AS_EXPECTED = "as_expected" + UNEXPECTED_FAILURE = "unexpected_failure" + UNEXPECTED_PASS = "unexpected_pass" + FLAKED = "flaked" + FILTERED = "filtered" + ERROR = "error" + + @property + def is_red(self) -> bool: + return self in ( + Verdict.UNEXPECTED_FAILURE, + Verdict.UNEXPECTED_PASS, + Verdict.ERROR, + ) + + +EXECUTIONS = ("normal", "timeout", "signal", "transport") + + +def _plain(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if dataclasses.is_dataclass(value): + return {f.name: _plain(getattr(value, f.name)) + for f in dataclasses.fields(value) if f.compare} + if isinstance(value, list): + return [_plain(v) for v in value] + if isinstance(value, dict): + return dict(value) + return value + + +class Doc: + """JSON mapping for the dataclasses below; compare=False stays out.""" + + def to_dict(self) -> Dict[str, Any]: + return _plain(self) + + @classmethod + def from_dict(cls, doc: Dict[str, Any]) -> Any: + kwargs = {f.name: _CONVERT.get(f.name, lambda v: v)(doc[f.name]) + for f in dataclasses.fields(cls) if f.compare and f.name in doc} + return cls(**kwargs) + + +@dataclass +class Invocation(Doc): + execution: str + wall_us: int + exit_code: Optional[int] = None + signal: Optional[int] = None + stdout: str = "" + stderr: str = "" + pid: Optional[int] = dataclasses.field(default=None, compare=False) + + def __post_init__(self) -> None: + if self.execution not in EXECUTIONS: + raise ValueError("unknown execution %r" % (self.execution,)) + if self.wall_us < 0: + raise ValueError("negative wall_us") + carries = {"normal": "exit_code", "signal": "signal"}.get(self.execution) + for name in ("exit_code", "signal"): + if (getattr(self, name) is not None) != (name == carries): + raise ValueError("%s execution %s carry %s" % ( + self.execution, "must" if name == carries else "cannot", name)) + + +@dataclass +class Attempt(Doc): + status: Status + invocation: Invocation + detail: str = "" + + +@dataclass +class CaseResult(Doc): + id: str + suite: str + backend: str + status: Status + verdict: Verdict + expectation: Dict[str, Any] = field(default_factory=dict) + attempts: List[Attempt] = field(default_factory=list) + detail: str = "" + + +_CONVERT = { + "status": Status, + "verdict": Verdict, + "invocation": Invocation.from_dict, + "attempts": lambda docs: [Attempt.from_dict(a) for a in docs], +} diff --git a/tests/conformance/payload.py b/tests/conformance/payload.py new file mode 100644 index 00000000..010e69e3 --- /dev/null +++ b/tests/conformance/payload.py @@ -0,0 +1,233 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import re +import tempfile +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence + +from conformance import EXIT_DRIFT, EXIT_OK, EXIT_USAGE + +MANIFEST = "manifest.json" +_HEX = {"hex40": re.compile(r"^[0-9a-f]{40}$"), "hex64": re.compile(r"^[0-9a-f]{64}$")} + + +class PayloadError(Exception): + def __init__(self, kind: str, message: str): + super().__init__(message) + self.kind = kind + + +class PinError(ValueError): + pass + + +class UpdateError(RuntimeError): + pass + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def fingerprint(pin_section: Dict[str, Any], files: Sequence[Path], flavor: str = "") -> str: + """Hash by basename and in the given order, so inputs need distinct names.""" + h = hashlib.sha256() + h.update(json.dumps(pin_section, sort_keys=True).encode()) + for path in files: + h.update(path.name.encode() + b"\0") + h.update(sha256_file(path).encode() + b"\0") + h.update(flavor.encode()) + return h.hexdigest() + + +def _walk(root: Path) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + dirnames.sort() + for name in list(dirnames): + path = Path(dirpath) / name + if path.is_symlink(): + out[path.relative_to(root).as_posix()] = {"link": os.readlink(path)} + dirnames.remove(name) + for name in sorted(filenames): + path = Path(dirpath) / name + rel = path.relative_to(root).as_posix() + if rel == MANIFEST: + continue + if path.is_symlink(): + out[rel] = {"link": os.readlink(path)} + else: + st = path.stat() + out[rel] = {"sha256": sha256_file(path), "size": st.st_size, + "mode": "%o" % (st.st_mode & 0o777)} + return out + + +def write_manifest(root: Path, fp: str, extra: Optional[Dict[str, Any]] = None, + volatile: Iterable[str] = ()) -> Dict[str, Any]: + """Permit new files below volatile prefixes during verification.""" + doc = {"schema_version": 1, "fingerprint": fp, "files": _walk(root), "extra": extra or {}, + "volatile": sorted(volatile)} + atomic_write(root / MANIFEST, json.dumps(doc, indent=1, sort_keys=True) + "\n") + return doc + + +def read_manifest(root: Path) -> Dict[str, Any]: + path = root / MANIFEST + if not path.is_file(): + raise PayloadError("missing", "no %s under %s" % (MANIFEST, root)) + try: + doc = json.loads(path.read_text()) + except ValueError as e: + raise PayloadError("corrupt", "%s: %s" % (path, e)) from None + volatile = doc.get("volatile", []) if isinstance(doc, dict) else [] + if (not isinstance(doc, dict) or doc.get("schema_version") != 1 + or not isinstance(doc.get("files"), dict) + or not isinstance(volatile, list) or not all(isinstance(v, str) for v in volatile)): + raise PayloadError("corrupt", "%s: unexpected shape" % path) + return doc + + +def verify(root: Path, expected_fp: Optional[str] = None) -> Dict[str, Any]: + doc = read_manifest(root) + if expected_fp is not None and doc.get("fingerprint") != expected_fp: + raise PayloadError( + "stale", + "%s was built for fingerprint %s, the tree wants %s" + % (root, str(doc.get("fingerprint"))[:12], expected_fp[:12]), + ) + actual = _walk(root) + want = doc["files"] + # A trailing slash, so a volatile "tmp" does not also absorb "tmplog/". + volatile = tuple(v.rstrip("/") + "/" for v in doc.get("volatile", [])) + missing = sorted(set(want) - set(actual)) + extra = sorted(k for k in set(actual) - set(want) if not k.startswith(volatile)) + changed = sorted(k for k in set(want) & set(actual) if want[k] != actual[k]) + if missing or extra or changed: + parts = [] + for label, items in (("missing", missing), ("extra", extra), ("changed", changed)): + if items: + parts.append("%s: %s" % (label, ", ".join(items[:5]) + (" ..." if len(items) > 5 else ""))) + raise PayloadError("corrupt", "%s does not match its manifest (%s)" % (root, "; ".join(parts))) + return doc + + +def status(root: Path, expected_fp: str) -> str: + try: + doc = read_manifest(root) + except PayloadError as e: + return e.kind + return "ok" if doc.get("fingerprint") == expected_fp else "stale" + + +def absent_message(suite: str, root: Path, state: str, build_hint: str) -> str: + return ("%s payload %s (%s); run: %s, see docs/conformance.md" + % (suite, state, root, build_hint)) + + +def atomic_write(path: Path, text: str) -> None: + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name + ".") + try: + os.fchmod(fd, 0o644) # generated files should not retain mkstemp's 0600 + with os.fdopen(fd, "w") as f: + f.write(text) + os.replace(tmp, path) + except BaseException: + os.unlink(tmp) + raise + + +def check_pins(doc: Any, schema: Dict[str, Dict[str, str]]) -> Dict[str, Any]: + if not isinstance(doc, dict) or doc.get("schema_version") != 1: + raise PinError("pins: schema_version must be 1") + for section, fields in schema.items(): + body = doc.get(section) + if not isinstance(body, dict): + raise PinError("pins: missing section %r" % section) + for name, kind in fields.items(): + value = body.get(name) + where = "pins: %s.%s" % (section, name) + if kind not in ("int", "url", "str") and kind not in _HEX: + raise PinError("%s: unknown kind %r in schema" % (where, kind)) + if kind == "int": + if not isinstance(value, int) or isinstance(value, bool): + raise PinError("%s must be an integer" % where) + elif not isinstance(value, str) or not value: + raise PinError("%s must be a non-empty string" % where) + elif kind in _HEX and not _HEX[kind].match(value): + raise PinError("%s is not a %s digest" % (where, kind)) + elif kind == "url" and not value.startswith("https://"): + raise PinError("%s must be an https URL" % where) + return doc + + +def load_pins(path: Path, schema: Dict[str, Dict[str, str]]) -> Dict[str, Any]: + try: + doc = json.loads(path.read_text()) + except (OSError, ValueError) as e: + raise PinError("%s: %s" % (path, e)) from None + return check_pins(doc, schema) + + +def write_pins(path: Path, doc: Dict[str, Any], schema: Dict[str, Dict[str, str]]) -> None: + check_pins(doc, schema) + atomic_write(path, json.dumps(doc, indent=2, sort_keys=True) + "\n") + + +def diff_pins(old: Dict[str, Any], new: Dict[str, Any]) -> List[str]: + out = [] + for section in sorted(set(old) | set(new)): + a, b = old.get(section), new.get(section) + if not isinstance(a, dict) or not isinstance(b, dict): + if a != b: + out.append("%s: %r -> %r" % (section, a, b)) + continue + for key in sorted(set(a) | set(b)): + if a.get(key) != b.get(key): + out.append("%s.%s: %r -> %r" % (section, key, a.get(key), b.get(key))) + return out + + +def refresh(provider: Any, ref: Optional[str] = None, check: bool = False, + out: Callable[[str], None] = print, + fail: Callable[[str], None] = print) -> int: + """Refresh pins through the provider schema and latest_pin hook.""" + try: + current = load_pins(provider.pins_path, provider.pins_schema) + fresh = provider.latest_pin(copy.deepcopy(current), ref) + check_pins(fresh, provider.pins_schema) + except (UpdateError, OSError, KeyError, IndexError, ValueError) as e: + # urllib failures are OSError; a malformed upstream response raises + # Key/Index/ValueError out of latest_pin. + fail(str(e)) + return EXIT_USAGE + changes = diff_pins(current, fresh) + if not changes: + out("%s: pins are current" % provider.pins_path) + return EXIT_OK + for line in changes: + out(" " + line) + if check: + out("%s: upstream has moved; run: scripts/conformance pins update %s" + % (provider.pins_path, provider.name)) + return EXIT_DRIFT + try: + write_pins(provider.pins_path, fresh, provider.pins_schema) + except OSError as e: + fail(str(e)) + return EXIT_USAGE + out("%s: rewritten" % provider.pins_path) + for step in provider.update_next_steps(): + out(" next: " + step) + return EXIT_OK diff --git a/tests/conformance/providers/__init__.py b/tests/conformance/providers/__init__.py new file mode 100644 index 00000000..230351d5 --- /dev/null +++ b/tests/conformance/providers/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib +from pathlib import Path +from typing import Dict, Type, Union + +from conformance.providers.base import Provider, ProviderError + +REGISTRY: Dict[str, Union[str, Type[Provider]]] = {} + + +def make(name: str, repo_root: Path) -> Provider: + if name not in REGISTRY: + raise ProviderError( + "unknown suite %r; registered: %s" + % (name, ", ".join(sorted(REGISTRY)) or "none") + ) + target = REGISTRY[name] + if isinstance(target, str): + module, _, cls = target.partition(":") + target = getattr(importlib.import_module(module), cls) + if target.name != name: + # A blank name would collapse suite_dir and payload_root onto the + # shared parents. + raise ProviderError("provider for %r declares name %r" % (name, target.name)) + return target(repo_root) diff --git a/tests/conformance/providers/base.py b/tests/conformance/providers/base.py new file mode 100644 index 00000000..879ed24e --- /dev/null +++ b/tests/conformance/providers/base.py @@ -0,0 +1,88 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +from conformance.backends.base import Backend +from conformance.model import Attempt +from conformance.selection import Entry, Selection + + +class ProviderError(RuntimeError): + pass + + +@dataclass +class Case: + id: str + group: str + scope: str + timeout_s: int + meta: Dict[str, Any] = field(default_factory=dict) + + +class Provider: + name = "" + default_timeout_s = 120 + pins_schema: Dict[str, Dict[str, str]] = {} + + def __init__(self, repo_root: Path): + self.repo_root = repo_root + self.suite_dir = repo_root / "tests" / "conformance" / self.name + + @property + def pins_path(self) -> Path: + return self.suite_dir / "pins.json" + + @property + def selection(self) -> Selection: + raise NotImplementedError + + @property + def expectations_dir(self) -> Path: + return self.suite_dir / "expectations" + + def backend_options(self, backend: str) -> Dict[str, Any]: + return {} + + def prerequisites(self, backend: str) -> Optional[str]: + """Return why the suite cannot run, or None when it can.""" + return None + + def enumerate(self, backend: Backend, entries: List[Entry]) -> List[Case]: + raise NotImplementedError + + def batch_key(self, case: Case) -> str: + return case.group + + def run_batch(self, backend: Backend, cases: List[Case], scratch: Path) -> Dict[str, Attempt]: + """Run cases that share a batch key; ids left out are rerun alone.""" + raise NotImplementedError + + def run_single(self, backend: Backend, case: Case, scratch: Path) -> Attempt: + raise NotImplementedError + + def payload_root(self) -> Path: + return self.repo_root / "externals" / "payloads" / self.name + + def fingerprint(self) -> str: + raise NotImplementedError + + def build_payload(self, force: bool = False) -> None: + raise NotImplementedError + + def build_hint(self) -> str: + return "make %s-payload" % self.name + + def latest_pin(self, doc: Dict[str, Any], ref: Optional[str]) -> Dict[str, Any]: + raise NotImplementedError + + def update_next_steps(self) -> List[str]: + return [self.build_hint()] + + def regen_selection(self, check: bool = False) -> List[str]: + return [] diff --git a/tests/conformance/report.py b/tests/conformance/report.py new file mode 100644 index 00000000..a0c959e7 --- /dev/null +++ b/tests/conformance/report.py @@ -0,0 +1,121 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List, Tuple + +from conformance import payload +from conformance.model import CaseResult, Verdict + +RESULTS = "results.json" + + +class ReportError(ValueError): + pass + + +def red_line(case: CaseResult) -> str: + return case.detail or "%s: %s" % (case.id, case.status.value) + + +def gate(cases: Iterable[CaseResult]) -> str: + cases = list(cases) + return "red" if not cases or any(c.verdict.is_red for c in cases) else "green" + + +def counts(cases: Iterable[CaseResult]) -> Dict[str, int]: + out = {v.value: 0 for v in Verdict} + for c in cases: + out[c.verdict.value] += 1 + return out + + +def write(results_dir: Path, meta: Dict[str, Any], cases: List[CaseResult]) -> Dict[str, Any]: + results_dir.mkdir(parents=True, exist_ok=True) + doc = document(meta, cases) + payload.atomic_write(results_dir / RESULTS, json.dumps(doc, indent=1, sort_keys=True) + "\n") + (results_dir / "summary.txt").write_text("\n".join(summary_lines(meta, cases, results_dir)) + "\n") + return doc + + +def document(meta: Dict[str, Any], cases: List[CaseResult]) -> Dict[str, Any]: + return {"schema_version": 1, "kind": "run", "run": dict(meta), + "gate": gate(cases), "counts": counts(cases), + "cases": [c.to_dict() for c in cases]} + + +def load(results_dir: Path) -> Tuple[Dict[str, Any], List[CaseResult]]: + path = results_dir / RESULTS + if not path.is_file(): + raise ReportError("no %s under %s" % (RESULTS, results_dir)) + try: + doc = json.loads(path.read_text()) + except (OSError, ValueError) as e: + raise ReportError("%s: %s" % (path, e)) from None + if (not isinstance(doc, dict) or doc.get("kind") != "run" + or not isinstance(doc.get("cases"), list) + or not isinstance(doc.get("run"), dict)): + raise ReportError("%s: unexpected shape" % path) + if doc.get("schema_version") != 1: + raise ReportError("%s: unknown schema" % path) + try: + cases = [CaseResult.from_dict(c) for c in doc["cases"]] + except (TypeError, ValueError, KeyError) as e: + raise ReportError("%s: malformed case record: %s" % (path, e)) from None + if doc.get("gate") != gate(cases) or doc.get("counts") != counts(cases): + raise ReportError("%s: stored gate or counts contradict the case records" % path) + return doc["run"], cases + + +def _duration(seconds: float) -> str: + seconds = int(seconds) + if seconds >= 3600: + return "%dh%02dm" % (seconds // 3600, seconds % 3600 // 60) + return "%dm%02ds" % (seconds // 60, seconds % 60) + + +def summary_lines(meta: Dict[str, Any], cases: List[CaseResult], results_dir: Path) -> List[str]: + n = counts(cases) + head = "conformance %s/%s %s: %d cases in %s" % ( + meta.get("suite", "?"), meta.get("backend", "?"), meta.get("scope", "?"), + len(cases), _duration(meta.get("elapsed_s", 0))) + lines = [head] + if meta.get("bootstrap"): + lines.append(" bootstrap: expectations not applied") + lines.append(" as_expected %d flaked %d filtered %d" % ( + n["as_expected"], n["flaked"], n["filtered"])) + lines.append(" unexpected_failure %d unexpected_pass %d error %d" % ( + n["unexpected_failure"], n["unexpected_pass"], n["error"])) + for c in cases: + if c.verdict.is_red: + lines.append(" RED " + red_line(c)) + if not cases: + lines.append(" RED no cases ran") + lines.append("RESULT: %s (results: %s)" % (gate(cases).upper(), results_dir)) + return lines + + +def markdown(root: Path) -> Tuple[str, bool]: + """The table, and whether any lane is red or unreadable.""" + rows = ["| lane | scope | gate | as_expected | flaked | filtered | red |", "|---|---|---|---|---|---|---|"] + found = red = False + for path in sorted(root.rglob(RESULTS)): + found = True + try: + meta, cases = load(path.parent) + except ValueError as e: + rows.append("| %s | | error | | | | %s |" % (path.parent, e)) + red = True + continue + n = counts(cases) + red = red or gate(cases) != "green" + rows.append("| %s/%s | %s | %s | %d | %d | %d | %d |" % ( + meta.get("suite"), meta.get("backend"), meta.get("scope"), gate(cases), + n["as_expected"], n["flaked"], n["filtered"], + n["unexpected_failure"] + n["unexpected_pass"] + n["error"])) + if not found: + return "no conformance results under %s\n" % root, False + return "\n".join(rows) + "\n", red diff --git a/tests/conformance/runner.py b/tests/conformance/runner.py new file mode 100644 index 00000000..e525fd93 --- /dev/null +++ b/tests/conformance/runner.py @@ -0,0 +1,114 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import concurrent.futures +from pathlib import Path +from typing import Callable, Dict, List, Optional + +from conformance import ids, judge +from conformance.backends.base import Backend +from conformance.expectations import Expectations, Resolution +from conformance.model import Attempt, CaseResult, Invocation, Status, Verdict +from conformance.providers.base import Case, Provider + +Log = Callable[[str], None] + + +def _relativize(inv: Invocation, root: Path) -> None: + for name in ("stdout", "stderr"): + value = getattr(inv, name) + if value: + try: + setattr(inv, name, str(Path(value).relative_to(root))) + except ValueError: + pass + + +def _finish(case: Case, attempts: List[Attempt], resolution: Resolution, + bootstrap: bool, results_dir: Path, backend: str) -> CaseResult: + last = attempts[-1] + if bootstrap: + verdict = Verdict.ERROR if last.status is Status.ERROR else Verdict.AS_EXPECTED + message = last.detail if verdict is Verdict.ERROR else "" + else: + verdict, message = judge.decide(case.id, last.status, resolution) + if verdict is Verdict.ERROR and last.detail: + message = "%s: %s" % (message, last.detail) + if (resolution.quarantined and verdict.is_red + and verdict is not Verdict.ERROR): + verdict, message = Verdict.FLAKED, "" + elif verdict is Verdict.AS_EXPECTED and len(attempts) > 1: + verdict = Verdict.FLAKED + for a in attempts: + _relativize(a.invocation, results_dir) + return CaseResult( + id=case.id, suite=ids.suite_of(case.id), backend=backend, status=last.status, + verdict=verdict, expectation=resolution.to_dict(), attempts=attempts, detail=message, + ) + + +def run_lane(provider: Provider, backend: Backend, cases: List[Case], + expectations: Expectations, results_dir: Path, jobs: int = 1, + retry: bool = True, bootstrap: bool = False, + log: Optional[Log] = None) -> List[CaseResult]: + log = log or (lambda _: None) + results: Dict[str, CaseResult] = {} + launch: List[Case] = [] + resolutions: Dict[str, Resolution] = {} + for case in cases: + resolution = expectations.resolve(case.id) + resolutions[case.id] = resolution + if resolution.type == "skip" and not bootstrap: + results[case.id] = CaseResult( + id=case.id, suite=ids.suite_of(case.id), backend=backend.name, + status=Status.SKIP, verdict=Verdict.FILTERED, expectation=resolution.to_dict(), + detail="skip: " + resolution.reason) + else: + launch.append(case) + + batches: Dict[str, List[Case]] = {} + for case in launch: + key = case.id if resolutions[case.id].quarantined else provider.batch_key(case) + batches.setdefault(key, []).append(case) + + def run_batch(key: str) -> List[CaseResult]: + members = batches[key] + quarantined = resolutions[members[0].id].quarantined + # A quarantined key holds one case; with no batch result it takes + # the single-case path below. + first = ({} if quarantined + else provider.run_batch(backend, members, + results_dir / "cases" / ("batch-" + ids.slug(key)))) + out = [] + for case in members: + case_dir = results_dir / "cases" / ids.slug(case.id) + if case.id in first: + attempts = [first[case.id]] + else: + if not quarantined: + log("%s: unresolved by the batch, rerunning alone" % case.id) + attempts = [provider.run_single(backend, case, case_dir / "attempt-1")] + resolution = resolutions[case.id] + while (retry and not bootstrap + and judge.decide(case.id, attempts[-1].status, resolution)[0].is_red + and attempts[-1].status is not Status.ERROR + and judge.may_retry(resolution, len(attempts))): + n = len(attempts) + 1 + log("%s: %s on attempt %d, quarantined, retrying" % (case.id, attempts[-1].status.value, n - 1)) + attempts.append(provider.run_single(backend, case, case_dir / ("attempt-%d" % n))) + out.append(_finish(case, attempts, resolution, bootstrap, results_dir, backend.name)) + return out + + workers = min(jobs, backend.max_jobs or jobs) + keys = list(batches) + if workers > 1 and len(keys) > 1: + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + batched = list(pool.map(run_batch, keys)) + else: + batched = [run_batch(k) for k in keys] + for group in batched: + for r in group: + results[r.id] = r + return [results[c.id] for c in cases] diff --git a/tests/conformance/seed.py b/tests/conformance/seed.py new file mode 100644 index 00000000..36516279 --- /dev/null +++ b/tests/conformance/seed.py @@ -0,0 +1,111 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List + +from conformance import ids, jsonc, payload +from conformance.expectations import SEEDED_PREFIX +from conformance.model import CaseResult, Status, Verdict + +class SeedError(ValueError): + pass + + +_ACTION_FOR = { + Status.FAIL: "expect_failure", + Status.BROK: "expect_failure", + Status.CONF: "expect_conf", + Status.TIMEOUT: "skip", + Status.CRASH: "skip", + Status.INCONSISTENT: "skip", +} +_ORDER = ("expect_pass", "expect_failure", "expect_conf", "skip") + + +def default_reason(results_dir: Path, date: str) -> str: + return "%s%s on %s; untriaged" % (SEEDED_PREFIX, results_dir, date) + + +def propose(cases: List[CaseResult], reason: str, bootstrap: bool, + whole_groups: bool = True) -> List[Dict[str, Any]]: + """Collapse complete groups when whole_groups is set.""" + wanted: Dict[str, str] = {} + for c in cases: + if c.status is Status.ERROR: + raise SeedError("%s is a harness ERROR; seeding refuses it" % c.id) + if bootstrap: + # What an earlier leaf recorded is not evidence. + action = _ACTION_FOR.get(c.status) + if action: + wanted[c.id] = action + elif c.verdict is Verdict.UNEXPECTED_PASS: + wanted[c.id] = "expect_pass" + elif c.verdict is Verdict.UNEXPECTED_FAILURE: + wanted[c.id] = _ACTION_FOR.get(c.status, "expect_failure") + by_group: Dict[tuple, List[str]] = {} + for c in cases: + by_group.setdefault((ids.suite_of(c.id), ids.group_of(c.id)), []).append(c.id) + out: Dict[str, List[str]] = {} + for (suite, group), members in sorted(by_group.items()): + actions = {wanted.get(m) for m in members} + if whole_groups and len(actions) == 1 and None not in actions and len(members) > 1: + out.setdefault(actions.pop(), []).append("%s:%s/*" % (suite, group)) + continue + for m in members: + if m in wanted: + out.setdefault(wanted[m], []).append(m) + return [{"type": kind, "reason": reason, "matchers": sorted(out[kind])} + for kind in _ORDER if kind in out] + + +def format_actions(actions: List[Dict[str, Any]], header: str = "") -> str: + lines = [header.rstrip("\n")] if header else [] + lines += ["{", ' "actions": ['] + for a in actions: + if "include" in a: + lines.append(' { "include": %s },' % json.dumps(a["include"])) + continue + lines.append(' { "type": %s,' % json.dumps(a["type"])) + for key in ("reason", "since", "tracking"): + if a.get(key): + lines.append(' "%s": %s,' % (key, json.dumps(a[key]))) + matchers = a["matchers"] + if len(matchers) == 1: + lines.append(' "matchers": [%s] },' % json.dumps(matchers[0])) + else: + lines.append(' "matchers": [') + lines.extend(" %s," % json.dumps(m) for m in matchers) + lines.append(" ] },") + lines += [" ],", "}", ""] + return "\n".join(lines) + + +def append(leaf: Path, actions: List[Dict[str, Any]]) -> None: + """Append actions new to the leaf, preserving the leading comments.""" + text = leaf.read_text() if leaf.exists() else "" + header_lines = [] + for line in text.splitlines(): + if line.startswith("//") or not line.strip(): + header_lines.append(line) + else: + break + if text.strip(): + existing = jsonc.loads(text)["actions"] + else: + # Prefer the shared suite default when present. + base = leaf.parent / (leaf.stem.split("_", 1)[0] + ".jsonc") + existing = ([{"include": base.name}] if base.exists() + else [{"type": "expect_pass", "matchers": ["*"]}]) + present = {(a["type"], m) for a in existing if "type" in a for m in a["matchers"]} + fresh = [] + for a in actions: + matchers = [m for m in a["matchers"] if (a["type"], m) not in present] + if matchers: + fresh.append(dict(a, matchers=matchers)) + if not fresh: + return + payload.atomic_write(leaf, format_actions(existing + fresh, "\n".join(header_lines))) diff --git a/tests/conformance/selection.py b/tests/conformance/selection.py new file mode 100644 index 00000000..eba62847 --- /dev/null +++ b/tests/conformance/selection.py @@ -0,0 +1,130 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import difflib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from conformance import ids, jsonc + +SCOPES = ("pr", "full") + + +class SelectionError(ValueError): + pass + + +@dataclass(frozen=True) +class Entry: + group: str + scope: str + timeout_s: Optional[int] = None + only: Tuple[str, ...] = () + + +@dataclass +class Selection: + enabled: List[Entry] + declined: List[Tuple[str, Tuple[str, ...]]] + extra: Dict[str, Any] = field(default_factory=dict) + source: str = "" + + def groups(self, scope: str) -> List[Entry]: + if scope not in SCOPES: + raise SelectionError("unknown scope %r" % (scope,)) + return [e for e in self.enabled if scope == "full" or e.scope == "pr"] + + def entry(self, group: str) -> Optional[Entry]: + return next((e for e in self.enabled if e.group == group), None) + + def lint(self) -> List[str]: + problems = [] + seen: Dict[str, str] = {} + for e in self.enabled: + if e.group in seen: + problems.append("%s: %s is enabled twice" % (self.source, e.group)) + seen[e.group] = "enabled" + for reason, groups in self.declined: + for g in groups: + if seen.get(g) == "enabled": + problems.append("%s: %s is both enabled and declined" % (self.source, g)) + elif g in seen: + problems.append("%s: %s is declined twice" % (self.source, g)) + seen[g] = "declined" + return problems + + +def _entry(doc: Any, where: str) -> Entry: + if not isinstance(doc, dict) or not isinstance(doc.get("group"), str): + raise SelectionError("%s: enabled entry needs a group" % where) + unknown = set(doc) - {"group", "scope", "timeout_s", "only"} + if unknown: + raise SelectionError("%s: unknown keys %s" % (where, sorted(unknown))) + scope = doc.get("scope") + if scope not in SCOPES: + raise SelectionError("%s: %s has scope %r, want pr or full" % (where, doc["group"], scope)) + timeout = doc.get("timeout_s") + if timeout is not None and (isinstance(timeout, bool) or not isinstance(timeout, int) + or timeout <= 0): + raise SelectionError("%s: timeout_s must be a positive integer" % where) + only = doc.get("only", []) + if not isinstance(only, list) or any(not isinstance(o, str) for o in only): + raise SelectionError("%s: only must be a list of case globs" % where) + return Entry(doc["group"], scope, timeout, tuple(only)) + + +def parse(doc: Any, source: str) -> Selection: + if not isinstance(doc, dict) or doc.get("schema_version") != 1: + raise SelectionError("%s: schema_version must be 1" % source) + enabled = doc.get("enabled") + if not isinstance(enabled, list): + raise SelectionError("%s: enabled must be a list" % source) + entries = [_entry(e, "%s:enabled[%d]" % (source, i)) for i, e in enumerate(enabled)] + declined_doc = doc.get("declined", []) + if not isinstance(declined_doc, list): + raise SelectionError("%s: declined must be a list" % source) + declined = [] + for i, d in enumerate(declined_doc): + where = "%s:declined[%d]" % (source, i) + if (not isinstance(d, dict) or set(d) != {"reason", "groups"} + or not isinstance(d["reason"], str) or not d["reason"].strip() + or not isinstance(d["groups"], list) or not d["groups"] + or any(not isinstance(g, str) for g in d["groups"])): + raise SelectionError("%s: a declined entry is a reason and a group list" % where) + declined.append((d["reason"], tuple(d["groups"]))) + extra = {k: v for k, v in doc.items() if k not in ("schema_version", "enabled", "declined")} + sel = Selection(entries, declined, extra, source) + problems = sel.lint() + if problems: + raise SelectionError("; ".join(problems)) + return sel + + +def load(path: Path) -> Selection: + return parse(jsonc.load(path), path.name) + + +def resolve_ids(patterns: Iterable[str], universe: Iterable[str], + suite: str) -> Tuple[List[str], List[str]]: + """Report unmatched patterns with nearby canonical ids.""" + known = list(universe) + chosen: List[str] = [] + seen = set() + errors: List[str] = [] + for pattern in patterns: + if ids.suite_of(pattern) != suite: + errors.append("%s: not a %s id (want %s:[/])" % (pattern, suite, suite)) + continue + # A bare group id also selects its cases. + pats = [pattern] if "/" in pattern else [pattern, pattern + "/*"] + hits = ids.expand(pats, known) + if not hits: + near = difflib.get_close_matches(pattern, known, n=3, cutoff=0.6) + errors.append("%s: no such test%s" % ( + pattern, ("; near: " + ", ".join(near)) if near else "")) + chosen.extend(h for h in hits if h not in seen) + seen.update(hits) + return chosen, errors diff --git a/tests/conformance/selftest/__init__.py b/tests/conformance/selftest/__init__.py new file mode 100644 index 00000000..13b8e032 --- /dev/null +++ b/tests/conformance/selftest/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/conformance/selftest/fixture.py b/tests/conformance/selftest/fixture.py new file mode 100644 index 00000000..81c8bf90 --- /dev/null +++ b/tests/conformance/selftest/fixture.py @@ -0,0 +1,123 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from typing import Dict, List + +from conformance import ids, payload, selection +from conformance.backends import proc +from conformance.backends.base import Backend +from conformance.model import Attempt, Status +from conformance.providers.base import Case, Provider + + +class TempDirTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = self.root = Path(tmp.name) + +DATA = { + "schema_version": 1, + "enabled": [ + {"group": "basic", "scope": "pr"}, + {"group": "slow", "scope": "full", "timeout_s": 1}, + ], +} +CASES = { + "basic": { + "pass": "exit 0", "fail": "exit 1", "flaky": "flaky", + "quarantined": "exit 1", + }, + "slow": {"timeout": "sleep 30", "unresolved": "unresolved"}, +} + + +def setup(root: Path) -> None: + expectations = root / "fixture" / "expectations" + expectations.mkdir(parents=True) + (expectations / "fixture.jsonc").write_text( + '{"actions":[{"type":"expect_pass","matchers":["*"]}]}\n' + ) + (expectations / "fixture_elfuse.jsonc").write_text( + '{"actions":[{"include":"fixture.jsonc"},' + '{"type":"expect_failure","reason":"fixture exit",' + '"matchers":["fixture:basic/fail"]}]}\n' + ) + (expectations / "flaky.jsonc").write_text( + '{"actions":[{"type":"quarantine","reason":"fixture retry",' + '"matchers":["fixture:basic/flaky",' + '"fixture:basic/quarantined"]}]}\n' + ) + + +class LocalBackend(Backend): + name = "elfuse" + + def run(self, argv, timeout_s, scratch, env=None, fetch=()): + return proc.run_local(argv, timeout_s, scratch, env=env) + + +class FixtureProvider(Provider): + name = "fixture" + + def __init__(self, repo_root: Path): + super().__init__(repo_root) + self.suite_dir = repo_root / "fixture" + self._selection = selection.parse(DATA, "fixture") + self.runs: Dict[str, int] = {} + + @property + def selection(self): + return self._selection + + def fingerprint(self) -> str: + return "0" * 64 + + def build_payload(self, force: bool = False) -> None: + root = self.payload_root() + root.mkdir(parents=True, exist_ok=True) + (root / "fixture").write_text("fixture\n") + payload.write_manifest(root, self.fingerprint()) + + def enumerate(self, backend: Backend, + entries: List[selection.Entry]) -> List[Case]: + return [ + Case("fixture:%s/%s" % (entry.group, name), entry.group, + entry.scope, entry.timeout_s or 5, {"script": script}) + for entry in entries + for name, script in CASES[entry.group].items() + ] + + def invoke(self, backend: Backend, case: Case, scratch: Path) -> Attempt: + self.runs[case.id] = self.runs.get(case.id, 0) + 1 + script = case.meta["script"] + if script == "flaky": + script = "exit %d" % (1 if self.runs[case.id] == 1 else 0) + inv = backend.run(["sh", "-c", script], case.timeout_s, scratch) + if inv.execution == "timeout": + status = Status.TIMEOUT + elif inv.execution != "normal": + status = Status.ERROR + else: + status = Status.PASS if inv.exit_code == 0 else Status.FAIL + return Attempt(status, inv, inv.execution) + + def run_batch(self, backend: Backend, cases: List[Case], + scratch: Path) -> Dict[str, Attempt]: + return { + case.id: self.invoke(backend, case, scratch / ids.slug(case.id)) + for case in cases + if case.meta["script"] != "unresolved" + } + + def run_single(self, backend: Backend, case: Case, + scratch: Path) -> Attempt: + if case.meta["script"] == "unresolved": + case = Case(case.id, case.group, case.scope, case.timeout_s, + {"script": "exit 0"}) + return self.invoke(backend, case, scratch) diff --git a/tests/conformance/selftest/test_backends.py b/tests/conformance/selftest/test_backends.py new file mode 100644 index 00000000..4cdf06c5 --- /dev/null +++ b/tests/conformance/selftest/test_backends.py @@ -0,0 +1,297 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import stat +import time +import unittest +import unittest.mock +from pathlib import Path + +from conformance.backends import BackendError, elfuse, proc, qemu, ssh +from conformance.selftest.fixture import TempDirTest + + +def script(path, body): + path.write_text("#!/bin/sh\n" + body) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return path + + +class ProcTest(TempDirTest): + def test_normal(self): + inv = proc.run_local(["sh", "-c", "echo out; echo err >&2; exit 3"], 10, self.dir) + self.assertEqual((inv.execution, inv.exit_code, inv.signal), ("normal", 3, None)) + self.assertEqual(Path(inv.stdout).read_text(), "out\n") + self.assertEqual(Path(inv.stderr).read_text(), "err\n") + self.assertGreater(inv.wall_us, 0) + + def test_unspawnable(self): + inv = proc.run_local([str(self.dir / "absent")], 10, self.dir) + self.assertEqual(inv.execution, "transport") + self.assertIn("cannot spawn", Path(inv.stderr).read_text()) + + def test_signal(self): + inv = proc.run_local(["sh", "-c", "kill -SEGV $$"], 10, self.dir) + self.assertEqual((inv.execution, inv.exit_code, inv.signal), ("signal", None, 11)) + + def test_timeout_kills_the_group(self): + started = time.monotonic() + inv = proc.run_local(["sh", "-c", "sleep 30 & echo $! > pid; wait"], 1, self.dir) + self.assertEqual(inv.execution, "timeout") + self.assertLess(time.monotonic() - started, 5) + self.assertIsNone(inv.exit_code) + child = int((self.dir / "pid").read_text()) + for _ in range(50): # SIGKILL reaches the orphan asynchronously + try: + os.kill(child, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail("background child %d survived the group kill" % child) + + def test_timeout_that_cannot_reap_is_transport(self): + kill_and_reap = proc._kill_and_reap + + def report_unreaped(child): + kill_and_reap(child) + return None + + with unittest.mock.patch.object(proc, "_kill_and_reap", report_unreaped): + inv = proc.run_local(["sleep", "30"], 0.01, self.dir) + self.assertEqual(inv.execution, "transport") + self.assertIn("not reaped after SIGKILL", Path(inv.stderr).read_text()) + + +class SshTest(TempDirTest): + def setUp(self): + super().setUp() + self.args = self.dir / "args" + self.stub = script( + self.dir / "ssh", + 'printf "%%s\\n" "$@" >> "%s"\n' % self.args + + 'case "$*" in *"cat /tmp/conf.abc/result.xml") printf ""; exit 0 ;; esac\n' + + 'case "$*" in *"cat /tmp/conf.abc/missing.bin") exit 1 ;; esac\n' + + 'case "$*" in *"rm -rf /tmp/conf"*) exit 0 ;; esac\n' + # ssh forwards the remote script's status, 0 once the sentinel + # printf ran; the guest rc travels inside the sentinel. + + 'case "$STUB" in\n' + ' ok) printf "hello\\n\\n__CONF_RC=3 __CONF_DIR=/tmp/conf.abc\\n"; exit 0 ;;\n' + ' sig) printf "__CONF_RC=139 __CONF_DIR=/tmp/x\\n"; exit 0 ;;\n' + ' bigrc) printf "__CONF_RC=255 __CONF_DIR=/tmp/x\\n"; exit 0 ;;\n' + ' spoof) printf "__CONF_RC=0 __CONF_DIR=/\\n"; exit 255 ;;\n' + " lost) exit 255 ;;\n" + "esac\n", + ) + self.session = ssh.SshSession(2222, Path("/k"), ssh=str(self.stub)) + + def run_stub(self, mode, **kw): + os.environ["STUB"] = mode + try: + return self.session.run(["true", "a b"], 5, self.dir / mode, **kw) + finally: + del os.environ["STUB"] + + def test_script(self): + text = ssh.SshSession.remote_script(["cmd", "a b"], 7, {"X": "y z"}, None) + self.assertIn("mktemp -d /tmp/conf.XXXXXX", text) + self.assertNotIn("rm -rf", text) + self.assertIn("export LC_ALL=C;", text) + self.assertIn("trap 'cd / && rm -rf \"$d\"' EXIT", ssh.SshSession.remote_script(["x"], 1, {}, None, cleanup=True)) + self.assertIn("/usr/bin/timeout -s KILL 7 cmd 'a b'", text) + self.assertIn("export X='y z';", text) + self.assertIn('export HOME="$PWD"', text) + self.assertIn(ssh.SENTINEL, text) + self.assertIn("cd /opt/ltp", ssh.SshSession.remote_script(["x"], 1, {}, "/opt/ltp")) + + def test_ok(self): + inv = self.run_stub("ok", fetch=["result.xml"]) + self.assertEqual((inv.execution, inv.exit_code), ("normal", 3)) + self.assertEqual(Path(inv.stdout).read_text(), "hello\n") + args = self.args.read_text().splitlines() + self.assertIn("BatchMode=yes", args) + self.assertEqual(args[args.index("-p") + 1], "2222") + self.assertEqual(args[-2], "root@127.0.0.1") + self.assertEqual((self.dir / "ok" / "result.xml").read_text(), "") + self.assertIn("rm -rf /tmp/conf.abc", self.args.read_text()) + + def test_signal_like_status_stays_an_exit_code(self): + inv = self.run_stub("sig") + self.assertEqual( + (inv.execution, inv.exit_code, inv.signal), ("normal", 139, None) + ) + + def test_an_exit_past_192_stays_an_exit_code(self): + inv = self.run_stub("bigrc") + self.assertEqual((inv.execution, inv.exit_code, inv.signal), ("normal", 255, None)) + + def test_a_sentinel_cut_mid_write_is_a_transport_loss(self): + self.assertIsNone(ssh.SshSession.parse_sentinel("x\n" + ssh.SENTINEL + "1")) + self.assertEqual(ssh.SshSession.parse_sentinel(ssh.SENTINEL + "12" + ssh.DIR_MARK + "/t"), (12, "/t")) + + def test_a_garbled_sentinel_is_a_transport_loss(self): + # A partial sentinel is a transport loss, not a parser error. + for line in ("__CONF_RC=", "__CONF_RC=xx __CONF_DIR=/tmp/a", "__CONF_RC=-"): + self.assertIsNone(ssh.SshSession.parse_sentinel(line), line) + self.assertEqual(ssh.SshSession.parse_sentinel("__CONF_RC=3 __CONF_DIR=/tmp/a b"), + (3, "/tmp/a b")) + + def test_transport(self): + inv = self.run_stub("lost") + self.assertEqual(inv.execution, "transport") + self.assertIsNone(inv.exit_code) + + def test_a_lookalike_sentinel_on_a_lost_connection_is_transport(self): + inv = self.run_stub("spoof") + self.assertEqual(inv.execution, "transport") + self.assertNotIn("rm -rf /", self.args.read_text()) + + def test_a_failed_fetch_is_a_transport_loss(self): + inv = self.run_stub("ok", fetch=["missing.bin"]) + self.assertEqual(inv.execution, "transport") + + +class ElfuseTest(TempDirTest): + def test_argv_and_prerequisites(self): + b = elfuse.ElfuseBackend(self.dir, sysroot=self.dir / "root") + self.assertIn("absent; run: make elfuse", b.prerequisites()) + script(self.dir / "elfuse", "exit 0\n") + (self.dir / "build").mkdir() + os.rename(self.dir / "elfuse", self.dir / "build" / "elfuse") + self.assertIn("sysroot", b.prerequisites()) + (self.dir / "root").mkdir() + self.assertIsNone(b.prerequisites()) + self.assertEqual( + b.argv(["/bin/true", "x"]), + [str(self.dir / "build" / "elfuse"), "--timeout", "0", "--sysroot", + str(self.dir / "root"), "/bin/true", "x"], + ) + + def test_run_scrubs_the_environment(self): + (self.dir / "build").mkdir() + script(self.dir / "build" / "elfuse", 'shift 2; echo "$TZ $HOME"; exec "$@"\n') + b = elfuse.ElfuseBackend(self.dir) + with unittest.mock.patch.dict(os.environ, {"LEAK": "1"}): + inv = b.run(["sh", "-c", 'echo "${LEAK:-clean}"'], 5, self.dir / "s") + self.assertEqual(inv.exit_code, 0) + self.assertEqual(Path(inv.stdout).read_text(), "UTC %s\nclean\n" % (self.dir / "s")) + + def test_orphan_pids(self): + listing = (" 12 1 40 /repo/build/elfuse --fork-child 8\n" + " 13 500 40 /repo/build/elfuse --fork-child 8\n" + " 14 1 40 /repo/build/elfuse --timeout 0 /bin/true\n" + " 15 1 40 /other/build/elfuse --fork-child 8\n" + " 16 1 41 /repo/build/elfuse --fork-child 9\n") + self.assertEqual(elfuse.orphan_pids(listing, "/repo/build/elfuse", 40), [12]) + + def test_serialize_excludes_a_second_session(self): + a, b = elfuse.ElfuseBackend(self.dir), elfuse.ElfuseBackend(self.dir) + a.lock_file = b.lock_file = self.dir / "lock" + with a.serialize(): + with self.assertRaises(BackendError): + with b.serialize(): + pass + with b.serialize(): + pass + + @unittest.skipIf(os.geteuid() == 0, "root ignores directory permissions") + def test_an_unopenable_lock_is_a_backend_error(self): + a = elfuse.ElfuseBackend(self.dir) + a.lock_file = self.dir / "locked" / "lock" + a.lock_file.parent.mkdir() + a.lock_file.parent.chmod(0o500) + try: + with self.assertRaises(BackendError): + with a.serialize(): + pass + finally: + a.lock_file.parent.chmod(0o700) + + +class QemuTest(TempDirTest): + def test_prerequisites_report_the_tree_before_the_host(self): + # Missing fixtures produce the same answer on every host. + b = qemu.QemuBackend(self.dir) + with unittest.mock.patch("shutil.which", return_value=None): + self.assertIn("QEMU fixtures missing", b.prerequisites()) + + def test_guest_path(self): + b = qemu.QemuBackend(self.dir) + (self.dir / "tests").mkdir() + self.assertEqual(b.guest_path(self.dir / "tests" / "x"), "/mnt/host/tests/x") + with self.assertRaises(BackendError): + b.guest_path(Path("/etc/passwd")) + + def test_start_reads_the_state_file(self): + runner = script( + self.dir / "runner.sh", + '[ "$1" = start ] && printf "port=2200\\nkey=/k\\npidfile=/p\\n" > "$3"\n' + 'echo "$QEMU_MEM $QEMU_BOOT_TIMEOUT" > "%s/mem"\n' % self.dir, + ) + b = qemu.QemuBackend(self.dir, mem_mib=4096, runner=runner, state_dir=self.dir) + with unittest.mock.patch.dict(os.environ, {"QEMU_BOOT_TIMEOUT": "300", "QEMU_MEM": "1"}): + b.start() + self.assertEqual((b.session.port, b.session.key), (2200, Path("/k"))) + self.assertEqual((self.dir / "mem").read_text().split(), ["4096", "300"]) + b.stop() + self.assertIsNone(b.session) + + def test_start_reaps_the_vm_a_stale_state_file_names(self): + runner = script( + self.dir / "runner.sh", + 'echo "$1" >> "%s/verbs"\n' % self.dir + + '[ "$1" = start ] && printf "port=2200\\nkey=/k\\n" > "$3"\nexit 0\n', + ) + (self.dir / "qemu.state").write_text("port=2199\nkey=/k\npidfile=/p\n") + b = qemu.QemuBackend(self.dir, runner=runner, state_dir=self.dir) + b.start() + self.assertEqual((self.dir / "verbs").read_text().split(), ["stop", "start"]) + self.assertEqual(b.session.port, 2200) + + def test_start_after_a_bad_state_stops_the_vm(self): + # ":" writes nothing, so there is no VM record left to stop. + for writer, verbs in (('printf "key=/k\\n" > "$3"', ["start", "stop"]), + ('printf "port=abc\\nkey=/k\\n" > "$3"', ["start", "stop"]), + (":", ["start"]), + ('mkdir "$3"', ["start", "stop"])): + runner = script( + self.dir / "runner.sh", + 'echo "$1" >> "%s/verbs"\n' % self.dir + + '[ "$1" = stop ] && rm -rf "$3"\n' + '[ "$1" = start ] && %s\nexit 0\n' % writer, + ) + b = qemu.QemuBackend(self.dir, runner=runner, state_dir=self.dir) + with self.assertRaises(BackendError, msg=writer): + b.start() + self.assertEqual((self.dir / "verbs").read_text().split(), verbs, writer) + (self.dir / "verbs").unlink() + + def test_a_failed_stop_is_an_error(self): + runner = script( + self.dir / "runner.sh", + '[ "$1" = start ] && printf "port=2200\\nkey=/k\\n" > "$3"\n' + '[ "$1" = stop ] && { echo "no such vm"; exit 1; }\nexit 0\n', + ) + b = qemu.QemuBackend(self.dir, runner=runner, state_dir=self.dir) + b.start() + with self.assertRaises(BackendError) as cm: + b.stop() + self.assertIn("no such vm", str(cm.exception)) + + def test_a_relative_scratch_still_names_an_absolute_home(self): + env = elfuse.base.guest_environment("rel/x") + self.assertTrue(Path(env["HOME"]).is_absolute()) + self.assertEqual(env["HOME"], env["TMPDIR"]) + + def test_serialize_excludes_a_second_session_on_the_state_dir(self): + a = qemu.QemuBackend(self.dir, state_dir=self.dir) + b = qemu.QemuBackend(self.dir, state_dir=self.dir) + with a.serialize(): + with self.assertRaises(BackendError): + with b.serialize(): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_cli.py b/tests/conformance/selftest/test_cli.py new file mode 100644 index 00000000..de127145 --- /dev/null +++ b/tests/conformance/selftest/test_cli.py @@ -0,0 +1,150 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import json +import os +import unittest +import unittest.mock +from pathlib import Path + +from conformance import cli, providers, report +from conformance.backends.base import BackendError +from conformance.selftest.fixture import FixtureProvider, LocalBackend, TempDirTest, setup + + +class CliTest(TempDirTest): + def setUp(self): + super().setUp() + setup(self.root) + self.out = [] + self.err = [] + self.registry = unittest.mock.patch.dict( + providers.REGISTRY, {"fixture": FixtureProvider}, clear=True + ) + self.backend = unittest.mock.patch.object( + cli.backends, "make", return_value=LocalBackend() + ) + self.registry.start() + self.backend.start() + + def tearDown(self): + self.backend.stop() + self.registry.stop() + + def invoke(self, *args): + self.out.clear() + self.err.clear() + return cli.main(list(args), self.root, self.out.append, self.err.append) + + def test_suite_json_uses_a_versioned_envelope(self): + self.assertEqual(self.invoke("suites", "--format", "json"), 0) + doc = json.loads(self.out[0]) + self.assertEqual(doc, { + "kind": "suite-list", "schema_version": 1, + "suites": ["fixture"], + }) + self.assertEqual(self.err, []) + + def test_list_all_uses_one_backend(self): + self.assertEqual( + self.invoke("list", "fixture", "--scope", "pr", "--backend", "all"), + 0, + ) + self.assertEqual(self.out, [ + "fixture:basic/pass", "fixture:basic/fail", "fixture:basic/flaky", + "fixture:basic/quarantined", + ]) + self.assertEqual(cli.backends.make.call_count, 1) + + def test_run_writes_results_and_report_reads_them(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + results = self.root / "results" + self.assertEqual(self.invoke( + "run", "fixture", "--scope", "pr", "--results", str(results) + ), 0, self.err) + paths = list(results.rglob(report.RESULTS)) + self.assertEqual(len(paths), 1) + doc = json.loads(paths[0].read_text()) + self.assertEqual((doc["kind"], doc["gate"]), ("run", "green")) + flaky = next(c for c in doc["cases"] if c["id"].endswith("/flaky")) + self.assertEqual(len(flaky["attempts"]), 2) + self.assertEqual(self.invoke( + "report", str(paths[0].parent), "--format", "json" + ), 0) + self.assertEqual(json.loads(self.out[0])["gate"], "green") + + def test_results_survive_a_failing_stop(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + results = self.root / "results" + with unittest.mock.patch.object(LocalBackend, "stop", + side_effect=BackendError("vm stuck")): + self.assertEqual(self.invoke( + "run", "fixture", "--scope", "pr", "--results", str(results) + ), 1) + self.assertEqual(len(list(results.rglob(report.RESULTS))), 1) + self.assertEqual(self.err[-1], "conformance: backend error: vm stuck") + + def test_a_backend_that_cannot_start_is_red(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + with unittest.mock.patch.object(LocalBackend, "start", + side_effect=BackendError("did not boot")): + self.assertEqual(self.invoke( + "run", "fixture", "--scope", "pr", + "--results", str(self.root / "results") + ), 1) + self.assertEqual(self.err[-1], "conformance: backend error: did not boot") + + def test_results_record_the_argv_that_ran(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + results = self.root / "results" + argv = ["run", "fixture", "--scope", "pr", "--results", str(results)] + self.assertEqual(self.invoke(*argv), 0, self.err) + doc = json.loads(next(results.rglob(report.RESULTS)).read_text()) + self.assertEqual(doc["run"]["argv"], argv) + + def test_case_selector_and_dry_run(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + self.assertEqual(self.invoke( + "run", "fixture", "--case", "fixture:basic/pa*", "--dry-run" + ), 0) + self.assertEqual(self.out, ["fixture:basic/pass"]) + self.assertEqual(self.invoke( + "run", "fixture", "--case", "fixture:basic/pas", "--dry-run" + ), 2) + self.assertIn("near: fixture:basic/pass", self.err[0]) + + def test_skip_and_require_are_stable(self): + args = argparse.Namespace(require=False) + instance = cli.Cli(self.root, self.out.append, self.err.append) + with unittest.mock.patch.dict(os.environ, {"CONF_REQUIRE": "0"}): + self.assertEqual(instance.skip(args, "missing"), 77) + args.require = True + self.assertEqual(instance.skip(args, "missing"), 2) + args.require = False + os.environ["CONF_REQUIRE"] = "1" + self.assertEqual(instance.skip(args, "missing"), 2) + self.assertEqual(self.out, []) + self.assertEqual(self.err, ["conformance: missing"] * 3) + + def test_expectation_errors_use_stderr(self): + leaf = self.root / "fixture" / "expectations" / "fixture_elfuse.jsonc" + leaf.write_text("{") + self.assertEqual(self.invoke("expectations", "check", "fixture"), 2) + self.assertEqual(self.out, []) + self.assertIn("line 1", self.err[0]) + + def test_a_provider_must_declare_its_registry_name(self): + with unittest.mock.patch.dict(providers.REGISTRY, {"other": FixtureProvider}): + with self.assertRaisesRegex(providers.ProviderError, "declares name"): + providers.make("other", self.root) + + def test_parser_has_no_suite_owned_commands(self): + help_text = cli.build_parser(["fixture"]).format_help() + self.assertIn("{suites,list,run,payload,selection,expectations,pins,report,selftest}", + help_text) + self.assertNotIn("fixture}", help_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_elfcheck.py b/tests/conformance/selftest/test_elfcheck.py new file mode 100644 index 00000000..e8ad7869 --- /dev/null +++ b/tests/conformance/selftest/test_elfcheck.py @@ -0,0 +1,89 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import struct +import unittest +from pathlib import Path + +from conformance import elfcheck +from conformance.selftest.fixture import TempDirTest + + +def elf(machine=elfcheck.EM_AARCH64, interp=None, needed=(), load=True): + phdrs = [] + body = bytearray() + strtab = b"\0" + b"\0".join(n.encode() for n in needed) + b"\0" + offsets, pos = [], 1 + for n in needed: + offsets.append(pos) + pos += len(n) + 1 + interp_bytes = (interp.encode() + b"\0") if interp else b"" + dyn = b"".join(struct.pack(" %r" % ("a" * 40, "b" * 40), self.lines[0]) + self.assertIn("scripts/conformance pins update tool", self.lines[-1]) + + def test_write(self): + rc = payload.refresh(Stub(self.path, "b" * 40), out=self.lines.append) + self.assertEqual(rc, EXIT_OK) + self.assertEqual(json.loads(self.path.read_text())["tool"]["commit"], "b" * 40) + self.assertIn("next: make tool-payload", self.lines[-1]) + + def test_invalid_never_replaces(self): + rc = payload.refresh(Stub(self.path, "not a digest"), fail=self.errors.append) + self.assertEqual(rc, EXIT_USAGE) + self.assertEqual(self.path.read_text(), self.before) + + def test_upstream_error(self): + rc = payload.refresh(Stub(self.path, "b" * 40, fail=True), fail=self.errors.append) + self.assertEqual(rc, EXIT_USAGE) + self.assertIn("api unreachable", self.errors[0]) + + def test_diff_sections(self): + self.assertEqual(payload.diff_pins({"a": {"x": 1}}, {"a": {"x": 1}, "b": {"y": 2}}), + ["b: None -> {'y': 2}"]) + self.assertEqual(payload.diff_pins({"a": {"x": 1, "y": 1}}, {"a": {"x": 2, "y": 1}}), + ["a.x: 1 -> 2"]) + + def test_a_bad_upstream_is_an_error_not_a_traceback(self): + for error in (OSError("unreachable"), KeyError("tag_name"), IndexError("x")): + out = [] + rc = payload.refresh(Stub(self.path, "b" * 40, error=error), + None, False, out.append, out.append) + self.assertEqual(rc, EXIT_USAGE, error) + self.assertEqual(self.path.read_text(), self.before) + + @unittest.skipIf(os.geteuid() == 0, "root ignores directory permissions") + def test_an_unwritable_pins_directory_is_an_error(self): + os.chmod(self.dir, 0o500) + try: + rc = payload.refresh(Stub(self.path, "b" * 40), out=self.lines.append, + fail=self.errors.append) + finally: + os.chmod(self.dir, 0o700) + self.assertEqual(rc, EXIT_USAGE) + self.assertTrue(self.errors) + self.assertEqual(self.path.read_text(), self.before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_report.py b/tests/conformance/selftest/test_report.py new file mode 100644 index 00000000..79d8af4e --- /dev/null +++ b/tests/conformance/selftest/test_report.py @@ -0,0 +1,90 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import json +import unittest +from pathlib import Path + +from conformance import report +from conformance.selftest.fixture import TempDirTest +from conformance.model import Attempt, CaseResult, Invocation, Status, Verdict + + +def case(cid, status, verdict, detail=""): + inv = Invocation("normal", 1500, exit_code=0 if status is Status.PASS else 1) + return CaseResult(cid, "fake", "host", status, verdict, {"type": "expect_pass"}, + [Attempt(status, inv)], detail) + + +class ReportTest(TempDirTest): + def setUp(self): + super().setUp() + self.meta = {"suite": "fake", "backend": "host", "scope": "pr", "elapsed_s": 61.4} + + def test_empty_is_red(self): + self.assertEqual(report.gate([]), "red") + lines = report.summary_lines(self.meta, [], self.dir) + self.assertIn(" RED no cases ran", lines) + self.assertTrue(lines[-1].startswith("RESULT: RED")) + + def test_round_trip_and_summary(self): + cases = [case("fake:b/p", Status.PASS, Verdict.AS_EXPECTED), + case("fake:b/f", Status.FAIL, Verdict.UNEXPECTED_FAILURE, "fake:b/f: FAIL \x01 raw")] + report.write(self.dir, self.meta, cases) + meta, again = report.load(self.dir) + self.assertEqual(again, cases) + self.assertEqual(meta["scope"], "pr") + text = (self.dir / "summary.txt").read_text() + self.assertIn("conformance fake/host pr: 2 cases in 1m01s", text) + self.assertIn(" unexpected_failure 1 unexpected_pass 0 error 0", text) + self.assertIn(" RED fake:b/f: FAIL", text) + self.assertIn("RESULT: RED", text) + doc = json.loads((self.dir / report.RESULTS).read_text()) + self.assertEqual(doc["kind"], "run") + + def test_contradiction_is_rejected(self): + report.write(self.dir, self.meta, [case("fake:b/p", Status.PASS, Verdict.AS_EXPECTED)]) + doc = json.loads((self.dir / report.RESULTS).read_text()) + doc["gate"] = "red" + (self.dir / report.RESULTS).write_text(json.dumps(doc)) + with self.assertRaises(report.ReportError): + report.load(self.dir) + + def test_a_wrong_shape_is_rejected(self): + for text in ("[]", '{"schema_version": 1, "kind": "run", ' + '"cases": null, "run": {}}', + '{"schema_version": 1, "kind": "run", ' + '"cases": [], "run": []}', + '{"schema_version": 1, "kind": "run", ' + '"cases": ["x"], "run": {}}'): + (self.dir / report.RESULTS).write_text(text) + with self.assertRaises(report.ReportError, msg=text): + report.load(self.dir) + + def test_markdown(self): + report.write(self.dir / "fake" / "host" / "1", self.meta, + [case("fake:b/p", Status.PASS, Verdict.AS_EXPECTED)]) + bad = self.dir / "fake" / "host" / "2" + bad.mkdir(parents=True) + (bad / report.RESULTS).write_text('{"schema_version": 1}') + worse = self.dir / "fake" / "host" / "3" + worse.mkdir(parents=True) + (worse / report.RESULTS).write_text("[]") + text, red = report.markdown(self.dir) + self.assertIn("| fake/host | pr | green | 1 | 0 | 0 | 0 |", text) + self.assertEqual(text.count("| error |"), 2) + self.assertTrue(red) # a lane that will not load is not green + text, red = report.markdown(self.dir / "none") + self.assertIn("no conformance results", text) + self.assertFalse(red) + + def test_markdown_reports_a_red_lane(self): + report.write(self.dir / "fake" / "host" / "1", self.meta, + [case("fake:b/p", Status.FAIL, Verdict.UNEXPECTED_FAILURE)]) + text, red = report.markdown(self.dir) + self.assertIn("| fake/host | pr | red | 0 | 0 | 0 | 1 |", text) + self.assertTrue(red) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_runner.py b/tests/conformance/selftest/test_runner.py new file mode 100644 index 00000000..81df02ee --- /dev/null +++ b/tests/conformance/selftest/test_runner.py @@ -0,0 +1,72 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from pathlib import Path + +from conformance import expectations, report, runner +from conformance.model import Status, Verdict +from conformance.selftest.fixture import FixtureProvider, LocalBackend, TempDirTest, setup + + +class RunnerTest(TempDirTest): + def setUp(self): + super().setUp() + setup(self.root) + self.provider = FixtureProvider(self.root) + self.backend = LocalBackend() + self.exps = expectations.load( + "fixture", "elfuse", self.provider.expectations_dir + ) + self.log = [] + + def lane(self, scope, **kwargs): + cases = self.provider.enumerate( + self.backend, self.provider.selection.groups(scope) + ) + results = runner.run_lane( + self.provider, self.backend, cases, self.exps, self.root / "results", + log=self.log.append, **kwargs + ) + return {result.id: result for result in results} + + def test_pr_scope_is_green(self): + results = self.lane("pr") + self.assertEqual(report.gate(results.values()), "green") + self.assertEqual( + results["fixture:basic/fail"].verdict, Verdict.AS_EXPECTED + ) + flaky = results["fixture:basic/flaky"] + self.assertEqual(flaky.verdict, Verdict.FLAKED) + self.assertEqual( + [attempt.status for attempt in flaky.attempts], + [Status.FAIL, Status.PASS], + ) + quarantined = results["fixture:basic/quarantined"] + self.assertEqual(quarantined.verdict, Verdict.FLAKED) + self.assertEqual( + [attempt.status for attempt in quarantined.attempts], + [Status.FAIL, Status.FAIL, Status.FAIL], + ) + + def test_no_retry_keeps_quarantine_non_gating(self): + result = self.lane("pr", retry=False)["fixture:basic/flaky"] + self.assertEqual(result.verdict, Verdict.FLAKED) + self.assertEqual(len(result.attempts), 1) + + def test_unresolved_batch_case_runs_alone(self): + result = self.lane("full")["fixture:slow/unresolved"] + self.assertEqual(result.verdict, Verdict.AS_EXPECTED) + self.assertTrue(any("rerunning alone" in line for line in self.log)) + + def test_bootstrap_records_without_expectations(self): + results = self.lane("pr", bootstrap=True) + self.assertEqual(report.gate(results.values()), "green") + self.assertEqual( + results["fixture:basic/fail"].verdict, Verdict.AS_EXPECTED + ) + self.assertEqual(len(results["fixture:basic/flaky"].attempts), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_seed.py b/tests/conformance/selftest/test_seed.py new file mode 100644 index 00000000..6fc112d4 --- /dev/null +++ b/tests/conformance/selftest/test_seed.py @@ -0,0 +1,98 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import tempfile +import unittest +from pathlib import Path + +from conformance import expectations, jsonc, seed +from conformance.model import Attempt, CaseResult, Invocation, Status, Verdict + + +def case(cid, status, verdict=Verdict.AS_EXPECTED, expectation="expect_pass"): + inv = Invocation("normal", 1, exit_code=0) + return CaseResult(cid, "s", "b", status, verdict, {"type": expectation}, [Attempt(status, inv)]) + + +class SeedTest(unittest.TestCase): + def test_bootstrap_collapses_whole_groups(self): + cases = [case("s:a/1", Status.FAIL), case("s:a/2", Status.BROK), + case("s:b/1", Status.FAIL), case("s:b/2", Status.PASS), + case("s:c/1", Status.CONF), case("s:d/1", Status.TIMEOUT), case("s:d/2", Status.CRASH), + case("s:e/1", Status.PASS), case("s:e/2", Status.SKIP)] + actions = seed.propose(cases, "r", bootstrap=True) + self.assertEqual(actions, [ + {"type": "expect_failure", "reason": "r", "matchers": ["s:a/*", "s:b/1"]}, + {"type": "expect_conf", "reason": "r", "matchers": ["s:c/1"]}, + {"type": "skip", "reason": "r", "matchers": ["s:d/*"]}, + ]) + + def test_a_partial_group_never_collapses(self): + cases = [case("s:a/1", Status.FAIL, Verdict.UNEXPECTED_FAILURE), + case("s:a/2", Status.PASS, Verdict.AS_EXPECTED)] + self.assertEqual(seed.propose(cases, "r", False)[0]["matchers"], ["s:a/1"]) + + def test_whole_groups_disabled_lists_members(self): + cases = [case("s:a/1", Status.FAIL, Verdict.UNEXPECTED_FAILURE), + case("s:a/2", Status.FAIL, Verdict.UNEXPECTED_FAILURE)] + self.assertEqual(seed.propose(cases, "r", False, whole_groups=False)[0]["matchers"], + ["s:a/1", "s:a/2"]) + + def test_bootstrap_ignores_what_an_earlier_leaf_recorded(self): + cases = [case("s:a/1", Status.FAIL, expectation="expect_failure"), case("s:a/2", Status.FAIL)] + self.assertEqual(seed.propose(cases, "r", True), + [{"type": "expect_failure", "reason": "r", "matchers": ["s:a/*"]}]) + + def test_gated_seeds_only_reds(self): + cases = [case("s:a/1", Status.PASS, Verdict.UNEXPECTED_PASS, "expect_failure"), + case("s:a/2", Status.FAIL, Verdict.AS_EXPECTED, "expect_failure"), + case("s:b/1", Status.FAIL, Verdict.UNEXPECTED_FAILURE)] + self.assertEqual(seed.propose(cases, "r", False), [ + {"type": "expect_pass", "reason": "r", "matchers": ["s:a/1"]}, + {"type": "expect_failure", "reason": "r", "matchers": ["s:b/1"]}, + ]) + + def test_error_refused(self): + with self.assertRaises(ValueError): + seed.propose([case("s:a/1", Status.ERROR, Verdict.ERROR)], "r", True) + + def test_append_keeps_header_and_loads(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "s.jsonc").write_text('{"actions":[{"type":"expect_pass","matchers":["*"]}]}') + leaf = root / "s_b.jsonc" + leaf.write_text('// head\n// two\n{\n "actions": [\n { "include": "s.jsonc" }, // inline\n ],\n}\n') + seed.append(leaf, [{"type": "skip", "reason": "r", "since": "2026-08-25", "matchers": ["s:a/1", "s:a/2"]}]) + text = leaf.read_text() + self.assertTrue(text.startswith("// head\n// two\n{")) + self.assertNotIn("inline", text) + self.assertEqual(jsonc.loads(text)["actions"][0], {"include": "s.jsonc"}) + e = expectations.load("s", "b", root) + self.assertEqual(e.resolve("s:a/2").type, "skip") + self.assertEqual(expectations.lint(root), []) + + def test_append_creates_a_loadable_leaf(self): + action = {"type": "skip", "reason": "r", "matchers": ["s:a/1"]} + for base in (True, False): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + if base: + (root / "s.jsonc").write_text('{"actions":[{"type":"expect_pass","matchers":["*"]}]}') + seed.append(root / "s_b.jsonc", [action]) + first = jsonc.loads((root / "s_b.jsonc").read_text())["actions"][0] + self.assertEqual(first, {"include": "s.jsonc"} if base + else {"type": "expect_pass", "matchers": ["*"]}) + self.assertEqual(expectations.load("s", "b", root).resolve("s:a/1").type, "skip") + + def test_append_skips_matchers_already_recorded(self): + with tempfile.TemporaryDirectory() as tmp: + leaf = Path(tmp) / "s_b.jsonc" + action = {"type": "skip", "reason": "r", "matchers": ["s:a/1"]} + seed.append(leaf, [action]) + before = leaf.read_text() + seed.append(leaf, [action]) + self.assertEqual(leaf.read_text(), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_selection.py b/tests/conformance/selftest/test_selection.py new file mode 100644 index 00000000..d555e781 --- /dev/null +++ b/tests/conformance/selftest/test_selection.py @@ -0,0 +1,68 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from conformance import selection + +DATA = { + "schema_version": 1, + "enabled": [ + {"group": "basic", "scope": "pr"}, + {"group": "slow", "scope": "full", "timeout_s": 1}, + {"group": "narrow", "scope": "full", "only": ["keep*"]}, + ], + "declined": [{"reason": "unsupported", "groups": ["kernel"]}], +} + + +class SelectionTest(unittest.TestCase): + def test_parse(self): + sel = selection.parse(DATA, "fixture") + self.assertEqual([e.group for e in sel.groups("pr")], ["basic"]) + self.assertEqual([e.group for e in sel.groups("full")], + ["basic", "slow", "narrow"]) + self.assertEqual(sel.entry("slow").timeout_s, 1) + self.assertEqual(sel.entry("narrow").only, ("keep*",)) + self.assertEqual(sel.declined, [("unsupported", ("kernel",))]) + + def test_full_is_a_superset(self): + sel = selection.parse(DATA, "fixture") + pr = {e.group for e in sel.groups("pr")} + self.assertTrue(pr <= {e.group for e in sel.groups("full")}) + with self.assertRaises(selection.SelectionError): + sel.groups("nightly") + + def test_rejections(self): + base = {"schema_version": 1, "enabled": [{"group": "a", "scope": "pr"}]} + cases = [ + ({**base, "enabled": [{"group": "a", "scope": "pr"}, {"group": "a", "scope": "full"}]}, + "enabled twice"), + ({**base, "declined": [{"reason": "r", "groups": ["a"]}]}, "both enabled and declined"), + ({**base, "enabled": [{"group": "a", "scope": "nightly"}]}, "want pr or full"), + ({**base, "enabled": [{"group": "a", "scope": "pr", "tier": 1}]}, "unknown keys"), + ({**base, "enabled": [{"group": "a", "scope": "pr", "timeout_s": 0}]}, "positive"), + # isinstance(True, int) holds. + ({**base, "enabled": [{"group": "a", "scope": "pr", "timeout_s": True}]}, "positive"), + ({**base, "declined": [{"reason": "", "groups": ["b"]}]}, "reason and a group list"), + ({"schema_version": 2, "enabled": []}, "schema_version"), + ] + for doc, fragment in cases: + with self.assertRaises(selection.SelectionError) as cm: + selection.parse(doc, "t.jsonc") + self.assertIn(fragment, str(cm.exception)) + + def test_resolve_ids(self): + universe = ["fake:basic/pass", "fake:basic/fail", "fake:slow/crash"] + chosen, errors = selection.resolve_ids( + ["fake:basic/*", "fake:basic/pass", "fake:basic/pas", "ltp:x"], universe, "fake") + self.assertEqual(chosen, ["fake:basic/pass", "fake:basic/fail"]) + self.assertEqual(len(errors), 2) + self.assertIn("near: fake:basic/pass", errors[0]) + self.assertIn("not a fake id", errors[1]) + chosen, errors = selection.resolve_ids(["fake:basic"], universe, "fake") + self.assertEqual((chosen, errors), (["fake:basic/pass", "fake:basic/fail"], [])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/lib/qemu-ssh.sh b/tests/lib/qemu-ssh.sh index 9ea7a40b..3d1a9b2e 100644 --- a/tests/lib/qemu-ssh.sh +++ b/tests/lib/qemu-ssh.sh @@ -5,7 +5,8 @@ # shellcheck shell=bash # timeout(1) cannot wrap a shell function, so what the callers share is the # argv: qemu_ssh_opts fills QEMU_SSH_OPTS from QEMU_SSH_KEY and QEMU_PORT at -# call time, and each caller builds its own ssh command line around it. +# call time, and each caller builds its own ssh command line around it. The +# conformance backend spells the same list in tests/conformance/backends/ssh.py. # shellcheck disable=SC2034 # Consumed by the sourcing script. qemu_ssh_opts() diff --git a/tests/qemu-runner.sh b/tests/qemu-runner.sh index 097a7c98..2855c186 100755 --- a/tests/qemu-runner.sh +++ b/tests/qemu-runner.sh @@ -41,6 +41,7 @@ QEMU_SHARE_PATH="${QEMU_SHARE_PATH:-${_QR_DIR}}" _QR_PIDFILE="" _QR_LOG="" +_QR_ERR="" _QR_CTL="" # Fixture path inside the VM (always /mnt/host/). @@ -101,6 +102,22 @@ qemu_pick_cpu() esac } +# qemu_stop removes the rundir, so what explains a failed start has to be +# printed before it: the harness sees only this output. +qemu_fail_start() +{ + echo "qemu-runner: $1" >&2 + local f + for f in "$_QR_ERR" "$_QR_LOG"; do + if [ -s "$f" ]; then + echo "qemu-runner: tail of ${f##*/}:" >&2 + tail -n 20 "$f" >&2 + fi + done + qemu_stop + return 1 +} + qemu_start() { qemu_ensure_fixtures || return 1 @@ -122,6 +139,7 @@ qemu_start() rundir="$(mktemp -d -t elfuse-qemu.XXXXXX)" _QR_PIDFILE="${rundir}/qemu.pid" _QR_LOG="${rundir}/qemu-serial.log" + _QR_ERR="${rundir}/qemu.log" _QR_CTL="${rundir}/ssh-ctl" QEMU_PORT="$(qemu_pick_port)" @@ -138,7 +156,7 @@ qemu_start() -nographic -display none -no-reboot -monitor none \ -serial "file:${_QR_LOG}" \ -pidfile "$_QR_PIDFILE" \ - > /dev/null 2>&1 & + > "$_QR_ERR" 2>&1 & disown # Wait for ssh port to come up. @@ -150,8 +168,7 @@ qemu_start() sleep 1 done if ! (echo > /dev/tcp/127.0.0.1/"$QEMU_PORT") 2> /dev/null; then - echo "qemu-runner: VM did not boot within ${QEMU_BOOT_TIMEOUT}s" >&2 - qemu_stop + qemu_fail_start "VM did not boot within ${QEMU_BOOT_TIMEOUT}s" return 1 fi @@ -168,7 +185,10 @@ qemu_start() # a dedicated tmpfs, as any regular system has, so paths under /tmp map to a # resolvable st_dev. Guarded so a repeated qemu_start against a running VM # does not stack mounts. - _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp' + if ! _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp'; then + qemu_fail_start "could not prepare /tmp in the guest" + return 1 + fi } # Each call opens a fresh ssh connection. Avoids ControlMaster pitfalls (master @@ -205,6 +225,14 @@ qemu_stop() if [ -n "$_QR_PIDFILE" ] && [ -s "$_QR_PIDFILE" ]; then local pid pid=$(cat "$_QR_PIDFILE" 2> /dev/null) + + # A state file outlives its VM, so the pid it names may since have been + # recycled. qemu_start gives qemu this pidfile, and mktemp makes the + # path unique, so the argv is what proves the process is ours. + case " $(ps -o command= -p "${pid:-0}" 2> /dev/null) " in + *" -pidfile $_QR_PIDFILE "*) ;; + *) pid="" ;; + esac if [ -n "$pid" ] && kill -0 "$pid" 2> /dev/null; then kill "$pid" 2> /dev/null # give qemu time to exit cleanly; force-kill if it lingers @@ -213,7 +241,15 @@ qemu_stop() kill -0 "$pid" 2> /dev/null || break sleep 1 done - kill -0 "$pid" 2> /dev/null && kill -9 "$pid" 2> /dev/null + if kill -0 "$pid" 2> /dev/null; then + kill -9 "$pid" 2> /dev/null + sleep 1 + # Keep the pidfile and state so a later stop can retry. + if kill -0 "$pid" 2> /dev/null; then + echo "qemu-runner: pid $pid survived SIGKILL" >&2 + return 1 + fi + fi fi fi if [ -n "$_QR_PIDFILE" ]; then @@ -221,33 +257,70 @@ qemu_stop() fi _QR_PIDFILE="" _QR_LOG="" + _QR_ERR="" _QR_CTL="" } -# When sourced, register a cleanup trap that does not clobber the caller's -# existing trap chain. When executed directly, the EXIT trap fires on script -# exit. -trap 'qemu_stop' EXIT +qemu_write_state() +{ + mkdir -p "$(dirname "$1")" || return 1 + printf 'port=%s\nkey=%s\npidfile=%s\n' \ + "$QEMU_PORT" "$QEMU_SSH_KEY" "$_QR_PIDFILE" > "$1.tmp" && mv -f "$1.tmp" "$1" +} + +qemu_read_state() +{ + [ -s "$1" ] || { + echo "qemu-runner: no state file $1" >&2 + return 1 + } + _QR_PIDFILE="$(sed -n 's/^pidfile=//p' "$1")" + + # Restrict cleanup to the directory shape created by mktemp. + case "$_QR_PIDFILE" in + */elfuse-qemu.*/qemu.pid) ;; + *) + echo "qemu-runner: $1 names no qemu-runner pidfile: $_QR_PIDFILE; remove the file once the VM is gone" >&2 + return 1 + ;; + esac +} -# CLI driver: when run directly, support 'qemu-runner.sh start|exec|stop'. if [ "${BASH_SOURCE[0]:-$0}" = "$0" ]; then cmd="${1:-help}" shift || true + state_file="" + if [ "$cmd" != exec ] && [ "${1:-}" = "--state-file" ]; then + state_file="${2:?--state-file needs a path}" + shift 2 + fi case "$cmd" in start) - qemu_start + if [ -n "$state_file" ] && [ -e "$state_file" ]; then + echo "qemu-runner: $state_file names a live VM; run stop first" >&2 + exit 1 + fi + trap 'qemu_stop' EXIT + qemu_start || exit 1 + if [ -n "$state_file" ]; then + qemu_write_state "$state_file" || exit 1 + trap - EXIT + fi echo "PORT=$QEMU_PORT KEY=$QEMU_SSH_KEY" ;; exec) + trap 'qemu_stop' EXIT qemu_start qemu_exec "$@" ;; stop) - qemu_stop + [ -z "$state_file" ] || qemu_read_state "$state_file" || exit 1 + qemu_stop || exit 1 + [ -z "$state_file" ] || rm -f "$state_file" ;; *) cat << EOF -Usage: $0 +Usage: $0 Boots qemu-system-aarch64 with the test fixtures and exposes ssh. The host repo is shared into the VM at /mnt/host (read-only). diff --git a/tests/test-qemu-runner.sh b/tests/test-qemu-runner.sh new file mode 100755 index 00000000..ce01c363 --- /dev/null +++ b/tests/test-qemu-runner.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +# test-qemu-runner.sh -- Pin qemu-runner.sh start and stop against stand-ins +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Usage: tests/test-qemu-runner.sh +# +# A state file outlives its VM, so the pid it records may since have been +# recycled by an unrelated process. stop --state-file has to leave such a +# process alone and still remove the record, and has to terminate a process +# whose argv names the run's own pidfile, as qemu's does. A start that never +# boots has to report what the VM said before its run directory goes. No VM +# boots here: a stub stands in for qemu. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +RUNNER="$SCRIPT_DIR/qemu-runner.sh" +# shellcheck source=tests/lib/report.sh +. "$SCRIPT_DIR/lib/report.sh" + +work="$(mktemp -d)" +victims=() +cleanup() +{ + local p + for p in ${victims[@]+"${victims[@]}"}; do + kill "$p" 2> /dev/null || true + done + rm -rf "$work" +} +trap cleanup EXIT + +# qemu_read_state accepts only the directory shape mktemp gives a run. +rundir="$work/elfuse-qemu.test" +pidfile="$rundir/qemu.pid" +state="$work/qemu.state" + +arm() +{ + mkdir -p "$rundir" + printf '%s\n' "$1" > "$pidfile" + printf 'port=1\nkey=/dev/null\npidfile=%s\n' "$pidfile" > "$state" +} + +check() +{ + local label="$1" want="$2" got="$3" + if [ "$want" = "$got" ]; then + report_pass "$label" + else + report_fail "$label (got $got, want $want)" + fi +} + +alive() +{ + kill -0 "$1" 2> /dev/null && echo alive || echo dead +} + +present() +{ + [ -e "$1" ] && echo present || echo gone +} + +has() +{ + case "$1" in + *"$2"*) echo yes ;; + *) echo no ;; + esac +} + +# A recycled pid: a sleep whose argv never mentions the pidfile. +sleep 300 & +bystander=$! +victims+=("$bystander") +arm "$bystander" +rc=0 +bash "$RUNNER" stop --state-file "$state" > /dev/null 2>&1 || rc=$? +check "stop returns 0 for a recycled pid" 0 "$rc" +check "a recycled pid survives stop" alive "$(alive "$bystander")" +check "stop removes the stale state file" gone "$(present "$state")" +check "stop removes the stale run directory" gone "$(present "$rundir")" + +# The run's own process: its argv carries the pidfile, and it exits on TERM. +loop='trap "exit 0" TERM; while :; do sleep 1; done' +bash -c "$loop" bash -pidfile "$pidfile" & +own=$! +victims+=("$own") +arm "$own" +rc=0 +bash "$RUNNER" stop --state-file "$state" > /dev/null 2>&1 || rc=$? +check "stop returns 0 for the run's own process" 0 "$rc" +check "the run's own process is terminated" dead "$(alive "$own")" +check "stop removes the state file after a kill" gone "$(present "$state")" + +# A failed start: the rundir it would explain itself with is removed on the way +# out, so the runner has to have said everything before returning. +stub="$work/qemu-stub" +cat > "$stub" << 'STUB' +#!/usr/bin/env bash +serial="" +while [ $# -gt 0 ]; do + [ "$1" = -serial ] && serial="${2#file:}" + shift +done +[ -z "$serial" ] || echo SERIAL-MARKER > "$serial" +echo STDERR-MARKER >&2 +STUB +chmod +x "$stub" +fixture="$work/fixture" +echo fixture > "$fixture" +rm -f "$state" +rc=0 +out="$(QEMU_BIN="$stub" QEMU_ACCEL=tcg QEMU_BOOT_TIMEOUT=1 \ + QEMU_KERNEL="$fixture" QEMU_INITRD="$fixture" QEMU_SSH_KEY="$fixture" \ + bash "$RUNNER" start --state-file "$state" 2>&1)" || rc=$? +check "start fails when the VM never boots" 1 "$rc" +check "the failure names the timeout" yes "$(has "$out" "did not boot")" +check "the serial console reaches the caller" yes "$(has "$out" SERIAL-MARKER)" +check "qemu's own output reaches the caller" yes "$(has "$out" STDERR-MARKER)" +check "a failed start writes no state file" gone "$(present "$state")" + +report_summary +[ "$fail" -eq 0 ] From 2e22e0cc95ecf00a40cfe45b1ac6014fdd51a348 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 2 Sep 2026 10:14:39 +0200 Subject: [PATCH 3/5] Add clean-conformance for interrupted runs Every elfuse cleanup is an atexit hook, so a guest killed on timeout, Ctrl-C, or a dead harness leaves its scratch, fork children, and SysV objects behind, and the run leaves the QEMU VM with them. make clean deletes build/conformance/qemu.state, the only record of that VM. scripts/conformance clean sweeps in dependency order: the VM through qemu-runner.sh stop, disowned VMs whose 9p share names this checkout, the harness's detached guest session leaders and --fork-child orphans, then the runtime scratch, and last a chmod u+rwx over the results tree so make clean can remove the mode-0 directories gVisor leaves. Only the exact scratch templates the runtime uses are swept, because review checkouts share the elfuse- prefix. SysV objects are reported, never removed. The guest key reaches the host unchanged, so an elfuse object is indistinguishable from another of the user's, and outliving its creator is ordinary SysV lifecycle. It keeps the lock files, since unlinking a held lock lets two sessions run; the results; another checkout's VMs and guests; and any elfuse run from a terminal. While any elfuse of the user is alive both the scratch sweep and the SysV report are skipped, since every elfuse shares the /tmp names. A held lock makes it refuse and name the pid holding it. --dry-run prefixes every action it would take with would. make clean-conformance is the alias and joins the goals the build-flavor guard skips. --- mk/common.mk | 2 +- mk/conformance.mk | 6 +- tests/conformance/cli.py | 18 +- tests/conformance/selftest/test_cli.py | 37 ++- tests/conformance/selftest/test_sweep.py | 283 +++++++++++++++++ tests/conformance/sweep.py | 372 +++++++++++++++++++++++ 6 files changed, 713 insertions(+), 5 deletions(-) create mode 100644 tests/conformance/selftest/test_sweep.py create mode 100644 tests/conformance/sweep.py diff --git a/mk/common.mk b/mk/common.mk index c8070069..b3d97a5e 100644 --- a/mk/common.mk +++ b/mk/common.mk @@ -86,7 +86,7 @@ BUILD_FLAVOR_STAMP := $(BUILD_DIR)/.build-flavor # skip, that sub-make evaluates the flavor guard with whatever CFLAGS its own # environment produces, so running the scanner beside a sanitizer build wipes # that build's objects from under it. -BUILD_FLAVOR_GOALS := $(filter-out clean distclean help print-%,$(MAKECMDGOALS)) +BUILD_FLAVOR_GOALS := $(filter-out clean distclean clean-conformance help print-%,$(MAKECMDGOALS)) ifneq ($(BUILD_FLAVOR_GOALS),) BUILD_FLAVOR_PREV := $(shell cat $(BUILD_FLAVOR_STAMP) 2>/dev/null) diff --git a/mk/conformance.mk b/mk/conformance.mk index af30828c..808a49f8 100644 --- a/mk/conformance.mk +++ b/mk/conformance.mk @@ -1,5 +1,5 @@ .PHONY: test-conformance-harness test-conformance test-conformance-full \ - conformance-payloads clean-payloads update-pins + conformance-payloads clean-payloads clean-conformance update-pins CONFORMANCE := python3 scripts/conformance # The suite registry lives in tests/conformance/providers/__init__.py. On a @@ -45,6 +45,10 @@ conformance-payloads: clean-payloads: rm -rf externals/payloads +## Stop a leaked QEMU VM, kill orphaned guests, sweep elfuse scratch (results survive) +clean-conformance: + $(CONFORMANCE) clean --results $(CONF_RESULTS) + UPDATE_CHECK ?= ## Refresh the conformance pins from upstream (UPDATE_CHECK=1 to report only) diff --git a/tests/conformance/cli.py b/tests/conformance/cli.py index 0b7d326d..cae14d41 100644 --- a/tests/conformance/cli.py +++ b/tests/conformance/cli.py @@ -16,7 +16,7 @@ from conformance import EXIT_DRIFT, EXIT_OK, EXIT_RED, EXIT_SKIP, EXIT_USAGE from conformance import backends, expectations, jsonc, payload, providers, report -from conformance import runner, seed, selection +from conformance import runner, seed, selection, sweep from conformance.backends.base import BackendError from conformance.model import Status from conformance.providers.base import Provider, ProviderError @@ -325,6 +325,19 @@ def report(self, args: argparse.Namespace) -> int: self.out(line) return EXIT_OK if report.gate(cases) == "green" else EXIT_RED + def clean(self, args: argparse.Namespace) -> int: + job = sweep.Sweep(self.repo_root, Path(args.results), self.out, self.fail, + dry_run=args.dry_run) + try: + with job.elfuse.serialize(), job.qemu.serialize(): + return EXIT_OK if job.run() else EXIT_RED + except BackendError as e: + self.fail(str(e)) + pids = sweep.lock_openers([job.elfuse.lock_file, job.qemu.lock_file]) + if pids: + self.fail("held open by pid %s" % " ".join(map(str, pids))) + return EXIT_USAGE + def selftest(self, args: argparse.Namespace) -> int: suite = unittest.defaultTestLoader.discover( str(self.repo_root / "tests" / "conformance" / "selftest"), @@ -421,6 +434,9 @@ def build_parser(suites: List[str]) -> argparse.ArgumentParser: p.add_argument("results") p.add_argument("--format", choices=("text", "markdown", "json"), default="text") + p = command(top, "clean", "clean") + p.add_argument("--results", default="build/conformance") + p.add_argument("--dry-run", action="store_true") command(top, "selftest", "selftest") return parser diff --git a/tests/conformance/selftest/test_cli.py b/tests/conformance/selftest/test_cli.py index de127145..7c4f7b77 100644 --- a/tests/conformance/selftest/test_cli.py +++ b/tests/conformance/selftest/test_cli.py @@ -8,7 +8,7 @@ import unittest.mock from pathlib import Path -from conformance import cli, providers, report +from conformance import cli, providers, report, sweep from conformance.backends.base import BackendError from conformance.selftest.fixture import FixtureProvider, LocalBackend, TempDirTest, setup @@ -139,9 +139,42 @@ def test_a_provider_must_declare_its_registry_name(self): with self.assertRaisesRegex(providers.ProviderError, "declares name"): providers.make("other", self.root) + def clean(self, *args): + real = sweep.Sweep + + def local(*a, **kw): + job = real(*a, tmp=self.root / "tmp", user_tmp=self.root / "ut", + roots=[self.root / "T"], **kw) + job.elfuse.lock_file = self.root / "lock" + return job + + (self.root / "tmp" / "elfuse-fork-Ab12Cd").mkdir(parents=True, exist_ok=True) + with unittest.mock.patch.object(sweep, "Sweep", local), \ + unittest.mock.patch.object(sweep, "ps_listing", return_value=""), \ + unittest.mock.patch.object(sweep, "ipcs_listing", return_value=""), \ + unittest.mock.patch.object(sweep, "lock_openers", return_value=[4242]): + return self.invoke("clean", "--results", str(self.root / "r"), *args) + + def test_clean_dry_run_lists_the_scratch_it_would_remove(self): + self.assertEqual(self.clean("--dry-run"), 0) + self.assertEqual(self.out, ["would rm -rf %s" % (self.root / "tmp" / "elfuse-fork-Ab12Cd")]) + self.assertEqual(self.err, []) + self.assertTrue((self.root / "tmp" / "elfuse-fork-Ab12Cd").is_dir()) + self.assertEqual(self.clean(), 0) + self.assertFalse((self.root / "tmp" / "elfuse-fork-Ab12Cd").exists()) + + def test_clean_refuses_while_a_session_holds_a_lock(self): + holder = LocalBackend() + holder.lock_file = self.root / "lock" + with holder.serialize(): + self.assertEqual(self.clean("--dry-run"), 2) + self.assertEqual(self.out, []) + self.assertIn("another elfuse conformance session holds", self.err[0]) + self.assertEqual(self.err[1], "conformance: held open by pid 4242") + def test_parser_has_no_suite_owned_commands(self): help_text = cli.build_parser(["fixture"]).format_help() - self.assertIn("{suites,list,run,payload,selection,expectations,pins,report,selftest}", + self.assertIn("{suites,list,run,payload,selection,expectations,pins,report,clean,selftest}", help_text) self.assertNotIn("fixture}", help_text) diff --git a/tests/conformance/selftest/test_sweep.py b/tests/conformance/selftest/test_sweep.py new file mode 100644 index 00000000..2229bf49 --- /dev/null +++ b/tests/conformance/selftest/test_sweep.py @@ -0,0 +1,283 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import stat +import unittest +import unittest.mock +from pathlib import Path + +from conformance import sweep +from conformance.backends import qemu +from conformance.selftest.fixture import TempDirTest +from conformance.selftest.test_backends import script + +BIN = "/repo/build/elfuse" +SHARE = "-fsdev local,id=share,path=%s,security_model=none,readonly=on" +PS = "\n".join([ + " 40 1 40 501 ?? %s --timeout 0 /bin/true" % BIN, + " 41 40 40 501 ?? %s --fork-child 8" % BIN, + " 42 1 42 501 ttys003 %s --timeout 0 /bin/true" % BIN, + " 43 1 43 501 ?? %s /bin/true" % BIN, + " 44 1 44 502 ?? %s --timeout 0 /bin/true" % BIN, + " 46 1 40 501 ?? %s --fork-child 9" % BIN, + " 47 1 47 501 ?? %s --fork-child 9" % BIN, + " 48 1 48 501 ?? /other/build/elfuse --fork-child 9", + " 50 1 50 501 ?? qemu-system-aarch64 -machine virt %s" + " -pidfile /var/T/elfuse-qemu.XXXXXX.abc/qemu.pid" % (SHARE % "/repo"), + " 51 1 51 501 ?? qemu-system-aarch64 %s" + " -pidfile /var/T/elfuse-qemu.XXXXXX.def/qemu.pid" % (SHARE % "/elsewhere"), + " 52 900 52 501 ?? qemu-system-aarch64 %s" + " -pidfile /var/T/elfuse-qemu.XXXXXX.ghi/qemu.pid" % (SHARE % "/repo"), + " 60 1 60 501 ttys001 /bin/zsh -c ls /Users/x/code/elfuse/src", + " 61 1 61 501 ?? (elfuse)", +]) + "\n" +IPCS_M = ("IPC status from as of Wed Sep 2 10:04:00 CEST 2026\n" + "T ID KEY MODE OWNER GROUP CPID LPID\n" + "Shared Memory:\n" + "m 65536 0x00000000 --rw------- henry staff 4321 4321\n" + "m 65537 0x0000abcd --rw------- henry staff 4322 4322\n\n") +IPCS_Q = ("T ID KEY MODE OWNER GROUP LSPID LRPID\n" + "Message Queues:\n" + "q 0 0x00000000 --rw------- henry staff 0 0\n" + "q 1 0x00000000 --rw------- henry staff 4321 0\n\n") +IPCS_S = ("T ID KEY MODE OWNER GROUP\n" + "Semaphores:\n" + "s 393216 0x6111bfce --ra-ra-ra- root wheel\n\n") + + +def pids(procs): + return [p.pid for p in procs] + + +class MatcherTest(unittest.TestCase): + def setUp(self): + self.procs = sweep.processes(PS) + + def test_only_a_detached_harness_leader_is_a_guest_group(self): + self.assertEqual(pids(sweep.guest_groups(self.procs, BIN, 501)), [40]) + + def test_fork_orphans_need_ppid_one_and_this_binary(self): + self.assertEqual(pids(sweep.fork_orphans(self.procs, BIN, 501)), [46, 47]) + + def test_a_stray_vm_names_this_checkout_and_is_reparented(self): + found = sweep.stray_qemu(self.procs, Path("/repo"), 501) + self.assertEqual([(p.pid, str(f)) for p, f in found], + [(50, "/var/T/elfuse-qemu.XXXXXX.abc/qemu.pid")]) + + def test_a_pathname_mentioning_elfuse_is_not_a_live_elfuse(self): + self.assertEqual(pids(sweep.live_elfuse(self.procs, 501)), + [40, 41, 42, 43, 46, 47, 48]) + + def test_ipc_rows_skip_the_preamble(self): + rows = sweep.ipc_rows(IPCS_M + IPCS_Q + IPCS_S) + self.assertEqual([(r.kind, r.ident, r.pids) for r in rows], [ + ("m", 65536, (4321, 4321)), ("m", 65537, (4322, 4322)), + ("q", 0, (0, 0)), ("q", 1, (4321, 0)), ("s", 393216, ()), + ]) + + def test_dead_ipc_reports_only_what_a_dead_pid_owned(self): + rows = sweep.ipc_rows(IPCS_M + IPCS_Q + IPCS_S) + leaked, keep = sweep.dead_ipc(rows, lambda pid: pid == 4322) + self.assertEqual([(r.kind, r.ident) for r, _ in leaked], [("m", 65536), ("q", 1)]) + self.assertEqual([(r.kind, r.ident) for r, _ in keep], + [("m", 65537), ("q", 0), ("s", 393216)]) + self.assertEqual(keep[2][1], "no creator pid in ipcs") + + +class PlantedTest(TempDirTest): + def plant(self): + self.tmp, self.user_tmp, self.t = self.dir / "tmp", self.dir / "ut", self.dir / "T" + for rel in ("tmp/elfuse-fork-Ab12Cd", "tmp/elfuse-task-Ab12Cd/123", + "tmp/elfuse-absock-42", "tmp/elfuse-shm-501", + "tmp/elfuse-review-clone.abc", "tmp/elfuse-conf-split.abc", + "tmp/elfuse-fork-toolong1", "tmp/elfuse-fork-Ab12C", + "T/elfuse-qemu.XXXXXX.abc", "T/elfuse-qemu.XXXXXX.def", "ut"): + (self.dir / rel).mkdir(parents=True) + for rel in ("tmp/elfuse-fork-Ab12Cd/snap", "tmp/elfuse-fuse-exec.Ab12Cd", + "tmp/elfuse-absock-42/x", "tmp/elfuse-shm-501/f", + "tmp/elfuse-shm-502", "tmp/elfuse-pr341-comments.json", + "T/elfuse-qemu.XXXXXX.abc/qemu.pid", "T/elfuse-qemu.XXXXXX.def/qemu.pid", + "T/elfuse-qemu.plainfile", "ut/elfuse-sig-4321", "ut/elfuse-procs-77", + "ut/elfuse-sig-x"): + (self.dir / rel).write_text("") + + def names(self, root): + return sorted(p.name for p in root.iterdir()) + + +class LayoutTest(PlantedTest): + def test_runtime_paths_match_the_fixed_templates_only(self): + self.plant() + found = sweep.runtime_paths(self.tmp, self.user_tmp, 501) + self.assertEqual([str(p.relative_to(self.dir)) for p in found], [ + "tmp/elfuse-absock-42", "tmp/elfuse-fork-Ab12Cd", "tmp/elfuse-fuse-exec.Ab12Cd", + "tmp/elfuse-shm-501/f", "tmp/elfuse-task-Ab12Cd", + "ut/elfuse-procs-77", "ut/elfuse-sig-4321", + ]) + self.assertEqual(sweep.runtime_paths(self.dir / "absent", None, 501), []) + + def test_qemu_rundirs_are_directories_under_every_root(self): + self.plant() + found = sweep.qemu_rundirs([self.t, self.dir / "absent"]) + self.assertEqual([p.name for p in found], + ["elfuse-qemu.XXXXXX.abc", "elfuse-qemu.XXXXXX.def"]) + + @unittest.skipIf(os.geteuid() == 0, "root ignores directory permissions") + def test_fix_modes_opens_nested_directories(self): + root = self.dir / "results" + inner = root / "a" / "gvisor_test_temp_1" / "b" / "gvisor_test_temp_2" + inner.mkdir(parents=True) + (root / "a" / "link").symlink_to("/") + (inner.parent / "b2").mkdir() + inner.chmod(0) + (root / "a" / "gvisor_test_temp_1").chmod(0o500) + try: + listed = sweep.fix_modes(root, apply=False) + self.assertEqual(listed, [root / "a" / "gvisor_test_temp_1"]) + fixed = sweep.fix_modes(root, apply=True) + self.assertEqual(sorted(fixed), sorted([root / "a" / "gvisor_test_temp_1", inner])) + self.assertEqual(stat.S_IMODE(inner.stat().st_mode) & 0o700, 0o700) + self.assertEqual(sweep.fix_modes(root, apply=True), []) + finally: + for d in (inner, inner.parents[1]): + d.chmod(0o700) + + +class SweepTest(PlantedTest): + def setUp(self): + super().setUp() + self.plant() + self.out, self.err = [], [] + self.state_dir = self.dir / "state" + self.state_dir.mkdir() + self.verbs = self.dir / "verbs" + self.runner = script(self.dir / "runner.sh", + 'echo "$1" >> "%s"\n[ "$1" = stop ] && rm -f "$3"\nexit 0\n' + % self.verbs) + + def sweep(self, ps="", dry_run=False, runner=None): + backend = qemu.QemuBackend(self.dir, runner=runner or self.runner, state_dir=self.state_dir) + job = sweep.Sweep(self.dir, self.dir / "results", self.out.append, self.err.append, + dry_run=dry_run, uid=501, tmp=self.tmp, user_tmp=self.user_tmp, + roots=[self.t], qemu=backend) + ipcs = {"m": IPCS_M, "q": IPCS_Q, "s": IPCS_S} + listings = ps if isinstance(ps, list) else [ps, ps] + with unittest.mock.patch.object(sweep, "ps_listing", side_effect=listings), \ + unittest.mock.patch.object(sweep, "ipcs_listing", lambda kind, uid: ipcs[kind]), \ + unittest.mock.patch.object(sweep, "alive", lambda pid: pid == 4322), \ + unittest.mock.patch.object(sweep.subprocess, "run") as run: + ok = job.run() + return ok, run + + def test_a_dead_host_is_swept_and_decoys_survive(self): + (self.state_dir / "qemu.state").write_text( + "port=1\nkey=/k\npidfile=%s/elfuse-qemu.XXXXXX.abc/qemu.pid\n" % self.t) + ok, run = self.sweep() + self.assertTrue(ok, self.err) + self.assertEqual(self.err, []) + self.assertEqual(self.verbs.read_text().split(), ["stop"]) + self.assertEqual(self.names(self.tmp), [ + "elfuse-conf-split.abc", "elfuse-fork-Ab12C", "elfuse-fork-toolong1", + "elfuse-pr341-comments.json", "elfuse-review-clone.abc", + "elfuse-shm-501", "elfuse-shm-502", + ]) + self.assertEqual(self.names(self.tmp / "elfuse-shm-501"), []) + self.assertEqual(self.names(self.user_tmp), ["elfuse-sig-x"]) + self.assertEqual(self.names(self.t), ["elfuse-qemu.plainfile"]) + self.assertEqual(run.call_args_list, []) # SysV objects are reported only + self.assertIn("leaked ipc -m 65536: cpid 4321 dead; ipcrm -m 65536 by hand", self.out) + self.assertIn("leaked ipc -q 1: pids 4321 dead; ipcrm -q 1 by hand", self.out) + self.assertIn("keep ipc -s: 1 sets, no creator pid in ipcs; ipcrm -s ID by hand", self.out) + self.assertIn("keep ipc -m 65537: cpid 4322 alive", self.out) + self.assertIn("keep ipc -q 0: never used; ipcrm -q 0 by hand", self.out) + self.assertNotIn("keep runtime scratch", " ".join(self.out)) + + def test_dry_run_reports_the_same_actions_and_touches_nothing(self): + before = sorted(str(p) for p in self.dir.rglob("*")) + ok, run = self.sweep(dry_run=True) + self.assertTrue(ok) + self.assertEqual(sorted(str(p) for p in self.dir.rglob("*")), before) + self.assertEqual(run.call_args_list, []) + self.assertTrue(all(line.startswith(("would ", "keep ", "leaked ")) for line in self.out), + self.out) + self.assertIn("would rm -rf %s" % (self.tmp / "elfuse-fork-Ab12Cd"), self.out) + self.assertIn("leaked ipc -m 65536: cpid 4321 dead; ipcrm -m 65536 by hand", self.out) + + def test_a_live_elfuse_keeps_the_runtime_scratch(self): + ok, run = self.sweep(ps=" 70 1 70 501 ?? /other/build/elfuse /bin/true\n") + self.assertTrue(ok) + self.assertIn("keep runtime scratch: elfuse pids 70 are alive", self.out) + self.assertTrue((self.tmp / "elfuse-fork-Ab12Cd").is_dir()) + self.assertEqual(run.call_args_list, []) + + def test_kills_target_only_this_checkout(self): + binary = str(self.dir / "build" / "elfuse") + ps = (" 40 1 40 501 ?? %s --timeout 0 /bin/true\n" + " 46 1 40 501 ?? %s --fork-child 9\n" + " 47 1 47 501 ?? %s --fork-child 9\n" + " 50 1 50 501 ?? qemu-system-aarch64 %s -pidfile %s/elfuse-qemu.XXXXXX.abc/qemu.pid\n" + " 51 1 51 501 ?? qemu-system-aarch64 %s -pidfile %s/elfuse-qemu.XXXXXX.def/qemu.pid\n" + % (binary, binary, binary, SHARE % self.dir, self.t, SHARE % "/elsewhere", self.t)) + with unittest.mock.patch.object(sweep.os, "killpg") as killpg, \ + unittest.mock.patch.object(sweep.os, "kill") as kill, \ + unittest.mock.patch.object(sweep, "terminate") as terminate: + ok, run = self.sweep(ps=ps) + self.assertTrue(ok, self.err) + self.assertEqual(terminate.call_args_list, [unittest.mock.call(50)]) + self.assertEqual(killpg.call_args_list, [unittest.mock.call(40, sweep.signal.SIGKILL)]) + self.assertEqual(kill.call_args_list, [unittest.mock.call(47, sweep.signal.SIGKILL)]) + # The other checkout's VM keeps its rundir; ours goes with its process, + # and with every guest gone the scratch sweep runs. + self.assertEqual(self.names(self.t), ["elfuse-qemu.XXXXXX.def", "elfuse-qemu.plainfile"]) + self.assertFalse((self.tmp / "elfuse-fork-Ab12Cd").exists()) + + def test_the_stopped_vm_is_not_signalled_again(self): + rundir = self.t / "elfuse-qemu.XXXXXX.abc" + pidfile = rundir / "qemu.pid" + (self.state_dir / "qemu.state").write_text( + "port=1\nkey=/k\npidfile=%s\n" % pidfile) + # qemu-runner.sh disowns the VM, so it outlives the shell that started + # it and is still in the pre-stop snapshot the sweep works from. + ps = (" 50 1 50 501 ?? qemu-system-aarch64 %s -pidfile %s\n" + % (SHARE % self.dir, pidfile)) + with unittest.mock.patch.object(sweep, "terminate") as terminate: + ok, _ = self.sweep(ps=ps) + self.assertTrue(ok, self.err) + self.assertEqual(self.err, []) + self.assertEqual(terminate.call_args_list, []) + self.assertEqual(self.verbs.read_text().split(), ["stop"]) + self.assertEqual(self.names(self.t), ["elfuse-qemu.plainfile"]) + + def test_a_results_tree_it_cannot_walk_is_reported(self): + (self.dir / "results").mkdir() + with unittest.mock.patch.object(sweep, "fix_modes", + side_effect=OSError(13, "Permission denied")): + ok, _ = self.sweep() + self.assertFalse(ok) + self.assertIn("chmod u+rwx under", self.err[0]) + + def test_a_vm_that_exits_during_the_sweep_loses_its_rundir(self): + ps = (" 51 1 51 501 ?? qemu-system-aarch64 %s -pidfile %s\n" + % (SHARE % "/elsewhere", self.t / "elfuse-qemu.XXXXXX.def" / "qemu.pid")) + ok, _ = self.sweep(ps=[ps, ""]) + self.assertTrue(ok, self.err) + self.assertEqual(self.names(self.t), ["elfuse-qemu.plainfile"]) + + def test_a_state_file_naming_a_live_vm_survives_a_failed_stop(self): + runner = script(self.dir / "failing.sh", 'echo "no such vm"; exit 1\n') + state = self.state_dir / "qemu.state" + pidfile = "%s/elfuse-qemu.XXXXXX.def/qemu.pid" % self.t + state.write_text("port=1\nkey=/k\npidfile=%s\n" % pidfile) + ps = (" 51 1 51 501 ?? qemu-system-aarch64 %s -pidfile %s\n" + % (SHARE % "/elsewhere", pidfile)) + ok, run = self.sweep(ps=ps, runner=runner) + self.assertFalse(ok) + self.assertIn("no such vm", self.err[0]) + self.assertTrue(state.exists()) + self.assertTrue((self.t / "elfuse-qemu.XXXXXX.def").is_dir()) + self.assertFalse((self.t / "elfuse-qemu.XXXXXX.abc").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/sweep.py b/tests/conformance/sweep.py new file mode 100644 index 00000000..ec0c24df --- /dev/null +++ b/tests/conformance/sweep.py @@ -0,0 +1,372 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import re +import shutil +import signal +import stat +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Callable, Iterable, List, NamedTuple, Optional, Tuple + +from conformance.backends.base import BackendError +from conformance.backends.elfuse import ElfuseBackend +from conformance.backends.qemu import QemuBackend, parse_state + +# elfuse names its host scratch with fixed templates (src/runtime/procemu.c, +# procemu-pty.c, forkipc.c, usb-sysfs.c, src/syscall/fuse.c, net-absock.c, +# proc.c). Review scratch shares the elfuse- prefix, so only these exact +# shapes are swept. The tmpfile_anon files in src/utils.h are unlinked at +# creation, so no name of theirs survives a kill. +RUNTIME_NAME = re.compile( + r"elfuse-(?:fork|proc|syscpu|tid|task|fd|fdinfo|pts|usbsys|usbdev)-[A-Za-z0-9]{6}$" + r"|elfuse-fuse-exec\.[A-Za-z0-9]{6}$" + r"|elfuse-absock-[0-9]+$" +) +TRANSPORT_NAME = re.compile(r"elfuse-(?:sig|procs|life|pidseq)-[0-9]+$") +# macOS mktemp keeps the literal XXXXXX and appends its own suffix. +QEMU_PIDFILE = re.compile(r" -pidfile (\S+/elfuse-qemu\.[^/\s]+/qemu\.pid)(?= |$)") +TERM_WAIT_S = 5 + + +class Process(NamedTuple): + pid: int + ppid: int + pgid: int + uid: int + tty: str + command: str + + +class IpcRow(NamedTuple): + kind: str + ident: int + pids: Tuple[int, ...] + + +def ps_listing() -> str: + return subprocess.run(["ps", "-eo", "pid=,ppid=,pgid=,uid=,tty=,command="], + capture_output=True, text=True).stdout + + +def ipcs_listing(kind: str, uid: int) -> str: + argv = ["ipcs", "-" + kind, "-u", str(uid)] + if kind != "s": + argv.insert(2, "-p") + return subprocess.run(argv, capture_output=True, text=True).stdout + + +def user_temp_dir() -> Optional[Path]: + """The per-user temp dir elfuse's proc.c transports live in (macOS only).""" + done = subprocess.run(["getconf", "DARWIN_USER_TEMP_DIR"], + capture_output=True, text=True) + text = done.stdout.strip() + return Path(text) if done.returncode == 0 and text else None + + +def alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + pass + return True + + +def processes(ps_text: str) -> List[Process]: + out = [] + for line in ps_text.splitlines(): + fields = line.split(None, 5) + if len(fields) == 6: + out.append(Process(int(fields[0]), int(fields[1]), int(fields[2]), + int(fields[3]), fields[4], fields[5])) + return out + + +def guest_groups(procs: Iterable[Process], binary: str, uid: int) -> List[Process]: + """Session leaders the harness started: backends/proc.py spawns each case + with start_new_session, so its pid is the pgid, while a make lane inherits + make's group. An interactive run is spared by its tty.""" + return [p for p in procs + if p.uid == uid and p.pgid == p.pid and p.tty in ("??", "?") + and p.command.startswith(binary + " --timeout 0 ")] + + +def fork_orphans(procs: Iterable[Process], binary: str, uid: int) -> List[Process]: + return [p for p in procs + if p.uid == uid and p.ppid == 1 and p.command.startswith(binary + " ") + and "--fork-child" in p.command.split()] + + +def qemu_pidfile(p: Process) -> Optional[Path]: + m = QEMU_PIDFILE.search(p.command) + return Path(m.group(1)) if m else None + + +def stray_qemu(procs: Iterable[Process], repo_root: Path, uid: int) -> List[Tuple[Process, Path]]: + """VMs qemu-runner.sh disowned for this checkout; the 9p share names the + checkout, so another worktree's VM does not match.""" + shares = {"path=%s," % root for root in (repo_root, repo_root.resolve())} + out = [] + for p in procs: + pidfile = qemu_pidfile(p) + if (pidfile is not None and p.uid == uid and p.ppid == 1 + and any(share in p.command for share in shares)): + out.append((p, pidfile)) + return out + + +def live_elfuse(procs: Iterable[Process], uid: int) -> List[Process]: + """Any elfuse of this uid, whatever its checkout: the /tmp scratch names are + shared by all of them.""" + return [p for p in procs + if p.uid == uid and os.path.basename(p.command.split(None, 1)[0]) == "elfuse"] + + +def runtime_paths(tmp: Path, user_tmp: Optional[Path], uid: int) -> List[Path]: + out = [p for p in tmp.iterdir() if RUNTIME_NAME.match(p.name)] if tmp.is_dir() else [] + shm = tmp / ("elfuse-shm-%d" % uid) + if shm.is_dir(): + out += list(shm.iterdir()) # the dir itself is shared by every elfuse of the uid + if user_tmp is not None and user_tmp.is_dir(): + out += [p for p in user_tmp.iterdir() if TRANSPORT_NAME.match(p.name)] + return sorted(out) + + +def qemu_rundirs(roots: Iterable[Path]) -> List[Path]: + out = [] + for root in roots: + if root.is_dir(): + out += [p for p in root.iterdir() + if p.name.startswith("elfuse-qemu.") and p.is_dir() and not p.is_symlink()] + return sorted(out) + + +def temp_roots() -> List[Path]: + """mktemp -t and Python disagree on the default when TMPDIR is unset.""" + out: List[Path] = [] + for root in (Path(tempfile.gettempdir()), user_temp_dir(), Path("/tmp")): + if root is not None and root.resolve() not in [r.resolve() for r in out]: + out.append(root) + return out + + +def ipc_rows(text: str) -> List[IpcRow]: + """Rows of ipcs -m -p / -q -p / -s on macOS: T ID KEY MODE OWNER GROUP, then + CPID LPID for shm and LSPID LRPID for queues; semaphores carry no pid.""" + out = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) >= 6 and fields[0] in ("m", "q", "s") and fields[1].isdigit(): + pids = tuple(int(f) for f in fields[6:8] if f.isdigit()) + out.append(IpcRow(fields[0], int(fields[1]), pids)) + return out + + +def dead_ipc(rows: Iterable[IpcRow], is_alive: Callable[[int], bool] + ) -> Tuple[List[Tuple[IpcRow, str]], List[Tuple[IpcRow, str]]]: + """Split SysV objects into leaked and kept, each with its reason.""" + leaked_rows, keep_rows = [], [] + for row in rows: + if row.kind == "s": + keep_rows.append((row, "no creator pid in ipcs")) + elif row.kind == "m": + cpid = row.pids[0] if row.pids else 0 + if not cpid: + keep_rows.append((row, "no creator pid in ipcs")) + elif not is_alive(cpid): + leaked_rows.append((row, "cpid %d dead" % cpid)) + else: + keep_rows.append((row, "cpid %d alive" % cpid)) + else: + users = [pid for pid in row.pids if pid] + if not users: + keep_rows.append((row, "never used; ipcrm -q %d by hand" % row.ident)) + elif all(not is_alive(pid) for pid in users): + leaked_rows.append((row, "pids %s dead" % " ".join(map(str, users)))) + else: + keep_rows.append((row, "in use")) + return leaked_rows, keep_rows + + +def fix_modes(root: Path, apply: bool) -> List[Path]: + """Directories rm -rf cannot descend into. Without apply the walk stops at + each one, since a mode 0 directory cannot be opened to look below it.""" + out = [] + stack = [root] + while stack: + d = stack.pop() + mode = stat.S_IMODE(os.lstat(d).st_mode) + if mode & 0o700 != 0o700: + out.append(d) + if not apply: + continue + os.chmod(d, mode | 0o700) + with os.scandir(d) as entries: + stack += [Path(e.path) for e in entries if e.is_dir(follow_symlinks=False)] + return out + + +def lock_openers(paths: Iterable[Path]) -> List[int]: + """lsof names openers, not flock holders; serialize() keeps the fd open only + while it holds the lock, so here the two coincide.""" + pids = set() + for path in paths: + if path.exists(): + done = subprocess.run(["lsof", "-t", str(path)], capture_output=True, text=True) + pids.update(int(pid) for pid in done.stdout.split() if pid.isdigit()) + return sorted(pids) + + +def remove(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + +def terminate(pid: int) -> None: + """A pid already gone is the goal, not a failure.""" + try: + os.kill(pid, signal.SIGTERM) + deadline = time.monotonic() + TERM_WAIT_S + while time.monotonic() < deadline: + if not alive(pid): + return + time.sleep(0.1) + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +class Sweep: + """Order matters: the VM goes before its state record, kills before the + scratch sweep, and the scratch sweep only when no elfuse is left alive.""" + + def __init__(self, repo_root: Path, results: Path, out: Callable[[str], None], + fail: Callable[[str], None], dry_run: bool = False, + uid: Optional[int] = None, tmp: Path = Path("/tmp"), + user_tmp: Optional[Path] = None, roots: Optional[List[Path]] = None, + qemu: Optional[QemuBackend] = None): + self.repo_root = repo_root + self.results = results + self.out = out + self.fail = fail + self.dry_run = dry_run + self.uid = os.getuid() if uid is None else uid + self.tmp = tmp + self.user_tmp = user_temp_dir() if user_tmp is None else user_tmp + self.roots = temp_roots() if roots is None else roots + self.qemu = qemu or QemuBackend(repo_root) + self.elfuse = ElfuseBackend(repo_root) + self.failed = False + + def act(self, line: str, action: Callable[[], None]) -> bool: + self.out(("would " if self.dry_run else "") + line) + if self.dry_run: + return True + try: + action() + except (OSError, BackendError) as e: + self.fail("%s: %s" % (line, e)) + self.failed = True + return False + return True + + def run(self) -> bool: + procs = processes(ps_listing()) + self.stop_qemu(procs) + gone = self.kill_guests(procs) + left = [p for p in live_elfuse(procs, self.uid) + if p.pid not in gone and p.pgid not in gone] + if left: + self.out("keep runtime scratch: elfuse pids %s are alive" + % " ".join(str(p.pid) for p in left)) + else: + self.sweep_runtime() + self.report_ipc() + if self.results.is_dir(): + self.fix_results_modes() + return not self.failed + + def fix_results_modes(self) -> None: + try: + found = fix_modes(self.results, not self.dry_run) + except OSError as e: + self.fail("chmod u+rwx under %s: %s" % (self.results, e)) + self.failed = True + return + for d in found: + self.out(("would " if self.dry_run else "") + "chmod u+rwx %s" % d) + + def stop_qemu(self, procs: List[Process]) -> None: + state = self.qemu.state_file + stopped = None + if state.exists(): + named = parse_state(state.read_text()).get("pidfile") + if self.act("stop qemu %s" % state, self.qemu.stop) and named is not None: + stopped = Path(named) + killed = set() + for p, pidfile in stray_qemu(procs, self.repo_root, self.uid): + # ps is a pre-stop snapshot, and a VM the runner disowned outlives + # its shell, so the one just stopped still matches here. + if pidfile == stopped: + killed.add(pidfile) + continue + if self.act("kill -TERM %d qemu-system-aarch64 (-pidfile %s)" % (p.pid, pidfile), + lambda pid=p.pid: terminate(pid)): + killed.add(pidfile) + # A fresh listing: a VM that exited on its own during the sweep would + # otherwise still read as referenced, and keep its rundir. + live = processes(ps_listing()) + referenced = {f for f in map(qemu_pidfile, live) if f is not None} - killed + for rundir in qemu_rundirs(self.roots): + if rundir / "qemu.pid" not in referenced: + self.act("rm -rf %s" % rundir, lambda d=rundir: shutil.rmtree(d)) + for leftover in (state, state.with_name(state.name + ".tmp")): + if not leftover.exists(): + continue + named = parse_state(leftover.read_text()).get("pidfile") + if named is None or Path(named) not in referenced: + self.act("rm -f %s" % leftover, leftover.unlink) + + def kill_guests(self, procs: List[Process]) -> set: + binary = str(self.elfuse.binary) + gone = set() + for p in guest_groups(procs, binary, self.uid): + if self.act("killpg -KILL %d %s" % (p.pgid, p.command), + lambda pgid=p.pgid: os.killpg(pgid, signal.SIGKILL)): + gone.add(p.pgid) + for p in fork_orphans(procs, binary, self.uid): + if p.pgid in gone: + continue + if self.act("kill -KILL %d %s" % (p.pid, p.command), + lambda pid=p.pid: os.kill(pid, signal.SIGKILL)): + gone.add(p.pid) + return gone + + def sweep_runtime(self) -> None: + for path in runtime_paths(self.tmp, self.user_tmp, self.uid): + self.act("rm -rf %s" % path, lambda p=path: remove(p)) + + def report_ipc(self) -> None: + """Report only. sys_shmget and sys_msgget forward the guest key to the + host unchanged, so an elfuse object is indistinguishable from any other + of the user's, and outliving its creator is ordinary SysV lifecycle.""" + rows = [row for kind in ("m", "q", "s") for row in ipc_rows(ipcs_listing(kind, self.uid))] + leaked_rows, keep_rows = dead_ipc(rows, alive) + for row, why in leaked_rows: + self.out("leaked ipc -%s %d: %s; ipcrm -%s %d by hand" + % (row.kind, row.ident, why, row.kind, row.ident)) + sems = [row for row, _ in keep_rows if row.kind == "s"] + for row, why in keep_rows: + if row.kind != "s": + self.out("keep ipc -%s %d: %s" % (row.kind, row.ident, why)) + if sems: # one line, not one per set: a leaked run leaves thousands + self.out("keep ipc -s: %d sets, no creator pid in ipcs; ipcrm -s ID by hand" % len(sems)) From a94ebe9e6d7bab12c1e214d4b83bdcd822e1454e Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Thu, 3 Sep 2026 09:56:33 +0200 Subject: [PATCH 4/5] Attribute leaked shared memory to its lane clean sees only that a segment's creator is dead, which any long-lived segment of the user's looks like, so it reports them all and removes none. A lane knows more: run records the segments that appeared while it held the backend, by id, key and creator pid, and clean removes those and reports the rest. Shared memory only. A segment's CPID is its creator and never changes, while ipcs gives a queue its last sender and receiver. --- tests/conformance/cli.py | 5 +++ tests/conformance/selftest/test_cli.py | 11 +++++ tests/conformance/selftest/test_sweep.py | 30 ++++++++++++-- tests/conformance/sweep.py | 53 +++++++++++++++++++----- 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/tests/conformance/cli.py b/tests/conformance/cli.py index cae14d41..1e1b57ef 100644 --- a/tests/conformance/cli.py +++ b/tests/conformance/cli.py @@ -174,6 +174,7 @@ def run_one(self, args: argparse.Namespace, provider: Provider, self.out(case.id) return EXIT_OK log = self.err if args.verbose else (lambda _: None) + before = sweep.shm_ids(os.getuid()) results = runner.run_lane( provider, backend, cases, exps, results_dir, args.jobs, not args.no_retry, args.bootstrap, log @@ -189,6 +190,10 @@ def run_one(self, args: argparse.Namespace, provider: Provider, "elapsed_s": round(time.monotonic() - started, 3), "bootstrap": args.bootstrap, "argv": args.argv, + # What this lane created, so clean can tell its own leaks + # from another process's segments. + "shm": [{"id": i, "key": k, "cpid": c} for i, k, c + in sorted(sweep.shm_ids(os.getuid()) - before)], } # Written before stop(), so a failing teardown cannot lose the lane. report.write(results_dir, meta, results) diff --git a/tests/conformance/selftest/test_cli.py b/tests/conformance/selftest/test_cli.py index 7c4f7b77..a8914ffc 100644 --- a/tests/conformance/selftest/test_cli.py +++ b/tests/conformance/selftest/test_cli.py @@ -103,6 +103,17 @@ def test_results_record_the_argv_that_ran(self): doc = json.loads(next(results.rglob(report.RESULTS)).read_text()) self.assertEqual(doc["run"]["argv"], argv) + def test_results_record_the_segments_a_lane_created(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + results = self.root / "results" + before, after = {(1, "0x1", 7)}, {(1, "0x1", 7), (2, "0x2", 8)} + with unittest.mock.patch.object(sweep, "shm_ids", side_effect=[before, after]): + self.assertEqual(self.invoke( + "run", "fixture", "--scope", "pr", "--results", str(results) + ), 0, self.err) + doc = json.loads(next(results.rglob(report.RESULTS)).read_text()) + self.assertEqual(doc["run"]["shm"], [{"id": 2, "key": "0x2", "cpid": 8}]) + def test_case_selector_and_dry_run(self): self.assertEqual(self.invoke("payload", "build", "fixture"), 0) self.assertEqual(self.invoke( diff --git a/tests/conformance/selftest/test_sweep.py b/tests/conformance/selftest/test_sweep.py index 2229bf49..fbb945a3 100644 --- a/tests/conformance/selftest/test_sweep.py +++ b/tests/conformance/selftest/test_sweep.py @@ -1,6 +1,7 @@ # Copyright 2026 elfuse contributors # SPDX-License-Identifier: Apache-2.0 +import json import os import stat import unittest @@ -71,9 +72,11 @@ def test_a_pathname_mentioning_elfuse_is_not_a_live_elfuse(self): def test_ipc_rows_skip_the_preamble(self): rows = sweep.ipc_rows(IPCS_M + IPCS_Q + IPCS_S) - self.assertEqual([(r.kind, r.ident, r.pids) for r in rows], [ - ("m", 65536, (4321, 4321)), ("m", 65537, (4322, 4322)), - ("q", 0, (0, 0)), ("q", 1, (4321, 0)), ("s", 393216, ()), + self.assertEqual([(r.kind, r.ident, r.pids, r.key) for r in rows], [ + ("m", 65536, (4321, 4321), "0x00000000"), + ("m", 65537, (4322, 4322), "0x0000abcd"), + ("q", 0, (0, 0), "0x00000000"), ("q", 1, (4321, 0), "0x00000000"), + ("s", 393216, (), "0x6111bfce"), ]) def test_dead_ipc_reports_only_what_a_dead_pid_owned(self): @@ -257,6 +260,27 @@ def test_a_results_tree_it_cannot_walk_is_reported(self): self.assertFalse(ok) self.assertIn("chmod u+rwx under", self.err[0]) + def test_only_a_recorded_segment_is_removed(self): + run = self.dir / "results" / "fake" / "host" / "1" + run.mkdir(parents=True) + (run / "results.json").write_text(json.dumps({"run": {"shm": [ + {"id": 65536, "key": "0x00000000", "cpid": 4321}]}})) + ok, ipcrm = self.sweep() + self.assertTrue(ok, self.err) + self.assertEqual([c.args[0] for c in ipcrm.call_args_list], + [["ipcrm", "-m", "65536"]]) + self.assertIn("ipcrm -m 65536 (cpid 4321 dead, a run recorded creating it)", self.out) + self.assertIn("leaked ipc -q 1: pids 4321 dead; ipcrm -q 1 by hand", self.out) + + def test_a_queue_sharing_a_recorded_id_is_kept(self): + run = self.dir / "results" / "fake" / "host" / "1" + run.mkdir(parents=True) + (run / "results.json").write_text(json.dumps({"run": {"shm": [ + {"id": 1, "key": "0x00000000", "cpid": 4321}]}})) + ok, ipcrm = self.sweep() + self.assertTrue(ok, self.err) + self.assertEqual(ipcrm.call_args_list, []) + def test_a_vm_that_exits_during_the_sweep_loses_its_rundir(self): ps = (" 51 1 51 501 ?? qemu-system-aarch64 %s -pidfile %s\n" % (SHARE % "/elsewhere", self.t / "elfuse-qemu.XXXXXX.def" / "qemu.pid")) diff --git a/tests/conformance/sweep.py b/tests/conformance/sweep.py index ec0c24df..cd3ea684 100644 --- a/tests/conformance/sweep.py +++ b/tests/conformance/sweep.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import os import re import shutil @@ -12,8 +13,9 @@ import tempfile import time from pathlib import Path -from typing import Callable, Iterable, List, NamedTuple, Optional, Tuple +from typing import Callable, Iterable, List, NamedTuple, Optional, Set, Tuple +from conformance import report from conformance.backends.base import BackendError from conformance.backends.elfuse import ElfuseBackend from conformance.backends.qemu import QemuBackend, parse_state @@ -47,6 +49,7 @@ class IpcRow(NamedTuple): kind: str ident: int pids: Tuple[int, ...] + key: str = "" def ps_listing() -> str: @@ -165,7 +168,29 @@ def ipc_rows(text: str) -> List[IpcRow]: fields = line.split() if len(fields) >= 6 and fields[0] in ("m", "q", "s") and fields[1].isdigit(): pids = tuple(int(f) for f in fields[6:8] if f.isdigit()) - out.append(IpcRow(fields[0], int(fields[1]), pids)) + out.append(IpcRow(fields[0], int(fields[1]), pids, fields[2])) + return out + + +def shm_ids(uid: int) -> Set[Tuple[int, str, int]]: + """Shared memory only: CPID is the creator and never changes, while a + queue's ipcs pids are its last sender and receiver.""" + return {(r.ident, r.key, r.pids[0] if r.pids else 0) + for r in ipc_rows(ipcs_listing("m", uid))} + + +def recorded_shm(results: Path) -> Set[Tuple[int, str, int]]: + """Segments the runs under this results tree recorded creating.""" + out: Set[Tuple[int, str, int]] = set() + if not results.is_dir(): + return out + for path in results.rglob(report.RESULTS): + try: + doc = json.loads(path.read_text()) + except (OSError, ValueError): + continue + for r in doc.get("run", {}).get("shm", []): + out.add((r.get("id"), r.get("key"), r.get("cpid"))) return out @@ -273,7 +298,7 @@ def act(self, line: str, action: Callable[[], None]) -> bool: return True try: action() - except (OSError, BackendError) as e: + except (OSError, BackendError, subprocess.CalledProcessError) as e: self.fail("%s: %s" % (line, e)) self.failed = True return False @@ -290,7 +315,7 @@ def run(self) -> bool: % " ".join(str(p.pid) for p in left)) else: self.sweep_runtime() - self.report_ipc() + self.sweep_ipc() if self.results.is_dir(): self.fix_results_modes() return not self.failed @@ -355,15 +380,23 @@ def sweep_runtime(self) -> None: for path in runtime_paths(self.tmp, self.user_tmp, self.uid): self.act("rm -rf %s" % path, lambda p=path: remove(p)) - def report_ipc(self) -> None: - """Report only. sys_shmget and sys_msgget forward the guest key to the - host unchanged, so an elfuse object is indistinguishable from any other - of the user's, and outliving its creator is ordinary SysV lifecycle.""" + def sweep_ipc(self) -> None: + """Removes only what a run recorded creating. sys_shmget and + sys_msgget forward the guest key to the host unchanged, so any other + object is indistinguishable from a third party's, and outliving its + creator is ordinary SysV lifecycle. Id, key and creator pid would all + have to be recycled together for the match to be wrong.""" rows = [row for kind in ("m", "q", "s") for row in ipc_rows(ipcs_listing(kind, self.uid))] leaked_rows, keep_rows = dead_ipc(rows, alive) + mine = recorded_shm(self.results) for row, why in leaked_rows: - self.out("leaked ipc -%s %d: %s; ipcrm -%s %d by hand" - % (row.kind, row.ident, why, row.kind, row.ident)) + if row.kind == "m" and (row.ident, row.key, row.pids[0] if row.pids else 0) in mine: + self.act("ipcrm -m %d (%s, a run recorded creating it)" % (row.ident, why), + lambda r=row: subprocess.run(["ipcrm", "-m", str(r.ident)], + check=True, capture_output=True)) + else: + self.out("leaked ipc -%s %d: %s; ipcrm -%s %d by hand" + % (row.kind, row.ident, why, row.kind, row.ident)) sems = [row for row, _ in keep_rows if row.kind == "s"] for row, why in keep_rows: if row.kind != "s": From ffa54f4a4424c5a50d087b690efd6c735420dcd9 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Mon, 31 Aug 2026 13:24:07 +0200 Subject: [PATCH 5/5] Document the conformance commands docs/testing.md lists the public commands with their defaults and the Make aliases. docs/conformance.md defines the result and extension contracts and what clean-conformance sweeps, reports, and keeps. --- README.md | 6 ++- docs/conformance.md | 117 ++++++++++++++++++++++++++++++++++++++++++++ docs/testing.md | 40 +++++++++++++++ 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 docs/conformance.md diff --git a/README.md b/README.md index a9d80967..87b9c671 100644 --- a/README.md +++ b/README.md @@ -156,8 +156,10 @@ The build signs `build/elfuse` before use. Override the signing identity with Rosetta, dynamic linking via `--sysroot`, and attaching `gdb` / `lldb` to the built-in stub. - [docs/testing.md](docs/testing.md): build prerequisites, the - `make check` flow, the QEMU and Rosetta cross-check matrices, and - fixture handling. + `make check` flow, the QEMU and Rosetta cross-check matrices, + fixture handling, and conformance commands. +- [docs/conformance.md](docs/conformance.md): the conformance harness, + expectations, payloads, and CI workflow. - [docs/oci-images.md](docs/oci-images.md): the `elfuse-oci` store, pull behavior, and validation. - [docs/filenames.md](docs/filenames.md): how a guest filename becomes a diff --git a/docs/conformance.md b/docs/conformance.md new file mode 100644 index 00000000..b44753c4 --- /dev/null +++ b/docs/conformance.md @@ -0,0 +1,117 @@ +# Conformance Harness + +The harness runs registered Linux test suites on elfuse and a QEMU reference. +It records suite status separately from the expectation verdict. The command +reference is in [testing.md](testing.md#conformance-tests). + +## Results + +`run` writes `results.json` below `///-/`, +where `` defaults to `build/conformance`. That file is the canonical +artifact: `schema_version: 1`, `kind: run`, run metadata, derived counts and +gate, and case records. Loading rejects a gate or count that disagrees with +the cases. An empty run is red. A `summary.txt` beside it carries the lines +`report` prints. + +Each attempt records `normal`, `timeout`, `signal`, or `transport`, elapsed +microseconds, output paths, and an exit code or signal when applicable. Case +statuses are `PASS`, `FAIL`, `SKIP`, `CONF`, `WARN`, `BROK`, `TIMEOUT`, +`CRASH`, `INCONSISTENT`, and `ERROR`. Verdicts are `as_expected`, +`unexpected_failure`, `unexpected_pass`, `flaked`, `filtered`, and `error`. + +JSON list output also has `schema_version: 1` and a `kind` field. Requested +machine data uses stdout. Diagnostics use stderr. + +Exit codes are: + +- `0`: the operation succeeded or the run is green. +- `1`: a run is red, a backend failed, or an artifact check is red. +- `2`: the command, configuration, or operation is invalid. +- `3`: a non-writing pin or selection check found drift. +- `77`: an optional prerequisite is absent. `--require` and `CONF_REQUIRE=1` + promote it to `2`. + +## IDs and Selection + +Case IDs have one of these forms: + +```text +: +:/[/...] +``` + +Selectors and expectation matchers use shell globs across the complete ID. +A bare group selector also selects its cases. An unmatched selector is an +error. + +A selection file assigns each upstream launch group to `pr`, `full`, or a +declined group with a reason. PR groups run in both scopes. Enabled entries +may set `timeout_s` and suite-specific case filters. `selection check` +compares the file against the pinned inventory and reports drift; `selection +update` rewrites the generated selection data. + +## Expectations + +Expectation files are JSONC and accept comments and trailing commas. Each +backend has a leaf, `_.jsonc`, which may `include` shared +files; the optional `flaky.jsonc` holds the quarantine actions. Files contain +ordered actions; the last matching non-quarantine action wins, and the first +effective action must be `expect_pass` on `*`. + +Actions are `expect_pass`, `expect_failure`, `expect_conf`, `skip`, and +`quarantine`. Every non-pass action needs a reason. A quarantined case runs +alone for at most three attempts and reports test mismatches as `flaked`. +Harness errors remain red. A full run rejects matchers that select no case. + +A skipped expectation prevents launch. `--bootstrap` launches skipped cases +and records status without applying expectations. `expectations seed` derives +actions from bootstrap statuses or red verdicts. It refuses harness errors. + +## Payloads and Pins + +Payloads live below `externals/payloads/` and are not committed. A fingerprint +hashes the pin and builder inputs. `manifest.json` records the fingerprint and +each staged file or symlink. Verification detects missing, extra, changed, and +stale content before a run starts. + +Pins are schema-checked JSON. `pins check` fetches the upstream ref without +writing and reports drift. `pins update` validates the new pin before +replacing the file. + +## Suite Interface + +`tests/conformance/providers/__init__.py` is the static suite registry; +`Provider` in `providers/base.py` declares what a suite supplies: selection, +prerequisites, payload and pin hooks, case discovery, batch keys, and result +decoding. + +The shared runner owns expectation resolution, skip handling, unresolved batch +reruns, quarantine retries, result ordering, and judgment. Providers map +suite output to statuses. Backends return process invocations. A provider +translates host paths through `backend.guest_path()` before putting them in +argv; `Backend.run` forwards argv unchanged, because only the provider knows +which elements are paths. QEMU records non-timeout shell statuses as exit +codes. Providers interpret `128+n` through the suite contract because the +shell cannot distinguish it from a plain exit with the same value. + +The elfuse backend starts one `build/elfuse --timeout 0` process for each +command. The QEMU backend starts one VM through `tests/qemu-runner.sh`, shares +the repository read-only at `/mnt/host`, and executes commands over SSH. + +## Make and CI + +The Make targets take their suite list from the registry through +`scripts/conformance suites`. An empty registry makes suite targets print +`SKIP`; harness selftests still run. + +`make clean-conformance` sweeps what an interrupted run left: the QEMU VM and +its state record, detached guest process groups, orphaned fork children, and +elfuse scratch under `/tmp`. Shared memory a run recorded creating is removed; +any other dead-creator SysV object is reported, since the guest key reaches the +host unchanged and nothing tells one apart from a third party's. It refuses +while a session holds a lock, skips the scratch and the SysV pass while any +elfuse of the user is alive, and keeps results. + +`.github/workflows/conformance.yml` runs QEMU before elfuse and gates on the +required `Conformance (make test-conformance)` job. Pull requests use the PR +scope. Schedules and `scope=full` dispatches use the full scope. diff --git a/docs/testing.md b/docs/testing.md index c356ae0a..bbe8e509 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -462,6 +462,8 @@ The repository contains several layers of validation: - shell integration suites such as BusyBox, coreutils, and dynamic-loader tests - debugger integration tests for the GDB stub - native macOS HVF checks such as multi-vCPU and RWX validation +- conformance lanes that judge test suites against elfuse and a QEMU + reference; see `docs/conformance.md` The quick suite is driven by `tests/driver.sh`, which supports: @@ -532,6 +534,44 @@ whose parent closes its copy of the fd before the backing has been drained answers with its primary alone, because the backing half belongs to a stream that has gone. Both rows are load-bearing in pairs -- neither number alone separates the answers the site could give -- so both are printed. +## Conformance Tests + +`scripts/conformance` runs registered suites on elfuse or QEMU. These are its +public commands: + +| Command | Result | +|---------|--------| +| `scripts/conformance suites [--format text\|json]` | List registered suites | +| `scripts/conformance list SUITE [--scope pr\|full] [--backend elfuse\|qemu\|all] [--format text\|json] [--require]` | List canonical case IDs; the default scope is `full` | +| `scripts/conformance run SUITE [--scope pr\|full] [--case ID_OR_GLOB] [--backend elfuse\|qemu\|all] [--jobs N] [--results DIR] [--bootstrap] [--require] [--no-retry] [--dry-run] [-v]` | Run cases; the defaults are the `pr` scope, elfuse, one job, and `build/conformance` | +| `scripts/conformance payload fingerprint SUITE` | Print the payload fingerprint | +| `scripts/conformance payload build SUITE [--force]` | Build the payload | +| `scripts/conformance payload verify SUITE [--fingerprint HASH]` | Verify the payload manifest and files | +| `scripts/conformance selection check SUITE` | Compare selection with the pinned inventory | +| `scripts/conformance selection update SUITE` | Rewrite generated selection | +| `scripts/conformance expectations check [SUITE]` | Validate expectation files | +| `scripts/conformance expectations seed SUITE RESULTS [--reason TEXT] [--write]` | Derive expectation actions from results | +| `scripts/conformance pins check [SUITE] [--ref REF]` | Report pin drift without writing | +| `scripts/conformance pins update SUITE [--ref REF]` | Update a pin | +| `scripts/conformance report RESULTS [--format text\|markdown\|json]` | Read canonical results | +| `scripts/conformance clean [--results DIR] [--dry-run]` | Sweep what an interrupted run left; results survive | +| `scripts/conformance selftest` | Run harness selftests | + +Examples: + +```sh +scripts/conformance run SUITE --scope full --backend all +scripts/conformance run SUITE --case 'SUITE:GROUP/*' --backend qemu +scripts/conformance report RESULTS --format markdown +``` + +The Make aliases are `test-conformance-harness`, `test-conformance`, +`test-conformance-full`, `conformance-payloads`, `clean-payloads`, +`clean-conformance`, and `update-pins`. `BACKEND`, `CONF_SCOPE`, `TEST`, +`CONF_JOBS`, and `CONF_RESULTS` configure the run targets. Run +`clean-conformance` after an interrupted run, before `make clean` deletes the +VM's state record. See [conformance.md](conformance.md) for result, +expectation, payload, and suite interfaces. ## Validation Strategy By Change Type