diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 5fe791a..947b16d 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -28,6 +28,9 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Install checksum-verified Zig 0.13.0 shell: bash @@ -41,37 +44,47 @@ jobs: tar -xf "$tarball" -C "$RUNNER_TEMP" echo "$RUNNER_TEMP/zig-linux-x86_64-0.13.0" >> "$GITHUB_PATH" - - name: Build native shared and static libraries - run: cd ffi/zig && zig build -Doptimize=ReleaseSafe + # Keep this existing job id and name stable for branch protection. + # Both pinned suites are candidate commits on live draft PRs. Repin to + # qualified, published SHAs before merge of this consuming PR. + - name: Checkout Proven Tests and Benches at exact commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/proven-tests-and-benches + ref: 6fb7bcb3ccd2933a13f0207b04e583317b2e9e2f + path: .proven-tests + persist-credentials: false - - name: Run native unit and linked integration tests - run: cd ffi/zig && zig build test + - name: Checkout Proven CI/CD and Workflows at exact commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/proven-cicd-and-workflows + ref: ed50b1eed7b4e11190615715777cab285bb5810c + path: .proven-cicd + persist-credentials: false - - name: Compile public C header consumer + - name: Execute Proven native build, tests, formatting and ABI contract shell: bash + env: + EXPECTED_SUBJECT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PROVEN_CICD_SHA: ed50b1eed7b4e11190615715777cab285bb5810c + PROVEN_TESTS_SHA: 6fb7bcb3ccd2933a13f0207b04e583317b2e9e2f + ZIG_PROJECT_DIR: ffi/zig + ZIG_TEST_ENTRY: src/main.zig + ZIG_VERSION: 0.13.0 + PROVEN_TESTS_ROOT: .proven-tests + C_ABI_HEADER: ffi/zig/include/checky_monkey.h + C_ABI_CONSUMER: tests/abi/consumer.c + C_ABI_ARCHIVE: ffi/zig/zig-out/lib/libchecky_monkey.a + CC: zig cc run: | set -euo pipefail - cat > "$RUNNER_TEMP/checky_abi_smoke.c" <<'EOF' - #include "checky_monkey.h" - _Static_assert(CHECKY_MONKEY_OK == 0, "ABI result drift"); - _Static_assert(CHECKY_MONKEY_UNSUPPORTED == 5, "ABI result drift"); - int main(void) { - checky_monkey_handle *handle = checky_monkey_init(); - if (!handle) return 1; - checky_monkey_result result = checky_monkey_process(handle, 1); - checky_monkey_free(handle); - return result == CHECKY_MONKEY_UNSUPPORTED ? 0 : 2; - } - EOF - zig cc -std=c11 -Wall -Wextra -Werror \ - -Iffi/zig/include "$RUNNER_TEMP/checky_abi_smoke.c" \ - ffi/zig/zig-out/lib/libchecky_monkey.a -o "$RUNNER_TEMP/checky_abi_smoke" - "$RUNNER_TEMP/checky_abi_smoke" + test "$(git rev-parse HEAD)" = "$EXPECTED_SUBJECT_SHA" + test "$(git -C .proven-cicd rev-parse HEAD)" = "$PROVEN_CICD_SHA" + bash .proven-cicd/scripts/zig-native-ffi-gate.sh - - name: Check Zig format and shell regression fixtures - run: | - zig fmt --check ffi/zig/build.zig ffi/zig/src/main.zig ffi/zig/test/integration_test.zig - bash tests/empty-lint/test-runner.sh + - name: Test invisible-character regressions + run: bash tests/empty-lint/test-runner.sh # --------------------------------------------------------------------------- # Job 1: DEED manifest validation @@ -392,3 +405,35 @@ jobs: *Dogfooding is guinea pig fooding — we test our tools on ourselves.* EOF + # The scorecard above reports the entire gate; it must never be a green + # wrapper around a failed/skipped/cancelled mandatory job. This step runs + # AFTER writing the summary, preserving diagnostics when it rejects. + - name: Checkout Proven CI final-verdict source at exact commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/proven-cicd-and-workflows + ref: ed50b1eed7b4e11190615715777cab285bb5810c + path: .proven-cicd + persist-credentials: false + + - name: Reject non-successful mandatory predecessor jobs + shell: bash + env: + PROVEN_CICD_SHA: ed50b1eed7b4e11190615715777cab285bb5810c + REQUIRED_JOB_IDS: | + native-ffi + deed-validate + k9-validate + empty-lint + groove-check + REQUIRED_RESULTS: | + native-ffi=${{ needs.native-ffi.result }} + deed-validate=${{ needs.deed-validate.result }} + k9-validate=${{ needs.k9-validate.result }} + empty-lint=${{ needs.empty-lint.result }} + groove-check=${{ needs.groove-check.result }} + run: | + set -euo pipefail + test "$(git -C .proven-cicd rev-parse HEAD)" = "$PROVEN_CICD_SHA" + bash .proven-cicd/scripts/required-job-verdict.sh + diff --git a/TEST-NEEDS.md b/TEST-NEEDS.md index a358598..188ef05 100644 --- a/TEST-NEEDS.md +++ b/TEST-NEEDS.md @@ -1,29 +1,72 @@ -# TEST-NEEDS.md — checky-monkey +# TEST-NEEDS — Checky Monkey: measured coverage and assurance debt -## CRG Grade: C — ACHIEVED 2026-04-04 + -## Current Test State +**Status: incomplete; no CRG C or proof certification claimed by this document.** The previous "CRG C ACHIEVED" assertion was not backed by executable product coverage. Count jobs that ran on an exact commit, not workflow files, as execution evidence. -| Category | Count | Notes | -|----------|-------|-------| -| Test directories | 1 | Location(s): /tests | -| CI workflows | 18 | Running tests on GitHub Actions | -| Tests | Present | Configured in CI workflows | +Canonical taxonomy: [metadatastician/standards 3-practice/testing-and-benchmarking/TESTING-TAXONOMY.adoc](https://github.com/metadatastician/standards/blob/main/3-practice/testing-and-benchmarking/TESTING-TAXONOMY.adoc), now **18 categories × 14 aspects**. Draw reusable tests from [proven-tests-and-benches](https://github.com/hyperpolymath/proven-tests-and-benches) FIRST. Workflow capabilities belong to [proven-cicd-and-workflows](https://github.com/hyperpolymath/proven-cicd-and-workflows). Do not promote advisory/placeholder actions into mandatory assurance. -## What's Covered +## Executable current scope -- [x] Tests present and running -- [x] CI integration active +The only exercised native subject is the `ffi/zig/` library, with Zig 0.13.0 unit and C ABI tests in `dogfood-gate.yml` job `native-ffi`. The additional `tests/abi/consumer.c` uses the reusable three-outcome C ABI contract harness from `proven-tests-and-benches` at an explicitly checked full source SHA. That harness is **unproven**; it runs caller-supplied assertions, not mathematical proofs. The existing shell regression suite tests invisible-character detection. There is still no userscript parser, static analyzer, registry, CLI, browser plugin, or complete Idris2 cross-language compilation gate. -## Still Missing (for CRG B+) +**Status terms:** REAL = specifically exercised with a clean and planted defective case; THIN = a real assertion but incomplete category scope; ABSENT = no subject assertion in current CI; N/A = genuinely not yet applicable with a reason. The matrix is an assessment of intended coverage, NOT a CI pass claim; only a run on the latest commit qualifies execution. -- [ ] Code coverage reports (codecov integration) -- [ ] Detailed test documentation in CONTRIBUTING.md -- [ ] Integration tests beyond unit tests -- [ ] Performance benchmarking suite +| # | Canonical test category | Assessment | Current evidence / missing obligation | +|---|---|---|---| +| 1 | Unit | THIN | Zig source tests cover some lifecycle and status paths; public APIs and all error branches not exhaustive | +| 2 | Point-to-point (P2P) | THIN | Linked C ABI consumer; Idris2→Zig seam not compiled | +| 3 | End-to-end | ABSENT | No user-facing userscript intake→report pipeline | +| 4 | Build | THIN | Pinned Zig 0.13.0 Linux build; other supported targets and reproducibility untested | +| 5 | Execution & runtime | THIN | The library executes; no product runtime exists | +| 6 | Reflexive | THIN | Refusal of unsupported operations and no-tool diagnostics; no full doctor or run-claim verifier | +| 7 | Lifecycle | THIN | Init/free, two live handles, borrowed/owned string; allocation failure, stale pointer, leak stress absent | +| 8 | Smoke | THIN | C consumer runs and calls symbols; no CLI/user-facing artifact | +| 9 | Property-based / generative | ABSENT | No 1,000-case generators or shrinkers | +| 10 | Mutation | ABSENT | Proven Tests self-tests plant mutations of the generic harness; Checky Monkey itself does not run a source mutation campaign or compute a score | +| 11 | Fuzz | ABSENT | No executable fuzz harness or corpus; the former scorecard-only placeholder has been removed | +| 12 | Contract / invariant | THIN | Public C status contract tested; Idris2/C/Zig layout not proven | +| 13 | Regression | THIN | Specific BOM, false-pass, error-ownership regressions covered; defects elsewhere still missing cases | +| 14 | Chaos / resilience | ABSENT | Allocation-failure injection and fault handling required before claiming resilience | +| 15 | Compatibility | ABSENT | First C ABI version, no versioned upgrade/consumer compatibility corpus | +| 16 | Proof regression | ABSENT | Idris2 module names, templates and ABI declarations remain inconsistent; no checked proofs | +| 17 | Type-safe | ABSENT | C static assertions are not Idris2 compiler negative-witness proofs | +| 18 | Coupling / drift-freedom | THIN | Public C header↔linked Zig code is exercised; Idris2 result enum and layout drift remain | -## Run Tests +## Cross-cutting aspects (14) -```bash -(check Makefile/justfile/package.json for test command) -``` +| Aspect | Present evidence and debt | +|---|---| +| Dependability | Handle lifetime tests; no long-run fault-injection | +| Security | Explicit unsupported paths; no script threat model exercised | +| Usability | N/A until UI or CLI is implemented | +| Interoperability | Executed native C ABI seam, not yet Idris2 | +| Safety | Reject null/unsupported, but no memory-safety proof | +| Performance | No calibrated baseline or regression threshold | +| Functionality | Native ABI shell only; userscript functionality absent | +| Versability | C ABI v0.1 string check; no version compatibility tests | +| Accessibility | N/A until browser or CLI artifact ships | +| Maintainability | Shared harness contract; incomplete build/docs | +| Privacy | No implemented data pipeline to inspect; policies not tests | +| Observability | Last-error/status checks, no complete evidence receipts | +| Reproducibility | Pinned native Zig release and Proven source revision; not multi-platform | +| Portability | Linux CI observed; no Windows/macOS matrix | + +## Next gates before claiming comprehensive assurance + +1. **Preserve stable emitted context:** `Native Zig library build and C-ABI tests`. Required contexts must appear on pull requests to all bases, with a nonzero job count, a pinned subject SHA and a final fail-closed verdict. Rulesets are managed separately; do not require a context until its real canary passes. +2. **Qualify the Proven dependency:** independently test its silence/firing/void harness, publish a reviewed immutable commit, repin this consumer, and keep both 1 and 2 exit outcomes blocking. +3. **Repair Idris2 ABI**, then run genuine compiler checks and negative witnesses. Compiler-not-installed means **not run**, never a green proof. +4. **Introduce the userscript `userscript-submission/0.1` contract** from issue #101 and plant malformed/privilege mismatch/provenance false-claim fixtures. Only then implement parser→analysis→record/report E2E tests. +5. Add reproducible property generation with shrinking, coverage and source mutation, actual fuzz corpus once an untrusted parser exists, allocation failure/chaos, benchmarks and documented portability/compatibility matrices. +6. Restore meaningful CI startup checks (governance, Hypatia, secret scanning, SonarCloud). A green Dogfood Gate does **not** compensate for any failed required check or absent workflow. + +## Governance, hooks and schema compatibility + +The installed hook surface on `main` is only four executable files under `hooks/` (SHA pins, permissions, CodeQL and workflow SPDX). The current standards source has **19 executable `.githooks/` files**, including `pre-commit`, `pre-push`, commit-message checking, staged secret scanning, format and K9 validators, an actions.lock validator and installer/uninstaller. Merely copying these is not an assurance result: executable bit, local `core.hooksPath` enablement and behavior under planted defects must be verified. **Hook parity is an open migration item**; workflow CI, not local hooks, is the non-optional merge control. + +The established `proven-tests-and-benches` evidence-report schema version 2 remains a **17 × 14 = 238-cell** reporting contract, despite the canonical taxonomy now having 18 categories. Category 18 is assessed explicitly above, but is not counted as a v2 certified report cell. A versioned schema migration with compatibility tests is required before 18 × 14 reports can be issued. + +## Explicit limitations + +CRG grade and test completeness are **not established**. No % coverage metric exists for the entire product, nor can the current native unit tests establish security of userscript processing that has not been built. A cancelled, skipped, empty, setup-only or startup-failed job cannot become a passing test through a summary job. Work on product internals belongs to a scoped PR and must pass latest-head CI and reviewer threads before closure. diff --git a/tests/abi/consumer.c b/tests/abi/consumer.c new file mode 100644 index 0000000..50a73f4 --- /dev/null +++ b/tests/abi/consumer.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: MPL-2.0 */ +/* Non-vacuous, linked C consumer for the Zig library's public contract. + * Source-of-truth harness: proven-tests-and-benches/integrations/contract/c-abi. + * This does not claim userscript analysis, Idris2 proofs, or concurrency safety. + */ +#include "checky_monkey.h" +#include +#include +#include + +_Static_assert(CHECKY_MONKEY_OK == 0, "ABI result OK drift"); +_Static_assert(CHECKY_MONKEY_ERROR == 1, "ABI result ERROR drift"); +_Static_assert(CHECKY_MONKEY_INVALID_PARAM == 2, "ABI result INVALID_PARAM drift"); +_Static_assert(CHECKY_MONKEY_OUT_OF_MEMORY == 3, "ABI result OOM drift"); +_Static_assert(CHECKY_MONKEY_NULL_POINTER == 4, "ABI result NULL drift"); +_Static_assert(CHECKY_MONKEY_UNSUPPORTED == 5, "ABI result UNSUPPORTED drift"); + +#define CHECK(cond, detail) do { \ + if (!(cond)) { fprintf(stderr, "contract failed: %s\n", detail); return 1; } \ +} while (0) + +static uint32_t callback(uint64_t id, uint32_t value) { + return (uint32_t)id + value; +} + +int main(void) { + /* Null handles and absent pointers must be rejected before use. */ + CHECK(checky_monkey_is_initialized(NULL) == 0, "null handle appears initialized"); + CHECK(checky_monkey_process(NULL, 3) == CHECKY_MONKEY_NULL_POINTER, + "null process handle did not fail closed"); + const char *error = checky_monkey_last_error(); + CHECK(error && strcmp(error, "Null handle") == 0, "borrowed last_error unavailable"); + CHECK(checky_monkey_get_string(NULL) == NULL, "null handle produced a string"); + checky_monkey_free(NULL); + checky_monkey_free_string(NULL); + + checky_monkey_handle *a = checky_monkey_init(); + CHECK(a != NULL, "initialization failed"); + checky_monkey_handle *b = checky_monkey_init(); + CHECK(b != NULL && a != b, "independent live handles alias"); + CHECK(checky_monkey_last_error() == NULL, "init did not clear prior error"); + CHECK(checky_monkey_is_initialized(a) == 1, "live handle not initialized"); + CHECK(checky_monkey_is_initialized(b) == 1, "second handle not initialized"); + + /* No real product pipeline exists; every such operation MUST refuse. */ + CHECK(checky_monkey_process(a, 42) == CHECKY_MONKEY_UNSUPPORTED, + "unimplemented process produced an apparent pass"); + CHECK(checky_monkey_last_error() != NULL, "unsupported process lost reason"); + CHECK(checky_monkey_process_array(a, NULL, 0) == CHECKY_MONKEY_NULL_POINTER, + "null array buffer accepted"); + const uint8_t bytes[] = { 0, 1, 255 }; + CHECK(checky_monkey_process_array(a, bytes, 3) == CHECKY_MONKEY_UNSUPPORTED, + "unimplemented array process produced an apparent pass"); + CHECK(checky_monkey_register_callback(a, NULL) == CHECKY_MONKEY_NULL_POINTER, + "null callback accepted"); + CHECK(checky_monkey_register_callback(a, callback) == CHECKY_MONKEY_UNSUPPORTED, + "unimplemented callback registration produced an apparent pass"); + + /* Owned strings have a deallocator; errors and version are borrowed. */ + const char *owned = checky_monkey_get_string(a); + CHECK(owned != NULL && strcmp(owned, "Example result") == 0, + "string result contract changed without update"); + CHECK(checky_monkey_last_error() == NULL, "successful getter did not clear error"); + checky_monkey_free_string(owned); + CHECK(strcmp(checky_monkey_version(), "0.1.0") == 0, "version contract drift"); + CHECK(strstr(checky_monkey_build_info(), "Zig") != NULL, "build metadata missing"); + + checky_monkey_free(a); + CHECK(checky_monkey_is_initialized(b) == 1, "freeing a corrupted other live handle"); + checky_monkey_free(b); + return 0; +} diff --git a/tests/fuzz/placeholder.txt b/tests/fuzz/placeholder.txt deleted file mode 100644 index 8621280..0000000 --- a/tests/fuzz/placeholder.txt +++ /dev/null @@ -1 +0,0 @@ -Scorecard requirement placeholder