Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 70 additions & 25 deletions .github/workflows/dogfood-gate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false

- name: Install checksum-verified Zig 0.13.0
shell: bash
Expand All @@ -41,37 +44,47 @@ jobs:
tar -xf "$tarball" -C "$RUNNER_TEMP"
echo "$RUNNER_TEMP/zig-linux-x86_64-0.13.0" >> "$GITHUB_PATH"

- name: Build native shared and static libraries
run: cd ffi/zig && zig build -Doptimize=ReleaseSafe
# Keep this existing job id and name stable for branch protection.
# Both pinned suites are candidate commits on live draft PRs. Repin to
# qualified, published SHAs before merge of this consuming PR.
- name: Checkout Proven Tests and Benches at exact commit
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: hyperpolymath/proven-tests-and-benches
ref: 6fb7bcb3ccd2933a13f0207b04e583317b2e9e2f
path: .proven-tests
persist-credentials: false

- name: Run native unit and linked integration tests
run: cd ffi/zig && zig build test
- name: Checkout Proven CI/CD and Workflows at exact commit
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: hyperpolymath/proven-cicd-and-workflows
ref: ed50b1eed7b4e11190615715777cab285bb5810c
path: .proven-cicd
persist-credentials: false

- name: Compile public C header consumer
- name: Execute Proven native build, tests, formatting and ABI contract
shell: bash
env:
EXPECTED_SUBJECT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
PROVEN_CICD_SHA: ed50b1eed7b4e11190615715777cab285bb5810c
PROVEN_TESTS_SHA: 6fb7bcb3ccd2933a13f0207b04e583317b2e9e2f
ZIG_PROJECT_DIR: ffi/zig
ZIG_TEST_ENTRY: src/main.zig
ZIG_VERSION: 0.13.0
PROVEN_TESTS_ROOT: .proven-tests
C_ABI_HEADER: ffi/zig/include/checky_monkey.h
C_ABI_CONSUMER: tests/abi/consumer.c
C_ABI_ARCHIVE: ffi/zig/zig-out/lib/libchecky_monkey.a
CC: zig cc
run: |
set -euo pipefail
cat > "$RUNNER_TEMP/checky_abi_smoke.c" <<'EOF'
#include "checky_monkey.h"
_Static_assert(CHECKY_MONKEY_OK == 0, "ABI result drift");
_Static_assert(CHECKY_MONKEY_UNSUPPORTED == 5, "ABI result drift");
int main(void) {
checky_monkey_handle *handle = checky_monkey_init();
if (!handle) return 1;
checky_monkey_result result = checky_monkey_process(handle, 1);
checky_monkey_free(handle);
return result == CHECKY_MONKEY_UNSUPPORTED ? 0 : 2;
}
EOF
zig cc -std=c11 -Wall -Wextra -Werror \
-Iffi/zig/include "$RUNNER_TEMP/checky_abi_smoke.c" \
ffi/zig/zig-out/lib/libchecky_monkey.a -o "$RUNNER_TEMP/checky_abi_smoke"
"$RUNNER_TEMP/checky_abi_smoke"
test "$(git rev-parse HEAD)" = "$EXPECTED_SUBJECT_SHA"
test "$(git -C .proven-cicd rev-parse HEAD)" = "$PROVEN_CICD_SHA"
bash .proven-cicd/scripts/zig-native-ffi-gate.sh

- name: Check Zig format and shell regression fixtures
run: |
zig fmt --check ffi/zig/build.zig ffi/zig/src/main.zig ffi/zig/test/integration_test.zig
bash tests/empty-lint/test-runner.sh
- name: Test invisible-character regressions
run: bash tests/empty-lint/test-runner.sh

# ---------------------------------------------------------------------------
# Job 1: DEED manifest validation
Expand Down Expand Up @@ -392,3 +405,35 @@ jobs:
*Dogfooding is guinea pig fooding — we test our tools on ourselves.*
EOF

# The scorecard above reports the entire gate; it must never be a green
# wrapper around a failed/skipped/cancelled mandatory job. This step runs
# AFTER writing the summary, preserving diagnostics when it rejects.
- name: Checkout Proven CI final-verdict source at exact commit
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: hyperpolymath/proven-cicd-and-workflows
ref: ed50b1eed7b4e11190615715777cab285bb5810c
path: .proven-cicd
persist-credentials: false

- name: Reject non-successful mandatory predecessor jobs
shell: bash
env:
PROVEN_CICD_SHA: ed50b1eed7b4e11190615715777cab285bb5810c
REQUIRED_JOB_IDS: |
native-ffi
deed-validate
k9-validate
empty-lint
groove-check
REQUIRED_RESULTS: |
native-ffi=${{ needs.native-ffi.result }}
deed-validate=${{ needs.deed-validate.result }}
k9-validate=${{ needs.k9-validate.result }}
empty-lint=${{ needs.empty-lint.result }}
groove-check=${{ needs.groove-check.result }}
run: |
set -euo pipefail
test "$(git -C .proven-cicd rev-parse HEAD)" = "$PROVEN_CICD_SHA"
bash .proven-cicd/scripts/required-job-verdict.sh

83 changes: 63 additions & 20 deletions TEST-NEEDS.md
Original file line number Diff line number Diff line change
@@ -1,29 +1,72 @@
# TEST-NEEDS.md — checky-monkey
# TEST-NEEDS — Checky Monkey: measured coverage and assurance debt

## CRG Grade: C — ACHIEVED 2026-04-04
<!-- SPDX-License-Identifier: CC-BY-SA-4.0 -->

## Current Test State
**Status: incomplete; no CRG C or proof certification claimed by this document.** The previous "CRG C ACHIEVED" assertion was not backed by executable product coverage. Count jobs that ran on an exact commit, not workflow files, as execution evidence.

| Category | Count | Notes |
|----------|-------|-------|
| Test directories | 1 | Location(s): /tests |
| CI workflows | 18 | Running tests on GitHub Actions |
| Tests | Present | Configured in CI workflows |
Canonical taxonomy: [metadatastician/standards 3-practice/testing-and-benchmarking/TESTING-TAXONOMY.adoc](https://github.com/metadatastician/standards/blob/main/3-practice/testing-and-benchmarking/TESTING-TAXONOMY.adoc), now **18 categories × 14 aspects**. Draw reusable tests from [proven-tests-and-benches](https://github.com/hyperpolymath/proven-tests-and-benches) FIRST. Workflow capabilities belong to [proven-cicd-and-workflows](https://github.com/hyperpolymath/proven-cicd-and-workflows). Do not promote advisory/placeholder actions into mandatory assurance.

## What's Covered
## Executable current scope

- [x] Tests present and running
- [x] CI integration active
The only exercised native subject is the `ffi/zig/` library, with Zig 0.13.0 unit and C ABI tests in `dogfood-gate.yml` job `native-ffi`. The additional `tests/abi/consumer.c` uses the reusable three-outcome C ABI contract harness from `proven-tests-and-benches` at an explicitly checked full source SHA. That harness is **unproven**; it runs caller-supplied assertions, not mathematical proofs. The existing shell regression suite tests invisible-character detection. There is still no userscript parser, static analyzer, registry, CLI, browser plugin, or complete Idris2 cross-language compilation gate.

## Still Missing (for CRG B+)
**Status terms:** REAL = specifically exercised with a clean and planted defective case; THIN = a real assertion but incomplete category scope; ABSENT = no subject assertion in current CI; N/A = genuinely not yet applicable with a reason. The matrix is an assessment of intended coverage, NOT a CI pass claim; only a run on the latest commit qualifies execution.

- [ ] Code coverage reports (codecov integration)
- [ ] Detailed test documentation in CONTRIBUTING.md
- [ ] Integration tests beyond unit tests
- [ ] Performance benchmarking suite
| # | Canonical test category | Assessment | Current evidence / missing obligation |
|---|---|---|---|
| 1 | Unit | THIN | Zig source tests cover some lifecycle and status paths; public APIs and all error branches not exhaustive |
| 2 | Point-to-point (P2P) | THIN | Linked C ABI consumer; Idris2→Zig seam not compiled |
| 3 | End-to-end | ABSENT | No user-facing userscript intake→report pipeline |
| 4 | Build | THIN | Pinned Zig 0.13.0 Linux build; other supported targets and reproducibility untested |
| 5 | Execution & runtime | THIN | The library executes; no product runtime exists |
| 6 | Reflexive | THIN | Refusal of unsupported operations and no-tool diagnostics; no full doctor or run-claim verifier |
| 7 | Lifecycle | THIN | Init/free, two live handles, borrowed/owned string; allocation failure, stale pointer, leak stress absent |
| 8 | Smoke | THIN | C consumer runs and calls symbols; no CLI/user-facing artifact |
| 9 | Property-based / generative | ABSENT | No 1,000-case generators or shrinkers |
| 10 | Mutation | ABSENT | Proven Tests self-tests plant mutations of the generic harness; Checky Monkey itself does not run a source mutation campaign or compute a score |
| 11 | Fuzz | ABSENT | No executable fuzz harness or corpus; the former scorecard-only placeholder has been removed |
| 12 | Contract / invariant | THIN | Public C status contract tested; Idris2/C/Zig layout not proven |
| 13 | Regression | THIN | Specific BOM, false-pass, error-ownership regressions covered; defects elsewhere still missing cases |
| 14 | Chaos / resilience | ABSENT | Allocation-failure injection and fault handling required before claiming resilience |
| 15 | Compatibility | ABSENT | First C ABI version, no versioned upgrade/consumer compatibility corpus |
| 16 | Proof regression | ABSENT | Idris2 module names, templates and ABI declarations remain inconsistent; no checked proofs |
| 17 | Type-safe | ABSENT | C static assertions are not Idris2 compiler negative-witness proofs |
| 18 | Coupling / drift-freedom | THIN | Public C header↔linked Zig code is exercised; Idris2 result enum and layout drift remain |

## Run Tests
## Cross-cutting aspects (14)

```bash
(check Makefile/justfile/package.json for test command)
```
| Aspect | Present evidence and debt |
|---|---|
| Dependability | Handle lifetime tests; no long-run fault-injection |
| Security | Explicit unsupported paths; no script threat model exercised |
| Usability | N/A until UI or CLI is implemented |
| Interoperability | Executed native C ABI seam, not yet Idris2 |
| Safety | Reject null/unsupported, but no memory-safety proof |
| Performance | No calibrated baseline or regression threshold |
| Functionality | Native ABI shell only; userscript functionality absent |
| Versability | C ABI v0.1 string check; no version compatibility tests |
| Accessibility | N/A until browser or CLI artifact ships |
| Maintainability | Shared harness contract; incomplete build/docs |
| Privacy | No implemented data pipeline to inspect; policies not tests |
| Observability | Last-error/status checks, no complete evidence receipts |
| Reproducibility | Pinned native Zig release and Proven source revision; not multi-platform |
| Portability | Linux CI observed; no Windows/macOS matrix |

## Next gates before claiming comprehensive assurance

1. **Preserve stable emitted context:** `Native Zig library build and C-ABI tests`. Required contexts must appear on pull requests to all bases, with a nonzero job count, a pinned subject SHA and a final fail-closed verdict. Rulesets are managed separately; do not require a context until its real canary passes.
2. **Qualify the Proven dependency:** independently test its silence/firing/void harness, publish a reviewed immutable commit, repin this consumer, and keep both 1 and 2 exit outcomes blocking.
3. **Repair Idris2 ABI**, then run genuine compiler checks and negative witnesses. Compiler-not-installed means **not run**, never a green proof.
4. **Introduce the userscript `userscript-submission/0.1` contract** from issue #101 and plant malformed/privilege mismatch/provenance false-claim fixtures. Only then implement parser→analysis→record/report E2E tests.
5. Add reproducible property generation with shrinking, coverage and source mutation, actual fuzz corpus once an untrusted parser exists, allocation failure/chaos, benchmarks and documented portability/compatibility matrices.
6. Restore meaningful CI startup checks (governance, Hypatia, secret scanning, SonarCloud). A green Dogfood Gate does **not** compensate for any failed required check or absent workflow.

## Governance, hooks and schema compatibility

The installed hook surface on `main` is only four executable files under `hooks/` (SHA pins, permissions, CodeQL and workflow SPDX). The current standards source has **19 executable `.githooks/` files**, including `pre-commit`, `pre-push`, commit-message checking, staged secret scanning, format and K9 validators, an actions.lock validator and installer/uninstaller. Merely copying these is not an assurance result: executable bit, local `core.hooksPath` enablement and behavior under planted defects must be verified. **Hook parity is an open migration item**; workflow CI, not local hooks, is the non-optional merge control.

The established `proven-tests-and-benches` evidence-report schema version 2 remains a **17 × 14 = 238-cell** reporting contract, despite the canonical taxonomy now having 18 categories. Category 18 is assessed explicitly above, but is not counted as a v2 certified report cell. A versioned schema migration with compatibility tests is required before 18 × 14 reports can be issued.

## Explicit limitations

CRG grade and test completeness are **not established**. No % coverage metric exists for the entire product, nor can the current native unit tests establish security of userscript processing that has not been built. A cancelled, skipped, empty, setup-only or startup-failed job cannot become a passing test through a summary job. Work on product internals belongs to a scoped PR and must pass latest-head CI and reviewer threads before closure.
72 changes: 72 additions & 0 deletions tests/abi/consumer.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/* SPDX-License-Identifier: MPL-2.0 */
/* Non-vacuous, linked C consumer for the Zig library's public contract.
* Source-of-truth harness: proven-tests-and-benches/integrations/contract/c-abi.
* This does not claim userscript analysis, Idris2 proofs, or concurrency safety.
*/
#include "checky_monkey.h"
#include <stdint.h>
#include <stdio.h>
#include <string.h>

_Static_assert(CHECKY_MONKEY_OK == 0, "ABI result OK drift");
_Static_assert(CHECKY_MONKEY_ERROR == 1, "ABI result ERROR drift");
_Static_assert(CHECKY_MONKEY_INVALID_PARAM == 2, "ABI result INVALID_PARAM drift");
_Static_assert(CHECKY_MONKEY_OUT_OF_MEMORY == 3, "ABI result OOM drift");
_Static_assert(CHECKY_MONKEY_NULL_POINTER == 4, "ABI result NULL drift");
_Static_assert(CHECKY_MONKEY_UNSUPPORTED == 5, "ABI result UNSUPPORTED drift");

#define CHECK(cond, detail) do { \
if (!(cond)) { fprintf(stderr, "contract failed: %s\n", detail); return 1; } \
} while (0)

static uint32_t callback(uint64_t id, uint32_t value) {
return (uint32_t)id + value;
}

int main(void) {
/* Null handles and absent pointers must be rejected before use. */
CHECK(checky_monkey_is_initialized(NULL) == 0, "null handle appears initialized");
CHECK(checky_monkey_process(NULL, 3) == CHECKY_MONKEY_NULL_POINTER,
"null process handle did not fail closed");
const char *error = checky_monkey_last_error();
CHECK(error && strcmp(error, "Null handle") == 0, "borrowed last_error unavailable");
CHECK(checky_monkey_get_string(NULL) == NULL, "null handle produced a string");
checky_monkey_free(NULL);
checky_monkey_free_string(NULL);

checky_monkey_handle *a = checky_monkey_init();
CHECK(a != NULL, "initialization failed");
checky_monkey_handle *b = checky_monkey_init();
CHECK(b != NULL && a != b, "independent live handles alias");
CHECK(checky_monkey_last_error() == NULL, "init did not clear prior error");
CHECK(checky_monkey_is_initialized(a) == 1, "live handle not initialized");
CHECK(checky_monkey_is_initialized(b) == 1, "second handle not initialized");

/* No real product pipeline exists; every such operation MUST refuse. */
CHECK(checky_monkey_process(a, 42) == CHECKY_MONKEY_UNSUPPORTED,
"unimplemented process produced an apparent pass");
CHECK(checky_monkey_last_error() != NULL, "unsupported process lost reason");
CHECK(checky_monkey_process_array(a, NULL, 0) == CHECKY_MONKEY_NULL_POINTER,
"null array buffer accepted");
const uint8_t bytes[] = { 0, 1, 255 };
CHECK(checky_monkey_process_array(a, bytes, 3) == CHECKY_MONKEY_UNSUPPORTED,
"unimplemented array process produced an apparent pass");
CHECK(checky_monkey_register_callback(a, NULL) == CHECKY_MONKEY_NULL_POINTER,
"null callback accepted");
CHECK(checky_monkey_register_callback(a, callback) == CHECKY_MONKEY_UNSUPPORTED,
"unimplemented callback registration produced an apparent pass");

/* Owned strings have a deallocator; errors and version are borrowed. */
const char *owned = checky_monkey_get_string(a);
CHECK(owned != NULL && strcmp(owned, "Example result") == 0,
"string result contract changed without update");
CHECK(checky_monkey_last_error() == NULL, "successful getter did not clear error");
checky_monkey_free_string(owned);
CHECK(strcmp(checky_monkey_version(), "0.1.0") == 0, "version contract drift");
CHECK(strstr(checky_monkey_build_info(), "Zig") != NULL, "build metadata missing");

checky_monkey_free(a);
CHECK(checky_monkey_is_initialized(b) == 1, "freeing a corrupted other live handle");
checky_monkey_free(b);
return 0;
}
1 change: 0 additions & 1 deletion tests/fuzz/placeholder.txt

This file was deleted.

Loading