diff --git a/.github/workflows/compiler-execution.yml b/.github/workflows/compiler-execution.yml new file mode 100644 index 0000000..8cb20c2 --- /dev/null +++ b/.github/workflows/compiler-execution.yml @@ -0,0 +1,76 @@ +name: Compiler execution acceptance +on: + push: + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: compiler-execution-${{ github.ref }} + cancel-in-progress: true +jobs: + realistic-compilers: + name: Realistic compilers (${{ matrix.runtime }}) + runs-on: ubuntu-24.04 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + runtime: [release-0.4.2, current] + steps: + - name: Check out reviewed validation scenarios + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + path: validation + persist-credentials: false + - name: Check out independently identified runtime + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + path: runtime + ref: ${{ matrix.runtime == 'release-0.4.2' && 'eb9b1cd0c3e3fd1e70e397edeffb059073464f74' || github.sha }} + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: '3.12' + - name: Install only controller validation dependencies + run: python -m pip install -e './runtime[dev]' + - name: Identify exact runtime source + shell: bash + run: | + set -euo pipefail + printf 'CFS_RUNTIME_SOURCE=%s\n' "$(git -C runtime rev-parse HEAD)" >> "$GITHUB_ENV" + - name: Build worker on disposable hosted runner + run: docker build --label org.opencontainers.image.revision="$CFS_RUNTIME_SOURCE" -t code-flow-worker:0.4.2 runtime/runner + - name: Run realistic programs and deliberate failure controls + working-directory: runtime + env: + PYTHONPATH: ${{ github.workspace }}/runtime:${{ github.workspace }}/validation + CFS_VALIDATION_LANE: ${{ matrix.runtime }} + run: >- + python ../validation/scripts/compiler_acceptance.py + --runtime-root . + --runtime-source "$CFS_RUNTIME_SOURCE" + --lane "$CFS_VALIDATION_LANE" + --output ../validation/artifacts/compiler-validation + - name: Test the actual published frozen 0.4.2 service + if: matrix.runtime == 'release-0.4.2' + working-directory: validation + run: python -m scripts.released_service_probe --output artifacts/released-service + - name: Retain a compact result table in the run summary + if: always() + shell: bash + run: | + if test -f validation/artifacts/compiler-validation/summary.md; then + cat validation/artifacts/compiler-validation/summary.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Save small synthetic reports, including failures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: compiler-validation-${{ matrix.runtime }} + path: | + validation/artifacts/compiler-validation/results.json + validation/artifacts/compiler-validation/summary.md + validation/artifacts/released-service/*.json + if-no-files-found: error + retention-days: 3 diff --git a/README.md b/README.md index 1676a5a..95536c4 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,10 @@ Download OS-specific installers/portable packages from [Releases](https://github 4. Review findings and add a candidate revision. In **Evidence**, filter by file/category and select a diagnostic or missing-coverage line. The inspector verifies the retained source fingerprint before displaying it. Browse nearby lines or open a matching current draft; changed drafts are never overwritten. Baseline expectations and existing tests cannot be removed. Compare results under the same environment and adjustable runtime/memory/readability/footprint preferences. 5. Download escaped HTML, machine-readable JSON and a source/test/harness/report ZIP. Reopen saved revisions and replay them. +## Hosted compiler acceptance + +The [realistic compiler suite](docs/compiler-execution.md) covers all twelve executable adapters with 72 independently authored input/output cases, deliberate compiler/runtime/wrong-answer controls, exact unused/uncovered anchors, and encrypted private-report replay. Its release lane identifies the published 0.4.2 source and tests the actual published Linux frozen service; a second lane tests the current checkout. Read the [dated observations and limits](docs/reports/hosted-compilers-2026-11-10.md) before interpreting a green run. This hosted validation does not install, repair or reset Docker on an end user's PC. + ## Features and limits **Located diagnostics:** static review and execution evidence list original filenames and local line locations, with snapshot hashes. Python checks include auxiliary-file syntax, possibly unused imports/locals/functions and directly unreachable statements. Every other supported execution language now has native unused-code candidates and actual language-specific coverage. UI/HTML show missed source lines, genuine branch arcs or native branch IDs/counts, and explicit partial/unknown states. SQL measures completed statement starts, Bash command-entry lines and assembly instruction-backed debug lines—not universal statement/branch coverage. Unsupported locations and wrong-answer failures without source references stay unlocated. See the [language-by-language tools and limits](docs/polyglot-locations.md), [file/line contract](docs/located-diagnostics.md) and [validation](docs/validation-polyglot.md). No tool guarantees discovery or root-cause localization of every bug. diff --git a/docs/code-index.md b/docs/code-index.md index b0fb7b6..e0600ac 100644 --- a/docs/code-index.md +++ b/docs/code-index.md @@ -75,6 +75,25 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `suffix` | 101 | | variable | `data` | 102 | +## examples/practical_projects.py + +| Kind | Declaration | Line | +|---|---|---| +| function | `_source(text)` | 25 | +| parameter | `text` | 25 | +| variable | `PROGRAMS` | 30 | +| variable | `CASES` | 321 | +| variable | `SYNTAX_FAILURES` | 437 | +| variable | `RUNTIME_FAILURES` | 458 | +| function | `practical_project(language)` | 477 | +| parameter | `language` | 477 | +| variable | `content` | 486 | +| variable | `entry` | 486 | +| variable | `exit_code` | 494 | +| variable | `name` | 494 | +| variable | `stdin` | 494 | +| variable | `stdout` | 494 | + ## examples/projects.py | Kind | Declaration | Line | @@ -715,6 +734,181 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `expected` | 139 | | variable | `output` | 140 | +## scripts/compiler_acceptance.py + +| Kind | Declaration | Line | +|---|---|---| +| variable | `RELEASE_SOURCE` | 40 | +| variable | `RELEASE_WORKER_SHA256` | 41 | +| function | `require(condition, message)` | 44 | +| parameter | `condition` | 44 | +| parameter | `message` | 44 | +| function | `source_digest(text)` | 50 | +| parameter | `text` | 50 | +| function | `source_line_count(text)` | 55 | +| parameter | `text` | 55 | +| function | `validate_coverage_completion(coverage, project)` | 60 | +| parameter | `coverage` | 60 | +| parameter | `project` | 60 | +| variable | `wrapped` | 62 | +| variable | `cases` | 64 | +| variable | `actual` | 69 | +| variable | `expected` | 69 | +| variable | `wrapped` | 85 | +| variable | `expected_status` | 86 | +| function | `validate_locations(result, project)` | 93 | +| parameter | `project` | 93 | +| parameter | `result` | 93 | +| variable | `file` | 95 | +| variable | `sources` | 95 | +| variable | `item` | 97 | +| variable | `line` | 98 | +| variable | `name` | 98 | +| function | `validate_positive(result, project)` | 106 | +| parameter | `project` | 106 | +| parameter | `result` | 106 | +| variable | `compilation` | 109 | +| variable | `checks` | 116 | +| variable | `observed` | 118 | +| variable | `actual` | 120 | +| variable | `expected` | 120 | +| variable | `oracle` | 130 | +| variable | `case` | 131 | +| variable | `benchmark` | 135 | +| variable | `samples` | 136 | +| variable | `value` | 139 | +| variable | `coverage` | 145 | +| variable | `entry` | 148 | +| variable | `content` | 149 | +| variable | `file` | 149 | +| variable | `measured` | 155 | +| variable | `name` | 155 | +| variable | `file` | 158 | +| variable | `source` | 158 | +| variable | `line` | 163 | +| variable | `executed` | 165 | +| variable | `missing` | 165 | +| variable | `summary` | 171 | +| variable | `covered` | 172 | +| variable | `extra` | 180 | +| variable | `first` | 184 | +| function | `negative_project(project, kind)` | 207 | +| parameter | `kind` | 207 | +| parameter | `project` | 207 | +| variable | `data` | 209 | +| variable | `replacement` | 215 | +| function | `validate_negative(result, project, kind)` | 226 | +| parameter | `kind` | 226 | +| parameter | `project` | 226 | +| parameter | `result` | 226 | +| variable | `expected_oracle` | 237 | +| variable | `case` | 238 | +| variable | `checks` | 241 | +| variable | `line` | 259 | +| variable | `item` | 265 | +| variable | `case` | 276 | +| variable | `exit_code` | 283 | +| variable | `marker` | 288 | +| class | `Evidence` | 305 | +| function | `__init__(self, output, provenance)` | 308 | +| parameter | `output` | 308 | +| parameter | `provenance` | 308 | +| parameter | `self` | 308 | +| function | `record(self, language, kind, project, result, error)` | 315 | +| parameter | `error` | 316 | +| parameter | `kind` | 316 | +| parameter | `language` | 316 | +| parameter | `project` | 316 | +| parameter | `result` | 316 | +| parameter | `self` | 316 | +| variable | `row` | 319 | +| variable | `key` | 333 | +| function | `save(self)` | 337 | +| parameter | `self` | 337 | +| variable | `record` | 339 | +| variable | `temporary` | 340 | +| variable | `lines` | 343 | +| variable | `row` | 351 | +| variable | `checks` | 352 | +| variable | `lines` | 356 | +| function | `broker_scenarios(evidence)` | 364 | +| parameter | `evidence` | 364 | +| variable | `language` | 366 | +| variable | `baseline` | 367 | +| variable | `kind` | 368 | +| variable | `project` | 371 | +| variable | `result` | 372 | +| variable | `error` | 373 | +| variable | `result` | 375 | +| variable | `error` | 381 | +| function | `anchor_scenarios(evidence)` | 385 | +| parameter | `evidence` | 385 | +| variable | `language` | 387 | +| variable | `missing_line` | 388 | +| variable | `project` | 388 | +| variable | `unused_line` | 388 | +| variable | `result` | 389 | +| variable | `error` | 390 | +| variable | `result` | 392 | +| variable | `item` | 403 | +| variable | `error` | 408 | +| function | `private_scenarios(evidence)` | 412 | +| parameter | `evidence` | 412 | +| variable | `credentials` | 414 | +| variable | `key` | 415 | +| variable | `directory` | 416 | +| variable | `app` | 417 | +| variable | `client` | 422 | +| variable | `jobs` | 430 | +| variable | `language` | 431 | +| variable | `project` | 432 | +| variable | `result` | 433 | +| variable | `error` | 434 | +| variable | `response` | 436 | +| variable | `revision` | 438 | +| variable | `response` | 439 | +| variable | `job` | 441 | +| variable | `deadline` | 442 | +| variable | `response` | 444 | +| variable | `report` | 446 | +| variable | `result` | 451 | +| variable | `exported` | 453 | +| variable | `html` | 458 | +| variable | `error` | 465 | +| variable | `archives` | 469 | +| variable | `response` | 484 | +| variable | `job` | 490 | +| variable | `result_sha256` | 490 | +| variable | `response` | 491 | +| variable | `exported` | 498 | +| function | `provenance(runtime, expected_source, lane)` | 516 | +| parameter | `expected_source` | 516 | +| parameter | `lane` | 516 | +| parameter | `runtime` | 516 | +| variable | `runtime` | 524 | +| variable | `source` | 529 | +| variable | `dirty` | 531 | +| variable | `worker` | 540 | +| variable | `command` | 545 | +| variable | `image` | 546 | +| function | `toolchain_versions()` | 570 | +| variable | `commands` | 572 | +| variable | `program` | 586 | +| variable | `name` | 591 | +| variable | `directory` | 592 | +| variable | `root` | 593 | +| variable | `argv` | 595 | +| variable | `argv` | 597 | +| variable | `code` | 599 | +| variable | `output` | 599 | +| variable | `versions` | 601 | +| variable | `value` | 607 | +| function | `main()` | 615 | +| variable | `parser` | 617 | +| variable | `args` | 622 | +| variable | `evidence` | 623 | +| variable | `language` | 644 | + ## scripts/draft_lookup.py | Kind | Declaration | Line | @@ -1013,6 +1207,235 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `identifier` | 572 | | variable | `record` | 577 | +## scripts/released_service_probe.py + +| Kind | Declaration | Line | +|---|---|---| +| variable | `SOURCE` | 42 | +| variable | `WORKER_SHA256` | 43 | +| variable | `VERSION` | 44 | +| variable | `RELEASE_ID` | 45 | +| variable | `REPOSITORY` | 46 | +| variable | `API` | 47 | +| variable | `PAYLOAD` | 48 | +| variable | `PAYLOAD_ID` | 49 | +| variable | `PAYLOAD_BYTES` | 50 | +| variable | `PAYLOAD_SHA256` | 51 | +| variable | `MAX_ARCHIVE` | 52 | +| variable | `MAX_EXPANDED` | 53 | +| variable | `MAX_MEMBER` | 54 | +| variable | `MAX_RESPONSE` | 55 | +| variable | `EXPECTED_LANGUAGES` | 56 | +| function | `require(condition, message)` | 72 | +| parameter | `condition` | 72 | +| parameter | `message` | 72 | +| function | `strict_json(body)` | 78 | +| parameter | `body` | 78 | +| function | `pairs(items)` | 81 | +| parameter | `items` | 81 | +| variable | `result` | 82 | +| function | `constant(_value)` | 86 | +| parameter | `_value` | 86 | +| function | `hosted_output(output)` | 92 | +| parameter | `output` | 92 | +| variable | `workspace` | 102 | +| variable | `selected` | 104 | +| class | `Evidence` | 117 | +| function | `__init__(self, output)` | 120 | +| parameter | `output` | 120 | +| parameter | `self` | 120 | +| function | `save(self)` | 134 | +| parameter | `self` | 134 | +| variable | `temporary` | 135 | +| function | `stage(self, stage)` | 141 | +| parameter | `public` | 141 | +| parameter | `self` | 141 | +| parameter | `stage` | 141 | +| function | `bounded_body(response, maximum)` | 147 | +| parameter | `maximum` | 147 | +| parameter | `response` | 147 | +| variable | `body` | 149 | +| variable | `chunk` | 150 | +| function | `api_json(client, path)` | 156 | +| parameter | `client` | 156 | +| parameter | `path` | 156 | +| variable | `response` | 159 | +| function | `release_assets(release, tag)` | 164 | +| parameter | `release` | 164 | +| parameter | `tag` | 164 | +| variable | `rows` | 182 | +| variable | `row` | 184 | +| variable | `row` | 188 | +| variable | `row` | 189 | +| variable | `row` | 190 | +| variable | `found` | 193 | +| variable | `row` | 193 | +| variable | `maximum` | 201 | +| variable | `name` | 201 | +| variable | `row` | 202 | +| variable | `expected` | 211 | +| function | `download_url(url, initial)` | 216 | +| parameter | `initial` | 216 | +| parameter | `url` | 216 | +| variable | `prefix` | 219 | +| variable | `parsed` | 221 | +| function | `download(client, row, target)` | 240 | +| parameter | `client` | 240 | +| parameter | `row` | 240 | +| parameter | `target` | 240 | +| variable | `initial` | 242 | +| variable | `url` | 251 | +| variable | `started` | 252 | +| variable | `_` | 253 | +| variable | `response` | 256 | +| variable | `url` | 258 | +| variable | `digest` | 262 | +| variable | `measured` | 262 | +| variable | `stream` | 263 | +| variable | `chunk` | 264 | +| variable | `measured` | 265 | +| function | `preflight_tar(stream)` | 280 | +| parameter | `stream` | 280 | +| variable | `expanded` | 282 | +| variable | `headers` | 282 | +| variable | `raw` | 283 | +| variable | `header` | 285 | +| variable | `expanded` | 286 | +| variable | `expanded` | 290 | +| variable | `chunk` | 292 | +| variable | `expanded` | 293 | +| variable | `headers` | 296 | +| variable | `number` | 298 | +| variable | `size` | 300 | +| variable | `kind` | 301 | +| variable | `limit` | 303 | +| variable | `remaining` | 305 | +| variable | `chunk` | 307 | +| variable | `remaining` | 311 | +| variable | `expanded` | 312 | +| function | `unpack(archive, destination)` | 316 | +| parameter | `archive` | 316 | +| parameter | `destination` | 316 | +| variable | `expanded` | 319 | +| variable | `seen` | 319 | +| variable | `stream` | 320 | +| variable | `package` | 324 | +| variable | `index` | 325 | +| variable | `member` | 325 | +| variable | `name` | 326 | +| variable | `parts` | 329 | +| variable | `c` | 337 | +| variable | `part` | 338 | +| variable | `key` | 341 | +| variable | `expanded` | 346 | +| variable | `selected` | 348 | +| variable | `source` | 353 | +| variable | `measured` | 355 | +| variable | `output` | 356 | +| variable | `chunk` | 357 | +| variable | `measured` | 358 | +| variable | `resources` | 363 | +| variable | `metadata` | 364 | +| variable | `value` | 366 | +| variable | `worker` | 382 | +| variable | `service` | 387 | +| function | `service_environment(home, token, nonce)` | 392 | +| parameter | `home` | 392 | +| parameter | `nonce` | 392 | +| parameter | `token` | 392 | +| variable | `config` | 394 | +| function | `request_json(client, method, path, data)` | 410 | +| parameter | `client` | 410 | +| parameter | `data` | 410 | +| parameter | `method` | 410 | +| parameter | `path` | 410 | +| variable | `response` | 416 | +| class | `Service` | 421 | +| function | `__init__(self, executable, data, home)` | 424 | +| parameter | `data` | 424 | +| parameter | `executable` | 424 | +| parameter | `home` | 424 | +| parameter | `self` | 424 | +| variable | `output` | 428 | +| function | `ready(self)` | 437 | +| parameter | `self` | 437 | +| variable | `deadline` | 439 | +| variable | `source` | 441 | +| variable | `body` | 442 | +| variable | `matches` | 444 | +| variable | `origin` | 447 | +| variable | `candidate` | 448 | +| variable | `health` | 456 | +| function | `stop(self)` | 471 | +| parameter | `self` | 471 | +| function | `validate_export(exported, result, revision)` | 484 | +| parameter | `exported` | 484 | +| parameter | `result` | 484 | +| parameter | `revision` | 484 | +| variable | `key` | 488 | +| variable | `value` | 488 | +| variable | `history` | 491 | +| variable | `snapshot` | 504 | +| variable | `attempt` | 518 | +| variable | `actual` | 519 | +| variable | `key` | 526 | +| function | `exercise(service, data, evidence)` | 536 | +| parameter | `data` | 536 | +| parameter | `evidence` | 536 | +| parameter | `service` | 536 | +| variable | `alias` | 539 | +| variable | `password` | 539 | +| variable | `key` | 540 | +| variable | `client` | 544 | +| variable | `stored` | 549 | +| variable | `language` | 550 | +| variable | `project` | 552 | +| variable | `revision` | 556 | +| variable | `identifier` | 557 | +| variable | `job` | 563 | +| variable | `deadline` | 565 | +| variable | `result` | 567 | +| variable | `exported` | 574 | +| variable | `digest` | 580 | +| function | `restored(service, data, private)` | 607 | +| parameter | `data` | 607 | +| parameter | `private` | 607 | +| parameter | `service` | 607 | +| variable | `client` | 609 | +| variable | `history` | 617 | +| variable | `row` | 620 | +| variable | `row` | 623 | +| variable | `result` | 624 | +| variable | `exported` | 625 | +| function | `canonical_digest(value)` | 640 | +| parameter | `value` | 640 | +| function | `cleanup_owned(services, temporary)` | 647 | +| parameter | `services` | 647 | +| parameter | `temporary` | 647 | +| variable | `failed` | 649 | +| variable | `service` | 650 | +| variable | `failed` | 660 | +| variable | `service` | 661 | +| function | `main()` | 670 | +| variable | `parser` | 672 | +| variable | `args` | 676 | +| variable | `output` | 677 | +| variable | `evidence` | 678 | +| variable | `temporary` | 679 | +| variable | `services` | 681 | +| variable | `client` | 683 | +| variable | `payload` | 684 | +| variable | `sidecar` | 684 | +| variable | `archive` | 688 | +| variable | `checksum` | 688 | +| variable | `digest` | 689 | +| variable | `executable` | 696 | +| variable | `data` | 698 | +| variable | `first` | 700 | +| variable | `private` | 703 | +| variable | `second` | 706 | +| variable | `row` | 715 | + ## scripts/training_checks.py | Kind | Declaration | Line | @@ -2512,6 +2935,46 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `_` | 107 | | variable | `saved` | 108 | +## tests/test_compiler_acceptance.py + +| Kind | Declaration | Line | +|---|---|---| +| function | `complete_result(project)` | 24 | +| parameter | `project` | 24 | +| variable | `content` | 26 | +| variable | `case` | 42 | +| variable | `case` | 46 | +| variable | `case` | 63 | +| function | `test_practical_matrix_is_complete_and_independent()` | 80 | +| variable | `language` | 84 | +| variable | `project` | 85 | +| variable | `case` | 87 | +| variable | `duplicate` | 91 | +| function | `test_positive_validator_rejects_incomplete_or_fabricated_evidence(defect)` | 124 | +| parameter | `defect` | 124 | +| variable | `project` | 126 | +| variable | `result` | 127 | +| function | `test_exact_compiler_location_and_nonexecution_are_required()` | 178 | +| variable | `project` | 180 | +| variable | `result` | 181 | +| variable | `defect` | 199 | +| variable | `changed` | 200 | +| function | `test_runtime_failure_is_distinct_from_incorrect_oracle()` | 211 | +| variable | `project` | 213 | +| variable | `result` | 214 | +| variable | `wrong` | 236 | +| function | `test_real_execution_guard_refuses_this_pc_before_subprocess(monkeypatch)` | 247 | +| parameter | `monkeypatch` | 247 | +| function | `test_actual_first_hosted_reports_retain_their_real_tool_semantics(language, kind)` | 258 | +| parameter | `kind` | 258 | +| parameter | `language` | 258 | +| variable | `observed` | 262 | +| variable | `item` | 265 | +| variable | `row` | 265 | +| variable | `project` | 266 | +| variable | `changed` | 271 | +| variable | `changed` | 275 | + ## tests/test_completed_offload.py | Kind | Declaration | Line | @@ -3627,6 +4090,114 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | parameter | `data` | 705 | | variable | `body` | 711 | +## tests/test_released_service_probe.py + +| Kind | Declaration | Line | +|---|---|---| +| function | `test_hosted_guard_refuses_user_pc_before_any_work(monkeypatch, tmp_path)` | 27 | +| parameter | `monkeypatch` | 27 | +| parameter | `tmp_path` | 27 | +| function | `test_hosted_output_is_confined_and_verifier_source_explicit(monkeypatch, tmp_path)` | 36 | +| parameter | `monkeypatch` | 36 | +| parameter | `tmp_path` | 36 | +| variable | `name` | 39 | +| variable | `value` | 39 | +| function | `metadata()` | 56 | +| variable | `prefix` | 58 | +| variable | `rows` | 59 | +| variable | `i` | 77 | +| variable | `rows` | 77 | +| function | `test_publication_or_replaced_asset_never_authorizes_execution(defect)` | 95 | +| parameter | `defect` | 95 | +| variable | `release` | 97 | +| variable | `tag` | 97 | +| function | `test_download_binds_canonical_hosts_actual_bytes_and_digest(tmp_path)` | 117 | +| parameter | `tmp_path` | 117 | +| variable | `body` | 119 | +| variable | `initial` | 120 | +| variable | `row` | 121 | +| variable | `seen` | 127 | +| function | `transport(request)` | 129 | +| parameter | `request` | 129 | +| variable | `client` | 143 | +| variable | `digest` | 144 | +| function | `archive(path, monkeypatch, extra, bad_schema)` | 153 | +| parameter | `bad_schema` | 153 | +| parameter | `extra` | 153 | +| parameter | `monkeypatch` | 153 | +| parameter | `path` | 153 | +| variable | `worker` | 155 | +| variable | `value` | 157 | +| variable | `output` | 165 | +| variable | `body` | 166 | +| variable | `name` | 166 | +| variable | `member` | 171 | +| variable | `content` | 176 | +| variable | `member` | 176 | +| function | `test_real_tar_provenance_and_no_setuid_preservation(monkeypatch, tmp_path)` | 180 | +| parameter | `monkeypatch` | 180 | +| parameter | `tmp_path` | 180 | +| variable | `source` | 181 | +| variable | `service` | 183 | +| function | `test_actual_tar_negative_members_fail_closed(monkeypatch, tmp_path, kind)` | 194 | +| parameter | `kind` | 194 | +| parameter | `monkeypatch` | 194 | +| parameter | `tmp_path` | 194 | +| variable | `source` | 195 | +| variable | `member` | 196 | +| variable | `content` | 199 | +| variable | `content` | 204 | +| function | `test_service_environment_cannot_inherit_provider_cloud_or_github_keys(monkeypatch, tmp_path)` | 219 | +| parameter | `monkeypatch` | 219 | +| parameter | `tmp_path` | 219 | +| variable | `result` | 223 | +| variable | `key` | 225 | +| function | `test_failed_startup_cleanup_targets_only_owned_child_and_preserves_live_recovery(tmp_path, can_exit)` | 230 | +| parameter | `can_exit` | 230 | +| parameter | `tmp_path` | 230 | +| variable | `temporary` | 232 | +| variable | `calls` | 235 | +| variable | `state` | 236 | +| function | `terminate()` | 238 | +| function | `wait(timeout)` | 241 | +| parameter | `timeout` | 241 | +| variable | `process` | 248 | +| function | `test_actual_store_timeline_identity_and_shape_fail_closed(tmp_path, defect)` | 255 | +| parameter | `defect` | 255 | +| parameter | `tmp_path` | 255 | +| variable | `store` | 257 | +| variable | `revision` | 258 | +| variable | `identifier` | 259 | +| variable | `result` | 263 | +| variable | `exported` | 264 | +| variable | `timeline` | 266 | +| function | `test_incremental_receipt_and_full_restored_export_equality(monkeypatch, tmp_path)` | 283 | +| parameter | `monkeypatch` | 283 | +| parameter | `tmp_path` | 283 | +| variable | `data` | 285 | +| variable | `store` | 287 | +| variable | `validations` | 288 | +| variable | `tamper` | 289 | +| function | `request(_client, method, path, body)` | 291 | +| parameter | `_client` | 291 | +| parameter | `body` | 291 | +| parameter | `method` | 291 | +| parameter | `path` | 291 | +| variable | `revision` | 300 | +| variable | `identifier` | 301 | +| variable | `project` | 302 | +| variable | `result` | 303 | +| variable | `identifier` | 311 | +| variable | `result` | 312 | +| function | `validate(result, project)` | 331 | +| parameter | `project` | 331 | +| parameter | `result` | 331 | +| variable | `evidence` | 337 | +| variable | `service` | 338 | +| variable | `private` | 339 | +| variable | `row` | 341 | +| variable | `text` | 342 | + ## tests/test_sanitizer_probe.py | Kind | Declaration | Line | diff --git a/docs/compiler-execution.md b/docs/compiler-execution.md new file mode 100644 index 0000000..33493c2 --- /dev/null +++ b/docs/compiler-execution.md @@ -0,0 +1,120 @@ +# Realistic compiler and execution validation + +By Jason Stys. The [dated results](reports/hosted-compilers-2026-11-10.md) +distinguish actual hosted observations from local unit tests or planned checks. + +## Realistic projects + +Twelve standard-library CLI projects each have six independently authored +input/output/exit-code oracles. No AI provider, private source or real account is used. + +| Adapter | Practical example | Important boundaries | +|---|---|---| +| Python | Telemetry count, sum, minimum and maximum | Empty, negative, malformed and out-of-range samples | +| JavaScript | Integer-cent invoice and basis-point discount | Zero quantity, rounding and discount limits | +| TypeScript | Typed temperature/vibration severity policy | Thresholds and malformed readings | +| C++ | Integer sensor calibration | Offset/scale changes and bounded inputs | +| C | Bounded byte XOR checksum | Empty/multiple inputs and rejected tokens | +| Java | Available-capacity allocation | Zero capacity, exhaustion and malformed fields | +| C# | Decimal invoice and tax | Decimal rounding, zero and rejected amounts | +| Rust | Inventory stock adjustments | Negative adjustments, insufficient stock and lexical errors | +| PHP | Bulk price quotation | Threshold discounts, zero and rejected quantities | +| Bash | Log severity summary | Empty/mixed input and invalid records | +| SQLite SQL | Product/order revenue reports | Joins, ordering, missing products, constraints and malformed queries | +| GNU x86-64 assembly | Alarm-bit decoder with real input and branches | Independent bits, all/zero bits and rejected input | + +Approved rejection cases pass only when their exact expected output and exit +status match. A job reporting `completed` means it ran; that alone is not a test pass. + +## Independent checks + +1. Compile/run every realistic program through the ordinary restricted broker. + Require every case's name, stdout, exit status and complete outcome; immutable + oracle hash; real timing/RSS samples; native tool completion; exact source + hashes and consistent coverage line partitions/counts. +2. Give a successful program an intentionally incorrect expected output. Its + actual answer must still match the independently known correct result and it + must exit normally, while the comparison is reported as failed. +3. Give all eleven adapters with a separate compile/syntax phase malformed source. + Require nonzero compilation, no executed cases, blocked coverage/unused tools, + and the exact independently authored filename/line/source hash. SQLite parses + its schema/query during execution and is not described as compiled. +4. Run valid programs with authored runtime faults or explicit exit 7. Require + the specific expected failure exit and, for exceptions, its synthetic marker; + an unrelated loader error is insufficient. +5. Retain eleven independent fixtures with exact unused and uncovered anchors. + SQL statement starts, Bash command entry and assembly debug-backed instructions + retain their different measurement meanings. +6. Save/run realistic programs through a real authenticated private account API. + Export JSON/HTML; seal; log out; prove anonymous history is denied; log back in; + unlock; compare complete saved results and exported JSON after reopening. +7. In the release lane, verify/download the published Linux portable package, + start its actual frozen service and repeat realistic API execution plus a full + service restart with encrypted history. This tests the released backend binary. + +## Hosted execution and evidence + +The [workflow](../.github/workflows/compiler-execution.yml) runs a fixed release +lane (`eb9b1cd0c3e3fd1e70e397edeffb059073464f74`, version 0.4.2) and a current-source +lane. It identifies validation and runtime source separately, rejects dirty +tracked runtime code or controller imports from the wrong checkout, and records +the worker image identity. The real-execution entry points refuse non-hosted +Linux execution. Docker on Jason's PC is unchanged. + +The worker retains no network, a read-only root, UID/GID 65534, dropped +capabilities, no privilege escalation, bounded memory/CPU/processes and one +read-only job input. No account store or provider key is mounted into the target. +These are development controls, not certification for public hostile multi-tenant use. + +The image is freshly built from the identified source. Mutable upstream Docker +tags and APT packages mean it is not asserted identical to a historical image. +Recorded tool versions/image identities/raw results apply to the observed run. +Hosted Linux success does not imply that Windows Docker was installed/repaired, +or that arbitrary frameworks, packages, formats and ISAs are supported. + +Only small synthetic JSON/table evidence is uploaded with three-day retention; +the run also has a result summary. Binaries and account directories are excluded. +This per-upload setting neither changes account retention nor deletes existing +artifacts/caches. See [GitHub artifact documentation](https://docs.github.com/en/actions/tutorials/store-and-share-data). + +## Files and reproduction + +| File | Responsibility | +|---|---| +| `examples/practical_projects.py` | Twelve programs, 72 literal oracles and failure controls | +| `scripts/compiler_acceptance.py` | Hosted source/broker/private-API checks and incremental evidence | +| `scripts/released_service_probe.py` | Published Linux package verification, frozen execution and restart | +| `tests/test_compiler_acceptance.py` | Evidence mutation regressions and local-execution refusal | +| `tests/test_released_service_probe.py` | Safe package/lifecycle regressions without local Docker | +| `.github/workflows/compiler-execution.yml` | Fixed source lanes, hosted worker build and small reports | + +Use **Actions → Compiler execution acceptance → Run workflow** on `main`, or +`gh workflow run compiler-execution.yml --repo JasonStys/code-flow-studio --ref main`. +Local unit tests can use `python -m pytest tests/test_compiler_acceptance.py +tests/test_released_service_probe.py`; they do not execute the target source. + +These are flat-project demonstrations, not production domain algorithms or an +empirical Big-O measurement. Timings include process startup; small changes and +cross-language rankings are not reliable performance comparisons. + +### Tool-specific coverage completion + +The C# malformed-input case correctly returns exit 2 in the ordinary execution. +Coverlet 8.0.1 wraps that expected nonzero target exit as collector exit 1. The +application conservatively retains `coverage.status: partial`, despite a valid +source-bound report. Acceptance requires all six ordinary case outputs/exits, +all instrumented case identities, successful zero-exit instrumentation, the exact +known nonzero wrapper behavior and a generated coverage report. It never relabels +that report as fully measured. An unrelated collector failure is rejected. +The [pinned Coverlet console implementation](https://github.com/coverlet-coverage/coverlet/blob/v8.0.1/src/coverlet.console/Program.cs) +maps positive target exits to its test-failed wrapper status. The exact nonzero +target exit is independently checked in ordinary execution, not inferred from +the instrumented wrapper's exit 1. + +coverage.py may retain the module-docstring trace event while excluding it from +its executable-statement count. The validator permits only that source-proven +one-line difference. PHP Xdebug can report an implicit return at an empty terminal +source line; location bounds use the same newline convention as the app/editor. +The [observed report snapshots](../tests/fixtures/compiler-reports-042.json) retain +these actual first-run results, including their original failed-acceptance labels; +replay unit tests are not a new compiler run. diff --git a/docs/handbook.md b/docs/handbook.md index 27cf5be..a51f2a9 100644 --- a/docs/handbook.md +++ b/docs/handbook.md @@ -2,9 +2,11 @@ By Jason Stys. -Current source is the 0.4.2 delivery candidate. Read its [release notes](releases/v0.4.2.md) -and the [observed draft-verification CI correction](reports/draft-release-visibility-2026-11-10.md) -before interpreting release availability. The unchanged Word/Excel editions below +Version [0.4.2 is published](https://github.com/JasonStys/code-flow-studio/releases/tag/v0.4.2). +Read its [release notes](releases/v0.4.2.md), the +[observed draft-verification CI correction](reports/draft-release-visibility-2026-11-10.md) +and the [realistic hosted compiler validation](compiler-execution.md). +The unchanged Word/Excel editions below remain explicit historical snapshots of accepted runtime source `7d2158a57bb49e8f837fffbfe23b301902465412`; they are not regenerated or relabeled as 0.4.2 acceptance. The 0.4.1 draft and its assets remain preserved and unpublished. diff --git a/docs/reports/hosted-compilers-2026-11-10.json b/docs/reports/hosted-compilers-2026-11-10.json new file mode 100644 index 0000000..0e19248 --- /dev/null +++ b/docs/reports/hosted-compilers-2026-11-10.json @@ -0,0 +1,385 @@ +{ + "schema": 1, + "recordedUtc": "2026-11-10T18:16:25.631553+00:00", + "version": "0.4.2", + "publishedRuntimeSource": "eb9b1cd0c3e3fd1e70e397edeffb059073464f74", + "validationHead": "9b5180489cb663a30e120c08ff37a469429db85f", + "observedRuns": [ + { + "databaseId": 42871745673, + "workflowName": "Compiler execution acceptance", + "event": "pull_request", + "headSha": "9b5180489cb663a30e120c08ff37a469429db85f", + "conclusion": "success", + "url": "https://github.com/JasonStys/code-flow-studio/actions/runs/42871745673", + "jobs": [ + { + "name": "Realistic compilers (release-0.4.2)", + "conclusion": "success" + }, + { + "name": "Realistic compilers (current)", + "conclusion": "success" + } + ] + }, + { + "databaseId": 42871731670, + "workflowName": "Compiler execution acceptance", + "event": "push", + "headSha": "9b5180489cb663a30e120c08ff37a469429db85f", + "conclusion": "success", + "url": "https://github.com/JasonStys/code-flow-studio/actions/runs/42871731670", + "jobs": [ + { + "name": "Realistic compilers (current)", + "conclusion": "success" + }, + { + "name": "Realistic compilers (release-0.4.2)", + "conclusion": "success" + } + ] + }, + { + "databaseId": 42871745652, + "workflowName": "Validate", + "event": "pull_request", + "headSha": "9b5180489cb663a30e120c08ff37a469429db85f", + "conclusion": "success", + "url": "https://github.com/JasonStys/code-flow-studio/actions/runs/42871745652", + "jobs": [ + { + "name": "isolated-workers", + "conclusion": "success" + }, + { + "name": "controller", + "conclusion": "success" + }, + { + "name": "training-smoke", + "conclusion": "success" + } + ] + }, + { + "databaseId": 42871731784, + "workflowName": "Validate", + "event": "push", + "headSha": "9b5180489cb663a30e120c08ff37a469429db85f", + "conclusion": "success", + "url": "https://github.com/JasonStys/code-flow-studio/actions/runs/42871731784", + "jobs": [ + { + "name": "training-smoke", + "conclusion": "success" + }, + { + "name": "isolated-workers", + "conclusion": "success" + }, + { + "name": "controller", + "conclusion": "success" + } + ] + }, + { + "databaseId": 42871731707, + "workflowName": "Desktop packages", + "event": "push", + "headSha": "9b5180489cb663a30e120c08ff37a469429db85f", + "conclusion": "success", + "url": "https://github.com/JasonStys/code-flow-studio/actions/runs/42871731707", + "jobs": [ + { + "name": "package (ubuntu-latest)", + "conclusion": "success" + }, + { + "name": "package (windows-latest)", + "conclusion": "success" + }, + { + "name": "package (macos-latest)", + "conclusion": "success" + } + ] + }, + { + "databaseId": 42871745412, + "workflowName": "CodeQL", + "event": "pull_request", + "headSha": "9b5180489cb663a30e120c08ff37a469429db85f", + "conclusion": "success", + "url": "https://github.com/JasonStys/code-flow-studio/actions/runs/42871745412", + "jobs": [ + { + "name": "Analyze / python", + "conclusion": "success" + }, + { + "name": "Analyze / actions", + "conclusion": "success" + }, + { + "name": "Analyze / javascript-typescript", + "conclusion": "success" + } + ] + } + ], + "sourceLanes": [ + { + "provenance": { + "lane": "current", + "runtime_source": "c8288ea0bdc229c7bd7ec1a5eff93a4c314f4318", + "runtime_version": "0.4.2", + "worker_node_sha256": "750b665e545b7cf0806d4c493db86174f7f4e7ab2218aa239718e652ccdce158", + "worker_image_id": "sha256:6d523e50215bb93c7dfa37d15426cd3a67147927c6446617e37659d8a0c0cc69", + "validation_source": "c8288ea0bdc229c7bd7ec1a5eff93a4c314f4318", + "run_id": "42871745673", + "run_attempt": "1", + "host": "GitHub-hosted Linux", + "toolchain_caveat": "The image was freshly built from the identified source. Mutable upstream base tags/APT packages mean it is not asserted identical to a historical image.", + "isolation": { + "network": "none", + "user": "65534:65534", + "read_only": true, + "memory_mib": 1024, + "cpus": 2, + "pids": 128 + }, + "toolchain": { + "python": "Python 3.12.12", + "javascript": "v24.22.2", + "typescript": "Version 5.9.3", + "cpp": "g++ (Debian 12.2.0-14+deb12u1) 12.2.0", + "c": "gcc (Debian 12.2.0-14+deb12u1) 12.2.0", + "java": "javac 17.0.21", + "csharp": "8.0.427", + "rust": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "php": "PHP 8.2.35 (cli) (built: Oct 21 2026 06:29:51) (NTS)", + "bash": "GNU bash, version 5.2.15(1)-release (x86_64-pc-linux-gnu)", + "sql": "3.40.1 2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f3alt1", + "assembly": "GNU assembler (GNU Binutils for Debian) 2.40" + }, + "private_history": { + "login_after_logout": "passed", + "sealed_and_reopened_reports": 12, + "plaintext_removed": true + }, + "acceptance": "passed" + }, + "scenarios": 70, + "scenarioKinds": { + "positive": 12, + "wrong-oracle": 12, + "syntax-error": 11, + "runtime-error": 12, + "native-location": 11, + "private-api": 12 + }, + "positiveCases": 72, + "positiveCoverage": { + "python": "measured", + "javascript": "measured", + "typescript": "measured", + "cpp": "measured", + "c": "measured", + "java": "measured", + "csharp": "partial", + "rust": "measured", + "php": "measured", + "bash": "measured", + "sql": "measured", + "assembly": "measured" + } + }, + { + "provenance": { + "lane": "release-0.4.2", + "runtime_source": "eb9b1cd0c3e3fd1e70e397edeffb059073464f74", + "runtime_version": "0.4.2", + "worker_node_sha256": "750b665e545b7cf0806d4c493db86174f7f4e7ab2218aa239718e652ccdce158", + "worker_image_id": "sha256:ff720b8a0013f86014339b56031b2758d80f4715daace564719341401e62e331", + "validation_source": "c8288ea0bdc229c7bd7ec1a5eff93a4c314f4318", + "run_id": "42871745673", + "run_attempt": "1", + "host": "GitHub-hosted Linux", + "toolchain_caveat": "The image was freshly built from the identified source. Mutable upstream base tags/APT packages mean it is not asserted identical to a historical image.", + "isolation": { + "network": "none", + "user": "65534:65534", + "read_only": true, + "memory_mib": 1024, + "cpus": 2, + "pids": 128 + }, + "toolchain": { + "python": "Python 3.12.12", + "javascript": "v24.22.2", + "typescript": "Version 5.9.3", + "cpp": "g++ (Debian 12.2.0-14+deb12u1) 12.2.0", + "c": "gcc (Debian 12.2.0-14+deb12u1) 12.2.0", + "java": "javac 17.0.21", + "csharp": "8.0.427", + "rust": "rustc 1.99.1 (58cf8f9c3 2026-10-13)", + "php": "PHP 8.2.35 (cli) (built: Oct 21 2026 06:29:51) (NTS)", + "bash": "GNU bash, version 5.2.15(1)-release (x86_64-pc-linux-gnu)", + "sql": "3.40.1 2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f3alt1", + "assembly": "GNU assembler (GNU Binutils for Debian) 2.40" + }, + "private_history": { + "login_after_logout": "passed", + "sealed_and_reopened_reports": 12, + "plaintext_removed": true + }, + "acceptance": "passed" + }, + "scenarios": 70, + "scenarioKinds": { + "positive": 12, + "wrong-oracle": 12, + "syntax-error": 11, + "runtime-error": 12, + "native-location": 11, + "private-api": 12 + }, + "positiveCases": 72, + "positiveCoverage": { + "python": "measured", + "javascript": "measured", + "typescript": "measured", + "cpp": "measured", + "c": "measured", + "java": "measured", + "csharp": "partial", + "rust": "measured", + "php": "measured", + "bash": "measured", + "sql": "measured", + "assembly": "measured" + } + } + ], + "publishedFrozenService": { + "approved_cases": 72, + "languages": [ + { + "approved_cases": 6, + "language": "python", + "result_sha256": "ccfbe8978e1b18ad63091e79f80ea5617468aa7ac7dc5b4a0e8b056754631485", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "javascript", + "result_sha256": "0a935e58d2b89363cb09308f8b8d03e1dadc53a6797cac597be25f8a79fc1af7", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "typescript", + "result_sha256": "59881a8cbb6234c4bf5a2ea025bdb515f40d715e7873c3c7d55a685e317c7468", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "cpp", + "result_sha256": "51a5e87307fb3b8c584acb356fd5ef2f234432bd4f4ac733d5569a2980a42fd2", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "c", + "result_sha256": "7f2e5f51256c7f17cfeaf8b328e7f1bac73c78f0e5eebe2048c252c4a9c1f6a3", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "java", + "result_sha256": "876c2c483f6d11cd336f3835cf2834f179edbb0eaab66436ab0dbcafb6504a7c", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "csharp", + "result_sha256": "337d85934549d51591059ad74c97df2bc87d7962be76f6b0e05a37428674742b", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "rust", + "result_sha256": "49432456d723c78c62d596065589833c5abbe9e38b5fb92288affbff3fe575f9", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "php", + "result_sha256": "d832630a2406064ff039aaacb021043a4548c173f53379eacb7134d618b5e059", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "bash", + "result_sha256": "155665a7c923c296da5d228249e7b29a3332d65b33d080f484b84a28907e3c8f", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "sql", + "result_sha256": "9ce95c61c6453b2cb58722827e7dd9c1d610cf61e6cb7bd2a37cc0b86cc3aafb", + "status": "passed" + }, + { + "approved_cases": 6, + "language": "assembly", + "result_sha256": "c7c664da3d7c467a1aaa39e1439d2896d847b83d16bf2795e2a1c7bc38fd1fca", + "status": "passed" + } + ], + "payload_bytes": 188603375, + "payload_sha256": "1e75085bae4ab2611ead61a513b60f444941495608840cf33875d940cf963ebe", + "release_id": 440504623, + "released_source": "eb9b1cd0c3e3fd1e70e397edeffb059073464f74", + "restored_reports": 12, + "schema": 1, + "scope": "Published Linux service with a fresh source-bound0.4.2 hosted Docker worker; not a local PC compiler run or historical image-byte proof", + "stage": "complete", + "status": "passed", + "verifier_source": "c8288ea0bdc229c7bd7ec1a5eff93a4c314f4318", + "version": "0.4.2", + "worker_sha256": "750b665e545b7cf0806d4c493db86174f7f4e7ab2218aa239718e652ccdce158" + }, + "evidenceHashes": [ + { + "evidence": "run-42871745673-current/results.json", + "sha256": "8560f6c35a38bfd4e4d22714f9083737476f72e0adfb1b5748df636b0e686d6c" + }, + { + "evidence": "run-42871745673-release/results.json", + "sha256": "40dd2f281d75277634130beefa219a54b2e4167ada5b5ed0579b1b3db2307671" + }, + { + "evidence": "released-service-probe.json", + "sha256": "769e03ab2f9bc68fbfe701ac08d322ce5ca809d3eeefd5f3f8e9a614e11655b7" + } + ], + "normalValidation": { + "pythonTests": 345, + "pythonSubtests": 28, + "measuredCoveragePercent": 85.01, + "coverageFloorPercent": 80, + "localFocusedRegressions": 59, + "codeqlOpenAlertsOnPrMerge": 0 + }, + "limits": [ + "C# Coverlet expected nonzero wrapper retains partial coverage; ordinary outputs/exits pass6/6.", + "Worker image freshly built from recorded source and mutable upstream tags/APT, not historical byte identity.", + "No local PC Docker/compiler execution, repairs or app/account changes.", + "The redundant broad local Windows unit attempt was stopped without a result; full Linux hosted suite passed.", + "These source-bound observations do not certify an unrun later commit; check the current branch Actions." + ] +} diff --git a/docs/reports/hosted-compilers-2026-11-10.md b/docs/reports/hosted-compilers-2026-11-10.md new file mode 100644 index 0000000..f88bd24 --- /dev/null +++ b/docs/reports/hosted-compilers-2026-11-10.md @@ -0,0 +1,65 @@ +# Hosted compiler and code-execution validation — 10 November 2026 + +By Jason Stys. **The released 0.4.2 service and both source lanes passed hosted compiler acceptance.** Docker on the owner's PC and existing local accounts/data were unchanged. + +## Observed results + +| Check | Result | Observed run | +|---|---|---| +| Compiler execution acceptance | 2 jobs passed | [42871745673](https://github.com/JasonStys/code-flow-studio/actions/runs/42871745673) | +| Validate | 3 jobs passed | [42871745652](https://github.com/JasonStys/code-flow-studio/actions/runs/42871745652) | +| Desktop packages | 3 jobs passed | [42871731707](https://github.com/JasonStys/code-flow-studio/actions/runs/42871731707) | +| CodeQL | 3 jobs passed | [42871745412](https://github.com/JasonStys/code-flow-studio/actions/runs/42871745412) | + +- **12 supported adapters, 72 distinct realistic input/output/exit oracles.** Each source lane passed 70 scenarios: 12 realistic programs, 12 wrong-oracle controls, 11 syntax failures, 12 runtime faults, 11 exact native-location fixtures and 12 authenticated private-API executions. +- Both source lanes sealed, logged out, logged in and reopened all 12 complete saved execution reports; anonymous history access was denied and managed plaintext was removed. +- The actual published Linux frozen service independently passed all 72 realistic cases. It sealed/exited, restarted, logged back in/unlocked, verified all 12 complete results and JSON exports, then sealed/exited again. +- The full hosted suite passed **345 tests and 28 subtests**, with **85.01% measured coverage** against the unchanged 80% floor. Browser, original worker/fuzz/location, private-worker, training and three-OS native package acceptance also passed. +- CodeQL completed Python, JavaScript/TypeScript and Actions analysis; the reviewed PR merge ref had zero open alerts. **59 focused local validator/protocol regressions passed.** + +These are observations for the linked runs, not a claim that every later commit is already tested. The [current main checks](https://github.com/JasonStys/code-flow-studio/actions?query=branch%3Amain) and [machine-readable dated evidence](hosted-compilers-2026-11-10.json) preserve that distinction. + +## Exact source and binary identity + +- Published release/source: [`eb9b1cd0c3e3fd1e70e397edeffb059073464f74`](https://github.com/JasonStys/code-flow-studio/tree/eb9b1cd0c3e3fd1e70e397edeffb059073464f74), [version 0.4.2](https://github.com/JasonStys/code-flow-studio/releases/tag/v0.4.2), release ID 440504623. +- Reviewed validation head: [`9b5180489cb663a30e120c08ff37a469429db85f`](https://github.com/JasonStys/code-flow-studio/commit/9b5180489cb663a30e120c08ff37a469429db85f); PR current-runtime merge checkout: `c8288ea0bdc229c7bd7ec1a5eff93a4c314f4318`. +- Accepted published Linux payload: asset 731850461, 188,603,375 bytes, SHA-256 `1e75085bae4ab2611ead61a513b60f444941495608840cf33875d940cf963ebe`. Live provider digest, exact sidecar, tag, bounded archive and embedded provenance were independently checked. +- Bundled worker `node_checks.cjs` SHA-256: `750b665e545b7cf0806d4c493db86174f7f4e7ab2218aa239718e652ccdce158`. Worker image IDs and raw evidence hashes are recorded in the JSON edition. + +## Actual observed toolchains + +| Adapter | Version reported inside restricted worker | +|---|---| +| python | Python 3.12.12 | +| javascript | v24.22.2 | +| typescript | Version 5.9.3 | +| cpp | g++ (Debian 12.2.0-14+deb12u1) 12.2.0 | +| c | gcc (Debian 12.2.0-14+deb12u1) 12.2.0 | +| java | javac 17.0.21 | +| csharp | 8.0.427 | +| rust | rustc 1.99.1 (58cf8f9c3 2026-10-13) | +| php | PHP 8.2.35 (cli) (built: Oct 21 2026 06:29:51) (NTS) | +| bash | GNU bash, version 5.2.15(1)-release (x86_64-pc-linux-gnu) | +| sql | 3.40.1 2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f3alt1 | +| assembly | GNU assembler (GNU Binutils for Debian) 2.40 | + +## Initial failures and corrections + +The [initial acceptance run](https://github.com/JasonStys/code-flow-studio/actions/runs/42869521158) is retained as failed evidence. All twelve languages already passed 6/6 ordinary cases in both source lanes and private execution. The new validator incorrectly assumed that every tool used the same line/count and collector-exit conventions. It was corrected without changing target sources, approved oracles, installed runtime or worker implementation. + +1. coverage.py can retain a module-docstring trace event that it excludes from executable-statement counts. Only the exact AST-proven one-line difference is allowed. +2. PHP Xdebug can report an implicit return at an empty terminal source line. Bounds now match the app/editor's original newline convention. +3. Coverlet 8.0.1 maps a nonzero target exit to collector exit 1. The C# malformed-price case correctly exits 2 in ordinary execution; instrumentation is honestly retained as **partial coverage**. Exact instrumented nonzero target exit is not independently recoverable. All successful collector exits, identities and generated reports are checked; unrelated collector errors and false fully-measured labels are rejected. +4. The frozen-service probe initially expected a list for `iteration_history`; the actual API returns the SQLite `Store.timeline` dictionary. The checker was fixed and its tests now exercise real temporary SQLite persistence and export identity instead of a mirrored mock schema. +5. The separate version probe now calls the actual trusted worker runtime-directory initializer before invoking tools; isolation permissions were not widened. + +Four original report snapshots remain unchanged in [observed regression fixtures](../../tests/fixtures/compiler-reports-042.json). Full source/result/export digests, explicit wrong-answer controls, authored failure exits and exact unused/uncovered anchors prevent a completed process or fabricated summary from counting as a pass. See the [test plan and measurement semantics](../compiler-execution.md). + +## Environment and remaining limits + +- No PC Docker changes, local compiler execution, real account/data upload, external AI API or paid provider call occurred in this validation. +- C# retains the stated partial collector coverage for approved nonzero rejection. Other eleven realistic positive adapters reported measured coverage. SQL statement starts, Bash command-entry and assembly debug-backed instructions retain their documented meanings. +- Images were freshly built from identified source; mutable upstream base tags/APT packages mean these are not historical image-byte claims. Timing includes startup/shared-runner noise and is not empirical Big-O or a cross-language ranking. +- The redundant broad local Windows unit attempt was stopped after ten minutes without a result and is not claimed as passed. The complete hosted Linux suite and separate genuine native package checks passed. +- Unsupported frameworks, dependencies, ISAs, every possible bug, production hostile multi-tenant safety and live AI quality are not established by these examples. +- Small synthetic Actions evidence has three-day per-upload retention; the dated summary is committed here. No old run/artifact/cache was deleted and no account retention policy changed. diff --git a/examples/practical_projects.py b/examples/practical_projects.py new file mode 100644 index 0000000..7a50336 --- /dev/null +++ b/examples/practical_projects.py @@ -0,0 +1,503 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# functions: _source(text)@L25; practical_project(language)@L477 +# variables: PROGRAMS@L30; CASES@L321; SYNTAX_FAILURES@L437; RUNTIME_FAILURES@L458; content@L486; entry@L486; exit_code@L494; name@L494; stdin@L494; stdout@L494 +# parameters: text@L25; language@L477 +# + +"""Jason Stys: independent, realistic CLI cases for every supported worker adapter. + +Only the restricted worker may compile or execute the embedded source strings. +``practical_project`` returns a fresh flat project; it never runs target code. +``PROGRAMS`` stores entry names/source, and ``CASES`` stores separately authored +input/output/exit oracles. ``SYNTAX_FAILURES`` binds deliberate compiler/parser +errors to exact authored lines. ``RUNTIME_FAILURES`` contains valid programs +which fail during execution. SQLite has no separate compilation phase. +Definition/variable line locations are maintained in docs/code-index.md. +""" + +from textwrap import dedent + +from studio.models import Project + + +def _source(text: str) -> str: + """Remove fixture indentation while preserving predictable one-based source lines.""" + return dedent(text).lstrip("\n") + + +PROGRAMS: dict[str, tuple[str, str]] = { + "python": ( + "main.py", + _source(''' + """Summarize at most 64 comma-separated, bounded integer telemetry samples.""" + import sys + + def summarize(text): + """Return count/sum/min/max; reject malformed or out-of-range samples.""" + if not text.strip(): + return "0|0|NA|NA" + values = [int(part.strip()) for part in text.strip().split(",")] + if len(values) > 64 or any(value < -1000 or value > 1000 for value in values): + raise ValueError("Invalid telemetry range") + return f"{len(values)}|{sum(values)}|{min(values)}|{max(values)}" + + try: + print(summarize(sys.stdin.read())) + except ValueError: + raise SystemExit(2) from None + '''), + ), + "javascript": ( + "main.js", + _source(""" + // Integer-cent invoice; half-up discount rounding avoids binary currency arithmetic. + const fields = require("node:fs").readFileSync(0, "utf8").trim().split(/\\s+/); + if (fields.length !== 3 || fields.some((field) => !/^[0-9]+$/.test(field))) { + process.exit(2); + } + const [quantity, unitCents, discountBps] = fields.map(Number); + if (quantity > 1000 || unitCents > 1000000 || discountBps > 10000) { + process.exit(2); + } + const subtotal = quantity * unitCents; + const discount = Math.floor((subtotal * discountBps + 5000) / 10000); + console.log(`${subtotal}|${discount}|${subtotal - discount}`); + """), + ), + "typescript": ( + "main.ts", + _source(""" + // Classify bounded temperature/vibration telemetry; this is an illustrative policy. + declare function require(name: "node:fs"): { + readFileSync(fd: number, encoding: "utf8"): string; + }; + declare const process: { exit(code: number): never }; + const fields = require("node:fs").readFileSync(0, "utf8").trim().split(/\\s+/); + if (fields.length !== 2 || fields.some((field) => !/^-?[0-9]+$/.test(field))) { + process.exit(2); + } + const temperature = Number(fields[0]); + const vibration = Number(fields[1]); + if (temperature < -100 || temperature > 200 || vibration < 0 || vibration > 100) { + process.exit(2); + } + const severity: "normal" | "warning" | "critical" = + temperature >= 80 || vibration >= 70 ? "critical" : + temperature >= 60 || vibration >= 40 ? "warning" : "normal"; + console.log(severity); + """), + ), + "cpp": ( + "main.cpp", + _source(""" + // Convert bounded sensor ticks using an explicit integer offset and gain. + #include + #include + int main() { + long long reading{}, offset{}, gain{}; + std::string extra; + if (!(std::cin >> reading >> offset >> gain) || (std::cin >> extra)) return 2; + if (reading < -10000 || reading > 10000 || offset < -10000 || + offset > 10000 || gain < 1 || gain > 100) return 2; + std::cout << (reading - offset) * gain << "\\n"; + return 0; + } + """), + ), + "c": ( + "main.c", + _source(""" + // XOR checksum for a bounded count followed by unsigned byte values. + #include + #include + #include + #include + #include + static int next_value(char **cursor, long *value) { + char *end; + errno = 0; + *value = strtol(*cursor, &end, 10); + if (end == *cursor || errno == ERANGE) return 0; + *cursor = end; + return 1; + } + int main(void) { + char buffer[8193]; + size_t length = fread(buffer, 1, sizeof(buffer) - 1, stdin); + if (ferror(stdin) || !feof(stdin) || memchr(buffer, 0, length)) return 2; + buffer[length] = 0; + char *cursor = buffer; + long count, value; + unsigned checksum = 0; + if (!next_value(&cursor, &count) || count < 0 || count > 16) return 2; + for (long index = 0; index < count; ++index) { + if (!next_value(&cursor, &value) || value < 0 || value > 255) return 2; + checksum ^= (unsigned)value; + } + while (isspace((unsigned char)*cursor)) ++cursor; + if (*cursor) return 2; + printf("%u\\n", checksum); + return 0; + } + """), + ), + "java": ( + "Main.java", + _source(""" + // Allocate a bounded capacity after reserving slots; reject invalid requests. + import java.util.Scanner; + public class Main { + private static long next(Scanner input) { + if (!input.hasNextLong()) throw new IllegalArgumentException(); + return input.nextLong(); + } + public static void main(String[] args) { + try (Scanner input = new Scanner(System.in)) { + long capacity = next(input), reserved = next(input), requested = next(input); + if (input.hasNext() || capacity < 0 || capacity > 1000000 || + reserved < 0 || reserved > capacity || requested < 0 || requested > 1000000) { + throw new IllegalArgumentException(); + } + long available = capacity - reserved; + long accepted = Math.min(available, requested); + System.out.println(accepted + "|" + (available - accepted)); + } catch (IllegalArgumentException error) { + System.exit(2); + } + } + } + """), + ), + "csharp": ( + "Program.cs", + _source(""" + // Decimal invoice; invariant formatting and explicit half-away-from-zero currency rounding. + using System.Globalization; + var fields = Console.In.ReadToEnd().Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + var culture = CultureInfo.InvariantCulture; + if (fields.Length != 3 || + !int.TryParse(fields[0], NumberStyles.None, culture, out var quantity) || + !decimal.TryParse(fields[1], NumberStyles.AllowDecimalPoint, culture, out var unitPrice) || + !decimal.TryParse(fields[2], NumberStyles.AllowDecimalPoint, culture, out var taxPercent) || + quantity > 1000 || unitPrice > 1000000m || taxPercent > 100m) return 2; + var subtotal = decimal.Round(quantity * unitPrice, 2, MidpointRounding.AwayFromZero); + var tax = decimal.Round(subtotal * taxPercent / 100m, 2, MidpointRounding.AwayFromZero); + Console.WriteLine(subtotal.ToString("F2", culture) + "|" + tax.ToString("F2", culture) + + "|" + (subtotal + tax).ToString("F2", culture)); + return 0; + """), + ), + "rust": ( + "main.rs", + _source(""" + // Apply a bounded inventory adjustment; never permit negative resulting stock. + use std::io::{self, Read}; + fn adjust(text: &str) -> Result { + let fields: Vec = text.split_whitespace() + .map(str::parse).collect::>().map_err(|_| ())?; + if fields.len() != 2 || !(0..=1000000).contains(&fields[0]) || + !(-1000000..=1000000).contains(&fields[1]) { return Err(()); } + let balance = fields[0] + fields[1]; + if !(0..=1000000).contains(&balance) { return Err(()); } + let state = if balance < 10 { "reorder" } else { "ok" }; + Ok(format!("{}|{}", balance, state)) + } + fn main() { + let mut input = String::new(); + if io::stdin().read_to_string(&mut input).is_err() { std::process::exit(2); } + match adjust(&input) { + Ok(output) => println!("{}", output), + Err(()) => std::process::exit(2), + } + } + """), + ), + "php": ( + "main.php", + _source(""" + 1000 || $unitCents > 100000 || $shippingCents > 100000) { exit(2); } + $subtotal = $quantity * $unitCents; + $discount = $quantity >= 10 ? intdiv($subtotal * 5 + 50, 100) : 0; + $total = $subtotal - $discount + ($quantity > 0 ? $shippingCents : 0); + echo $subtotal . '|' . $discount . '|' . $total . "\\n"; + """), + ), + "bash": ( + "main.sh", + _source(""" + # Count exact log severities; blank lines are ignored, unknown values are rejected. + ok=0 + warning=0 + error=0 + while IFS= read -r severity || [[ -n "$severity" ]]; do + case "$severity" in + OK) ((ok += 1)) ;; + WARN) ((warning += 1)) ;; + ERROR) ((error += 1)) ;; + '') continue ;; + *) exit 2 ;; + esac + done + printf '%s|%s|%s\\n' "$ok" "$warning" "$error" + """), + ), + "sql": ( + "main.sql", + _source(""" + -- Fresh in-memory order schema; test queries supply independently reviewed reports. + PRAGMA foreign_keys = ON; + CREATE TABLE products ( + sku TEXT PRIMARY KEY, unit_cents INTEGER NOT NULL CHECK (unit_cents >= 0) + ); + CREATE TABLE orders ( + id INTEGER PRIMARY KEY, + sku TEXT NOT NULL REFERENCES products(sku), + quantity INTEGER NOT NULL CHECK (quantity >= 0) + ); + INSERT INTO products VALUES ('sensor', 1200), ('cable', 500), ('spare', 0); + INSERT INTO orders VALUES (1, 'sensor', 2), (2, 'cable', 3), + (3, 'sensor', 1), (4, 'spare', 0); + """), + ), + "assembly": ( + "main.S", + _source(""" + # GNU Linux x86-64 C ABI: decode three illustrative alarm bits from a bounded input mask. + .section .rodata + scan_format: .string "%1[0-7] %c" + report_format: .string "%d|%d|%d\\n" + .text + .global main + .type main, @function + main: + subq $24, %rsp + leaq 8(%rsp), %rsi + leaq 12(%rsp), %rdx + leaq scan_format(%rip), %rdi + xorl %eax, %eax + call scanf@PLT + cmpl $1, %eax + jne invalid_input + movzbl 8(%rsp), %eax + subl $48, %eax + cmpl $7, %eax + ja invalid_input + movl %eax, %esi + andl $1, %esi + movl %eax, %edx + shrl $1, %edx + andl $1, %edx + movl %eax, %ecx + shrl $2, %ecx + andl $1, %ecx + leaq report_format(%rip), %rdi + xorl %eax, %eax + call printf@PLT + xorl %eax, %eax + jmp finish + invalid_input: + movl $2, %eax + finish: + addq $24, %rsp + ret + .size main, .-main + .section .note.GNU-stack,"",@progbits + """), + ), +} + +# Oracles are literal reviewed expectations, not values calculated by the target programs. +# Tuple fields: descriptive case name, standard input, expected standard output, expected exit code. +CASES: dict[str, tuple[tuple[str, str, str, int], ...]] = { + "python": ( + ("telemetry-window", "20,30,10\n", "3|60|10|30", 0), + ("zero-readings", "0,0\n", "2|0|0|0", 0), + ("signed-readings", "-5,5\n", "2|0|-5|5", 0), + ("empty-window", "", "0|0|NA|NA", 0), + ("range-boundaries", "-1000,1000\n", "2|0|-1000|1000", 0), + ("malformed-reading", "12,bad\n", "", 2), + ), + "javascript": ( + ("discounted-invoice", "3 1999 1000\n", "5997|600|5397", 0), + ("zero-quantity", "0 1999 1000\n", "0|0|0", 0), + ("full-discount", "2 2500 10000\n", "5000|5000|0", 0), + ("half-cent-rounding", "1 1 5000\n", "1|1|0", 0), + ("malformed-price", "2 nope 0\n", "", 2), + ("negative-quantity", "-1 100 0\n", "", 2), + ), + "typescript": ( + ("normal-sample", "25 10\n", "normal", 0), + ("warning-boundary", "60 0\n", "warning", 0), + ("critical-boundary", "20 70\n", "critical", 0), + ("zero-sample", "0 0\n", "normal", 0), + ("maximum-sample", "200 100\n", "critical", 0), + ("malformed-sample", "x 10\n", "", 2), + ), + "cpp": ( + ("calibrated-reading", "1200 200 2\n", "2000", 0), + ("zero-adjustment", "200 200 3\n", "0", 0), + ("negative-adjustment", "-100 100 2\n", "-400", 0), + ("bounded-maximum", "10000 -10000 100\n", "2000000", 0), + ("malformed-reading", "x 0 1\n", "", 2), + ("invalid-gain", "100 0 0\n", "", 2), + ), + "c": ( + ("packet-checksum", "3 16 32 255\n", "207", 0), + ("empty-packet", "0\n", "0", 0), + ("maximum-byte", "1 255\n", "255", 0), + ("cancelling-bytes", "4 12 12 0 0\n", "0", 0), + ("malformed-byte", "2 1 bad\n", "", 2), + ("out-of-range-byte", "1 256\n", "", 2), + ), + "java": ( + ("capacity-allocation", "100 20 50\n", "50|30", 0), + ("zero-capacity", "0 0 0\n", "0|0", 0), + ("malformed-capacity", "many 0 1\n", "", 2), + ("exact-capacity", "100 0 100\n", "100|0", 0), + ("demand-over-capacity", "100 20 120\n", "80|0", 0), + ("invalid-reservation", "100 101 1\n", "", 2), + ), + "csharp": ( + ("taxed-invoice", "3 19.99 8.25\n", "59.97|4.95|64.92", 0), + ("zero-quantity", "0 19.99 8.25\n", "0.00|0.00|0.00", 0), + ("half-cent-tax", "1 0.05 10\n", "0.05|0.01|0.06", 0), + ("tax-free-invoice", "2 7.50 0\n", "15.00|0.00|15.00", 0), + ("bounded-maximum", "1 1000000 100\n", "1000000.00|1000000.00|2000000.00", 0), + ("malformed-price", "1 money 8\n", "", 2), + ), + "rust": ( + ("inventory-withdrawal", "20 -5\n", "15|ok", 0), + ("zero-inventory", "0 0\n", "0|reorder", 0), + ("reorder-boundary", "9 1\n", "10|ok", 0), + ("empty-inventory", "10 -10\n", "0|reorder", 0), + ("reject-overdraft", "5 -6\n", "", 2), + ("malformed-adjustment", "bad 1\n", "", 2), + ), + "php": ( + ("ordinary-quote", "3 1200 500\n", "3600|0|4100", 0), + ("bulk-discount", "10 1000 500\n", "10000|500|10000", 0), + ("zero-order", "0 1000 500\n", "0|0|0", 0), + ("half-cent-discount", "10 1 0\n", "10|1|9", 0), + ("bounded-maximum", "1000 100000 100000\n", "100000000|5000000|95100000", 0), + ("malformed-price", "2 nope 1\n", "", 2), + ), + "bash": ( + ("mixed-log", "OK\nWARN\nERROR\nWARN\n", "1|2|1", 0), + ("empty-log", "", "0|0|0", 0), + ("blank-lines", "\n\n", "0|0|0", 0), + ("missing-final-newline", "ERROR", "0|0|1", 0), + ("all-ok", "OK\nOK\n", "2|0|0", 0), + ("unknown-severity", "OK\nMAYBE\n", "", 2), + ), + "sql": ( + ( + "joined-revenue", + "SELECT SUM(o.quantity * p.unit_cents) FROM orders o JOIN products p USING (sku);", + "5100", + 0, + ), + ( + "ordered-product-report", + "SELECT p.sku, COALESCE(SUM(o.quantity * p.unit_cents), 0) " + "FROM products p LEFT JOIN orders o USING (sku) GROUP BY p.sku ORDER BY p.sku;", + "cable|1500\nsensor|3600\nspare|0", + 0, + ), + ( + "missing-product-zero", + "SELECT COALESCE(SUM(quantity), 0) FROM orders WHERE sku = 'missing';", + "0", + 0, + ), + ("zero-quantity-order", "SELECT COUNT(*) FROM orders WHERE quantity = 0;", "1", 0), + ("reject-negative-quantity", "INSERT INTO orders VALUES (5, 'sensor', -1);", "", 1), + ("malformed-query", "SELECT FROM orders;", "", 1), + ), + "assembly": ( + ("no-alarm", "0\n", "0|0|0", 0), + ("first-alarm", "1\n", "1|0|0", 0), + ("third-alarm", "4\n", "0|0|1", 0), + ("all-alarm-bits", "7\n", "1|1|1", 0), + ("out-of-range-mask", "8\n", "", 2), + ("malformed-mask", "bad\n", "", 2), + ), +} + + +SYNTAX_FAILURES: dict[str, dict[str, str | int]] = { + "python": {"content": "def broken(:\n pass\n", "line": 1}, + "javascript": {"content": "const broken = ;\n", "line": 1}, + "typescript": {"content": "const broken: number = ;\n", "line": 1}, + "cpp": {"content": "int main() {\n int value = ;\n}\n", "line": 2}, + "c": {"content": "int main(void) {\n int value = ;\n}\n", "line": 2}, + "java": { + "content": "public class Main {\n public static void main(String[] args) {\n int value = ;\n }\n}\n", + "line": 3, + }, + "csharp": {"content": "using System;\nConsole.WriteLine(;\n", "line": 2}, + "rust": {"content": "fn main() {\n let value = ;\n}\n", "line": 2}, + "php": {"content": " Project: + """Build one fresh approved project; unknown languages fail without choosing a fallback. + + ``language`` selects a fixed worker adapter. Returned sources and six literal + cases are independent copies; callers may replace them for negative tests + without changing another scenario. Benchmarks repeat once and fuzzing is + disabled here, keeping these deterministic CLI oracles distinct from the + existing dedicated fuzz/mutation/location suites. + """ + entry, content = PROGRAMS[language] + return Project( + title=f"Practical {language} standard-library CLI validation", + language=language, + entry=entry, + files=[{"name": entry, "content": content}], + cases=[ + {"name": name, "stdin": stdin, "stdout": stdout, "exit_code": exit_code} + for name, stdin, stdout, exit_code in CASES[language] + ], + requirements=( + "Synthetic bounded CLI demonstration. Preserve the approved stdout and exit-code " + "oracles, reject invalid input explicitly, and use no network or external packages." + ), + benchmark_repeats=1, + fuzz_cases=0, + mutation=False, + ) diff --git a/scripts/compiler_acceptance.py b/scripts/compiler_acceptance.py new file mode 100644 index 0000000..8430388 --- /dev/null +++ b/scripts/compiler_acceptance.py @@ -0,0 +1,656 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# classes: Evidence@L305 +# functions: require(condition, message)@L44; source_digest(text)@L50; source_line_count(text)@L55; validate_coverage_completion(coverage, project)@L60; validate_locations(result, project)@L93; validate_positive(result, project)@L106; negative_project(project, kind)@L207; validate_negative(result, project, kind)@L226; __init__(self, output, provenance)@L308; record(self, language, kind, project, result, error)@L315; save(self)@L337; broker_scenarios(evidence)@L364; anchor_scenarios(evidence)@L385; private_scenarios(evidence)@L412; provenance(runtime, expected_source, lane)@L516; toolchain_versions()@L570; main()@L615 +# variables: RELEASE_SOURCE@L40; RELEASE_WORKER_SHA256@L41; wrapped@L62; cases@L64; actual@L69; expected@L69; wrapped@L85; expected_status@L86; file@L95; sources@L95; item@L97; line@L98; name@L98; compilation@L109; checks@L116; observed@L118; actual@L120; expected@L120; oracle@L130; case@L131; benchmark@L135; samples@L136; value@L139; coverage@L145; entry@L148; content@L149; file@L149; measured@L155; name@L155; file@L158; source@L158; line@L163; executed@L165; missing@L165; summary@L171; covered@L172; extra@L180; first@L184; data@L209; replacement@L215; expected_oracle@L237; case@L238; checks@L241; line@L259; item@L265; case@L276; exit_code@L283; marker@L288; row@L319; key@L333; record@L339; temporary@L340; lines@L343; row@L351; checks@L352; lines@L356; language@L366; baseline@L367; kind@L368; project@L371; result@L372; error@L373; result@L375; error@L381; language@L387; missing_line@L388; project@L388; unused_line@L388; result@L389; error@L390; result@L392; item@L403; error@L408; credentials@L414; key@L415; directory@L416; app@L417; client@L422; jobs@L430; language@L431; project@L432; result@L433; error@L434; response@L436; revision@L438; response@L439; job@L441; deadline@L442; response@L444; report@L446; result@L451; exported@L453; html@L458; error@L465; archives@L469; response@L484; job@L490; result_sha256@L490; response@L491; exported@L498; runtime@L524; source@L529; dirty@L531; worker@L540; command@L545; image@L546; commands@L572; program@L586; name@L591; directory@L592; root@L593; argv@L595; argv@L597; code@L599; output@L599; versions@L601; value@L607; parser@L617; args@L622; evidence@L623; language@L644 +# parameters: condition@L44; message@L44; text@L50; text@L55; coverage@L60; project@L60; project@L93; result@L93; project@L106; result@L106; kind@L207; project@L207; kind@L226; project@L226; result@L226; output@L308; provenance@L308; self@L308; error@L316; kind@L316; language@L316; project@L316; result@L316; self@L316; self@L337; evidence@L364; evidence@L385; evidence@L412; expected_source@L516; lane@L516; runtime@L516 +# + +"""Jason Stys: real hosted compiler scenarios, independent oracles and private history. + +Only main() is hosted-only. Pure evidence validators can run without Docker. +Source maps in this file and docs/code-index.md describe exact declarations. +""" + +import argparse +import ast +import asyncio +import base64 +import hashlib +import json +import math +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from uuid import uuid4 + +from fastapi.testclient import TestClient + +import studio +from examples.language_locations import SAMPLES, location_fixture +from examples.practical_projects import RUNTIME_FAILURES, SYNTAX_FAILURES, practical_project +from studio import execution +from studio.models import LANGUAGES, Project +from studio.private_api import create_private_app + +RELEASE_SOURCE = "eb9b1cd0c3e3fd1e70e397edeffb059073464f74" +RELEASE_WORKER_SHA256 = "750b665e545b7cf0806d4c493db86174f7f4e7ab2218aa239718e652ccdce158" + + +def require(condition: bool, message: str) -> None: + """Fail explicitly even under optimized Python; never downgrade a missing tool to success.""" + if not condition: + raise ValueError(message) + + +def source_digest(text: str) -> str: + """Bind evidence to exact UTF-8 bytes, including its original newlines.""" + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def source_line_count(text: str) -> int: + """Use the app's original-source convention, retaining an empty terminal line for EOF opcodes.""" + return len(text.replace("\r\n", "\n").replace("\r", "\n").split("\n")) + + +def validate_coverage_completion(coverage: dict, project: Project) -> None: + """Preserve the documented Coverlet nonzero-exit limitation without inventing full coverage.""" + wrapped = False + if project.language != "python": + cases = coverage.get("cases", []) + require( + coverage.get("requested_cases") == len(project.cases) and len(cases) == len(project.cases), + "Missing instrumented case evidence", + ) + for expected, actual in zip(project.cases, cases, strict=True): + require( + actual.get("name") == expected.name + and actual.get("expected_exit_code") == expected.exit_code + and not actual.get("timeout"), + "Instrumented case identity/deadline mismatch", + ) + if actual.get("exit_code") != expected.exit_code: + require( + project.language == "csharp" + and coverage.get("tool") == "Coverlet 8.0.1" + and expected.exit_code != 0 + and actual.get("exit_code") == 1 + and "Generating report" in actual.get("collector_messages", ""), + "Unexpected instrumented execution failure", + ) + wrapped = True + expected_status = "partial" if wrapped else "measured" + require( + coverage.get("status") == expected_status, + "Coverage completion status disagrees with actual collector exits", + ) + + +def validate_locations(result: dict, project: Project) -> None: + """Accept only reported in-range references to the supplied source snapshot.""" + sources = {file.name: file.content for file in project.files if file.encoding == "text"} + require(not result.get("diagnostics_truncated"), "Diagnostic evidence was truncated") + for item in result.get("diagnostics", []): + name, line = item.get("file"), item.get("line") + require(name in sources, "Diagnostic references an unknown file") + require( + type(line) is int and 0 < line <= source_line_count(sources[name]), "Diagnostic line is invalid" + ) + require(item.get("source_sha256") == source_digest(sources[name]), "Diagnostic source hash mismatch") + + +def validate_positive(result: dict, project: Project) -> None: + """Require actual compilation, every approved output, measurements and source-bound coverage.""" + require(result.get("tool_status") == "completed", "Worker did not complete") + compilation = result.get("compilation", {}) + require( + compilation.get("exit_code") == 0 + and not compilation.get("timeout") + and not compilation.get("output_truncated"), + "Compilation failed or was truncated", + ) + checks = result.get("checks", {}) + require(checks.get("passed") == len(project.cases) and checks.get("failed") == 0, "Approved cases failed") + observed = checks.get("cases", []) + require(len(observed) == len(project.cases), "Missing or extra execution results") + for expected, actual in zip(project.cases, observed, strict=True): + require( + actual.get("name") == expected.name and actual.get("passed") is True, + "Case identity/pass mismatch", + ) + require(actual.get("exit_code") == expected.exit_code, "Wrong target exit status") + require(actual.get("stdout", "").strip() == expected.stdout.strip(), "Wrong target output") + require( + not actual.get("timeout") and not actual.get("output_truncated"), "Execution evidence incomplete" + ) + oracle = hashlib.sha256( + json.dumps([case.model_dump() for case in project.cases], sort_keys=True).encode() + ).hexdigest() + require(result.get("oracle_hash") == oracle, "Approved oracle identity changed") + require(result.get("environment", {}).get("language") == project.language, "Wrong language adapter") + benchmark = result.get("benchmark", {}) + samples = benchmark.get("samples_ms", []) + require(len(samples) == len(project.cases) * project.benchmark_repeats, "Missing benchmark repetitions") + require( + all(type(value) in (int, float) and math.isfinite(value) and value >= 0 for value in samples), + "Invalid timing sample", + ) + require( + type(benchmark.get("max_rss_kib")) is int and benchmark["max_rss_kib"] > 0, "Missing measured RSS" + ) + coverage = result.get("coverage", {}) + validate_coverage_completion(coverage, project) + require(not coverage.get("locations_truncated"), "Coverage locations were truncated") + entry = coverage.get("files", {}).get(project.entry, {}) + content = next(file.content for file in project.files if file.name == project.entry) + require( + entry.get("status") == "measured" and entry.get("executed_lines"), "Entry has no executed coverage" + ) + require(entry.get("source_sha256") == source_digest(content), "Coverage source hash mismatch") + require(not entry.get("truncated"), "Entry coverage was truncated") + for name, measured in coverage.get("files", {}).items(): + if measured.get("status") != "measured": + continue + source = next((file.content for file in project.files if file.name == name), None) + require( + source is not None and measured.get("source_sha256") == source_digest(source), + "Unknown measured source", + ) + for line in measured.get("executed_lines", []) + measured.get("missing_lines", []): + require(type(line) is int and 0 < line <= source_line_count(source), "Coverage line out of range") + executed, missing = measured.get("executed_lines", []), measured.get("missing_lines", []) + require( + len(executed) == len(set(executed)) and len(missing) == len(set(missing)), + "Duplicate coverage line", + ) + require(not set(executed).intersection(missing), "Coverage line is both executed and missing") + summary = measured.get("summary", {}) + covered = summary.get("covered_lines") + require( + type(covered) is int + and covered >= 0 + and summary.get("missing_lines") == len(missing) + and summary.get("num_statements") == covered + len(missing), + "Coverage statement counts disagree", + ) + extra = len(executed) - covered + if extra and project.language == "python": + # coverage.py retains the module docstring trace event, but excludes + # that nonstatement from its executable-statement denominator. + first = ast.parse(source).body[0] + require( + extra == 1 + and isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + and first.lineno in executed, + "Unexplained Python trace/statement count difference", + ) + else: + require(extra == 0, "Coverage counts disagree with retained locations") + if project.language != "python": + require( + result.get("unused", {}).get("status") == "completed", "Native unused-code tool did not complete" + ) + else: + require( + result.get("static", {}).get("files", {}).get(project.entry, {}).get("parsed") is True, + "Python AST was not parsed", + ) + validate_locations(result, project) + + +def negative_project(project: Project, kind: str) -> Project: + """Make independent negative controls; never alter an approved production revision.""" + data = project.model_dump() + data["cases"] = [data["cases"][0]] + data["cases"][0]["name"] = kind + if kind == "wrong-oracle": + data["cases"][0]["stdout"] = "INTENTIONALLY INCORRECT EXPECTED OUTPUT" + else: + replacement = ( + SYNTAX_FAILURES[project.language]["content"] + if kind == "syntax-error" + else RUNTIME_FAILURES[project.language] + ) + data["files"] = [{"name": project.entry, "content": replacement, "encoding": "text"}] + data["cases"][0]["stdout"] = "UNREACHABLE SUCCESS OUTPUT" + data["cases"][0]["exit_code"] = 0 + return Project.model_validate(data) + + +def validate_negative(result: dict, project: Project, kind: str) -> None: + """A deliberately bad target/oracle must be detected, rather than making CI falsely green.""" + validate_locations(result, project) + require( + not result.get("compilation", {}).get("timeout") + and not result.get("compilation", {}).get("output_truncated"), + "Negative compilation was interrupted or truncated", + ) + require( + result.get("environment", {}).get("language") == project.language, "Wrong negative-control adapter" + ) + expected_oracle = hashlib.sha256( + json.dumps([case.model_dump() for case in project.cases], sort_keys=True).encode() + ).hexdigest() + require(result.get("oracle_hash") == expected_oracle, "Negative-control oracle identity changed") + checks = result.get("checks", {}) + if kind == "syntax-error": + require( + result.get("tool_status") == "compile-error", "Invalid source was not rejected at compilation" + ) + require(result.get("compilation", {}).get("exit_code") not in (None, 0), "Missing compiler failure") + require( + checks.get("passed") == 0 and checks.get("failed") == 0 and not checks.get("cases"), + "Uncompiled source was executed", + ) + require( + result.get("coverage", {}).get("status") == "blocked", + "Failed compilation claimed measured coverage", + ) + require( + result.get("unused", {}).get("status") == "blocked", + "Failed compilation claimed unused-code completion", + ) + line = SYNTAX_FAILURES[project.language]["line"] + require( + any( + item.get("file") == project.entry + and item.get("line") == line + and item.get("stage") == "compilation" + for item in result.get("diagnostics", []) + ), + "Compiler failure lacks its independently specified exact source line", + ) + else: + require(result.get("tool_status") == "completed", "Negative target did not reach execution") + require(result.get("compilation", {}).get("exit_code") == 0, "Runtime/oracle control did not compile") + require( + checks.get("passed") == 0 and checks.get("failed") == 1 and len(checks.get("cases", [])) == 1, + "Bad execution/oracle was not rejected", + ) + case = checks["cases"][0] + require(case.get("name") == project.cases[0].name, "Wrong negative-control case identity") + require( + case.get("passed") is False and not case.get("timeout") and not case.get("output_truncated"), + "Negative result is not a completed observed failure", + ) + if kind == "runtime-error": + exit_code = 1 if project.language in {"python", "javascript", "typescript", "sql"} else 7 + require( + case.get("exit_code") == exit_code, + "Faulting program did not produce its authored failure exit", + ) + marker = "missing_table" if project.language == "sql" else "Synthetic runtime failure" + if project.language in {"python", "javascript", "typescript", "sql"}: + require( + marker in case.get("stderr", ""), + "Runtime diagnostic does not identify the authored fault", + ) + else: + require( + case.get("exit_code") == project.cases[0].exit_code, + "Wrong-oracle control also had a runtime fault", + ) + require( + case.get("stdout", "").strip() == practical_project(project.language).cases[0].stdout.strip(), + "Wrong-oracle control also had an implementation/output fault", + ) + + +class Evidence: + """Incrementally preserve synthetic results and fail closed if any language or scenario is missing.""" + + def __init__(self, output: Path, provenance: dict): + """output is a dedicated CI report directory; provenance identifies tested code and worker.""" + self.output, self.provenance = output, provenance + self.rows: list[dict] = [] + self.errors: list[str] = [] + output.mkdir(parents=True, exist_ok=True) + + def record( + self, language: str, kind: str, project: Project, result: dict, error: str | None = None + ) -> None: + """Persist before continuing so a later tool failure cannot erase earlier evidence.""" + row = { + "language": language, + "kind": kind, + "acceptance": "failed" if error else "passed", + "project": project.model_dump(), + "result": result, + "error": error, + } + self.rows.append(row) + if error: + self.errors.append(language + "/" + kind + ": " + error) + self.save() + print( + "COMPILER_SCENARIO=" + + json.dumps({key: row[key] for key in ("language", "kind", "acceptance", "error")}), + flush=True, + ) + + def save(self) -> None: + """Write a replace-atomic JSON receipt and a small human-readable result table.""" + record = {"schema": 1, "provenance": self.provenance, "scenarios": self.rows, "errors": self.errors} + temporary = self.output / "results.tmp" + temporary.write_text(json.dumps(record, indent=2, allow_nan=False) + "\n", encoding="utf-8") + temporary.replace(self.output / "results.json") + lines = [ + "# Hosted compiler and execution validation", + "", + "Runtime source: `" + self.provenance["runtime_source"] + "`", + "", + "| Language | Scenario | Acceptance | Target passed / failed | Coverage |", + "|---|---|---|---|---|", + ] + for row in self.rows: + checks = row["result"].get("checks", {}) + lines.append( + f"| {row['language']} | {row['kind']} | {row['acceptance']} | {checks.get('passed', '—')} / {checks.get('failed', '—')} | {row['result'].get('coverage', {}).get('status', '—')} |" + ) + lines += [ + "", + "Negative-control acceptance means the intentionally invalid target/oracle was correctly rejected. Completion alone is not a correctness pass. Timings include process startup and are not cross-language performance rankings.", + "", + ] + (self.output / "summary.md").write_text("\n".join(lines), encoding="utf-8") + + +async def broker_scenarios(evidence: Evidence) -> None: + """Run positive, incorrect-oracle, compiler-error and runtime-error controls for all adapters.""" + for language in LANGUAGES: + baseline = practical_project(language) + for kind in ("positive", "wrong-oracle", "syntax-error", "runtime-error"): + if kind == "syntax-error" and language not in SYNTAX_FAILURES: + continue # SQLite parses/runs the schema at execution, not a separate compiler phase. + project = baseline if kind == "positive" else negative_project(baseline, kind) + result: dict = {} + error = None + try: + result = await execution.run(project, local_only=True) + if kind == "positive": + validate_positive(result, project) + else: + validate_negative(result, project, kind) + except (OSError, ValueError, KeyError, TypeError) as failure: + error = str(failure)[:400] + evidence.record(language, kind, project, result, error) + + +async def anchor_scenarios(evidence: Evidence) -> None: + """Retain independent exact unused/uncovered anchors alongside realistic multi-case execution.""" + for language in SAMPLES: + project, unused_line, missing_line = location_fixture(language) + result: dict = {} + error = None + try: + result = await execution.run(project, local_only=True) + validate_positive(result, project) + require( + missing_line in result["coverage"]["files"][project.entry]["missing_lines"], + "Authored uncovered anchor was not measured as missing", + ) + require( + any( + item.get("file") == project.entry + and item.get("line") == unused_line + and item.get("stage") == "unused-code" + for item in result.get("diagnostics", []) + ), + "Authored unused anchor was not detected", + ) + except (OSError, ValueError, KeyError, TypeError) as failure: + error = str(failure)[:400] + evidence.record(language, "native-location", project, result, error) + + +def private_scenarios(evidence: Evidence) -> None: + """Exercise realistic programs through real authenticated account API and encrypted history.""" + credentials = {"alias": "synthetic_compiler_ci", "password": "fictional-compiler-ci-password-2026"} + key = {"key": base64.b64encode(b"k" * 32).decode(), "salt": base64.b64encode(b"s" * 16).decode()} + with tempfile.TemporaryDirectory(prefix="code-flow-practical-account-") as directory: + app = create_private_app(Path(directory) / "accounts", 8090) + with TestClient( + app, + base_url="http://127.0.0.1:8090", + headers={"X-CFS-Request": "v1", "Content-Type": "application/json"}, + ) as client: + require( + client.post("/private/register", json=credentials).status_code == 200, + "Synthetic registration failed", + ) + require( + client.post("/private/unlock", json=key).status_code == 200, "Synthetic vault unlock failed" + ) + jobs = [] + for language in LANGUAGES: + project = practical_project(language) + result: dict = {} + error = None + try: + response = client.post("/private/api/revisions", json=project.model_dump()) + require(response.status_code == 200, "Revision save failed") + revision = response.json()["id"] + response = client.post(f"/private/api/revisions/{revision}/run", json={}) + require(response.status_code == 200, "Compiler job queue failed") + job = response.json()["id"] + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + response = client.get("/private/api/runs/" + job) + require(response.status_code == 200, "Compiler job poll failed") + report = response.json() + if report["status"] not in {"queued", "running"}: + break + time.sleep(0.5) + require(report["status"] == "completed", "Private job did not complete") + result = report["result"] + validate_positive(result, project) + exported = client.get(f"/private/api/runs/{job}/report?format=json") + require( + exported.status_code == 200 and exported.json()["result"] == result, + "JSON export differs from observed execution", + ) + html = client.get(f"/private/api/runs/{job}/report?format=html") + require( + html.status_code == 200 and "text/html" in html.headers.get("content-type", ""), + "HTML report export failed", + ) + jobs.append((job, source_digest(json.dumps(result, sort_keys=True, allow_nan=False)))) + except (OSError, ValueError, KeyError, TypeError) as failure: + error = str(failure)[:400] + evidence.record(language, "private-api", project, result, error) + require(client.post("/private/lock", json={}).status_code == 200, "Private history seal failed") + require(not app.state.manager.active.exists(), "Plaintext workspace retained after lock") + archives = list(app.state.manager.archives.glob("*.cfsvault")) + require(len(archives) == 1 and archives[0].stat().st_size > 0, "Encrypted archive missing") + require(client.post("/private/logout", json={}).status_code == 200, "Synthetic logout failed") + require( + client.get("/private/api/revisions").status_code == 401, + "History remained visible after logout", + ) + require( + client.post("/private/login", json=credentials).status_code == 200, + "Existing synthetic account login failed", + ) + require( + client.post("/private/unlock", json=key).status_code == 200, + "Saved encrypted history did not reopen", + ) + response = client.get("/private/api/revisions") + require( + response.status_code == 200 and len(response.json()) == len(LANGUAGES), + "History lost saved revisions", + ) + require(len(jobs) == len(LANGUAGES), "Missing successful private compiler jobs") + for job, result_sha256 in jobs: + response = client.get("/private/api/runs/" + job) + require( + response.status_code == 200 + and source_digest(json.dumps(response.json()["result"], sort_keys=True, allow_nan=False)) + == result_sha256, + "Encrypted report identity was not retained", + ) + exported = client.get(f"/private/api/runs/{job}/report?format=json") + require( + exported.status_code == 200 + and source_digest(json.dumps(exported.json()["result"], sort_keys=True, allow_nan=False)) + == result_sha256, + "Reopened JSON export changed execution evidence", + ) + require( + client.post("/private/lock", json={}).status_code == 200, "Final synthetic vault lock failed" + ) + evidence.provenance["private_history"] = { + "login_after_logout": "passed", + "sealed_and_reopened_reports": len(jobs), + "plaintext_removed": not app.state.manager.active.exists(), + } + evidence.save() + + +def provenance(runtime: Path, expected_source: str, lane: str) -> dict: + """Refuse accidental local execution or source/import/image substitutions before running programs.""" + require( + os.environ.get("GITHUB_ACTIONS") == "true" + and os.environ.get("RUNNER_ENVIRONMENT") == "github-hosted" + and sys.platform == "linux", + "Real compiler validation runs only on GitHub-hosted Linux; local Docker is unchanged", + ) + runtime = runtime.resolve(strict=True) + require( + Path(studio.__file__).resolve().parent.parent == runtime, + "Controller imported from the wrong checkout", + ) + source = subprocess.check_output(["git", "-C", str(runtime), "rev-parse", "HEAD"], text=True).strip() # noqa: S603,S607 -- fixed CI git operation. + require(source == expected_source and len(source) == 40, "Runtime source identity mismatch") + dirty = subprocess.check_output( # noqa: S603 -- fixed read-only CI git operation. + ["git", "-C", str(runtime), "status", "--porcelain", "--untracked-files=no"], # noqa: S607 + text=True, + ).strip() + require(not dirty, "Runtime tracked source is dirty") + require( + studio.__version__ == "0.4.2" and execution.IMAGE == "code-flow-worker:0.4.2", + "Wrong product/worker version", + ) + worker = hashlib.sha256((runtime / "runner/node_checks.cjs").read_bytes()).hexdigest() + if lane == "release-0.4.2": + require( + source == RELEASE_SOURCE and worker == RELEASE_WORKER_SHA256, "Published release baseline changed" + ) + command = [execution.DOCKER, "image", "inspect", execution.IMAGE, "--format", "{{.Id}}"] + image = subprocess.check_output(command, text=True).strip() # noqa: S603 -- trusted broker-selected Docker executable. + require(image.startswith("sha256:") and len(image) == 71, "Worker image identity missing") + return { + "lane": lane, + "runtime_source": source, + "runtime_version": studio.__version__, + "worker_node_sha256": worker, + "worker_image_id": image, + "validation_source": os.environ.get("GITHUB_SHA"), + "run_id": os.environ.get("GITHUB_RUN_ID"), + "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "host": "GitHub-hosted Linux", + "toolchain_caveat": "The image was freshly built from the identified source. Mutable upstream base tags/APT packages mean it is not asserted identical to a historical image.", + "isolation": { + "network": "none", + "user": "65534:65534", + "read_only": True, + "memory_mib": 1024, + "cpus": 2, + "pids": 128, + }, + } + + +async def toolchain_versions() -> dict: + """Read actual tool versions inside the same restricted worker, with no packages/network.""" + commands = { + "python": ["python", "--version"], + "javascript": ["node", "--version"], + "typescript": ["tsc", "--version"], + "cpp": ["g++", "--version"], + "c": ["gcc", "--version"], + "java": ["javac", "-version"], + "csharp": ["dotnet", "--version"], + "rust": ["rustc", "--version"], + "php": ["php", "--version"], + "bash": ["bash", "--version"], + "sql": ["sqlite3", "--version"], + "assembly": ["as", "--version"], + } + program = ( + "import json,subprocess,sys\nsys.path.insert(0,'/opt')\nfrom worker import prepare_runtime\nprepare_runtime()\ncommands=" + + repr(commands) + + "\nversions={}\nfor name,argv in commands.items():\n result=subprocess.run(argv,check=True,capture_output=True,text=True,timeout=10)\n versions[name]=(result.stdout or result.stderr).splitlines()[0]\nprint(json.dumps(versions))\n" + ) + name = "code-flow-toolchain-" + uuid4().hex + with tempfile.TemporaryDirectory(prefix="code-flow-toolchain-") as directory: + root = Path(directory) + (root / "job.json").write_text("{}", encoding="utf-8") + argv = execution.container_arguments(root, name) + argv.insert(-1, "--entrypoint=python") + argv += ["-c", program] + try: + code, output = await execution.command(argv, timeout=90, output_limit=16384) + require(code == 0, "Worker tool-version probe failed") + versions = json.loads(output) + require( + isinstance(versions, dict) and set(versions) == set(LANGUAGES), + "Incomplete real toolchain manifest", + ) + require( + all(isinstance(value, str) and 0 < len(value) <= 500 for value in versions.values()), + "Invalid tool-version evidence", + ) + return versions + finally: + await execution.command([execution.DOCKER, "rm", "--force", name], timeout=10, output_limit=4096) + + +def main() -> None: + """Run the hosted acceptance suite and retain failed evidence without converting it to a pass.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runtime-root", type=Path, required=True) + parser.add_argument("--runtime-source", required=True) + parser.add_argument("--lane", choices=["release-0.4.2", "current"], required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + evidence = Evidence(args.output.resolve(), provenance(args.runtime_root, args.runtime_source, args.lane)) + try: + evidence.provenance["toolchain"] = asyncio.run(toolchain_versions()) + asyncio.run(broker_scenarios(evidence)) + asyncio.run(anchor_scenarios(evidence)) + private_scenarios(evidence) + require( + len(evidence.rows) == len(LANGUAGES) * 4 + len(SYNTAX_FAILURES) + len(SAMPLES), + "Incomplete scenario matrix", + ) + require(not evidence.errors, "Real compiler scenario failures: " + "; ".join(evidence.errors)) + evidence.provenance["acceptance"] = "passed" + print( + "COMPILER_ACCEPTANCE=" + + json.dumps( + { + "lane": args.lane, + "runtime_source": args.runtime_source, + "languages": list(LANGUAGES), + "scenarios": len(evidence.rows), + "positive_cases_per_flow": sum( + len(practical_project(language).cases) for language in LANGUAGES + ), + "acceptance": "passed", + } + ), + flush=True, + ) + finally: + evidence.save() + + +if __name__ == "__main__": + main() diff --git a/scripts/released_service_probe.py b/scripts/released_service_probe.py new file mode 100644 index 0000000..e62abc7 --- /dev/null +++ b/scripts/released_service_probe.py @@ -0,0 +1,734 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# classes: Evidence@L117; Service@L421 +# functions: require(condition, message)@L72; strict_json(body)@L78; pairs(items)@L81; constant(_value)@L86; hosted_output(output)@L92; __init__(self, output)@L120; save(self)@L134; stage(self, stage)@L141; bounded_body(response, maximum)@L147; api_json(client, path)@L156; release_assets(release, tag)@L164; download_url(url, initial)@L216; download(client, row, target)@L240; preflight_tar(stream)@L280; unpack(archive, destination)@L316; service_environment(home, token, nonce)@L392; request_json(client, method, path, data)@L410; __init__(self, executable, data, home)@L424; ready(self)@L437; stop(self)@L471; validate_export(exported, result, revision)@L484; exercise(service, data, evidence)@L536; restored(service, data, private)@L607; canonical_digest(value)@L640; cleanup_owned(services, temporary)@L647; main()@L670 +# variables: SOURCE@L42; WORKER_SHA256@L43; VERSION@L44; RELEASE_ID@L45; REPOSITORY@L46; API@L47; PAYLOAD@L48; PAYLOAD_ID@L49; PAYLOAD_BYTES@L50; PAYLOAD_SHA256@L51; MAX_ARCHIVE@L52; MAX_EXPANDED@L53; MAX_MEMBER@L54; MAX_RESPONSE@L55; EXPECTED_LANGUAGES@L56; result@L82; workspace@L102; selected@L104; temporary@L135; body@L149; chunk@L150; response@L159; rows@L182; row@L184; row@L188; row@L189; row@L190; found@L193; row@L193; maximum@L201; name@L201; row@L202; expected@L211; prefix@L219; parsed@L221; initial@L242; url@L251; started@L252; _@L253; response@L256; url@L258; digest@L262; measured@L262; stream@L263; chunk@L264; measured@L265; expanded@L282; headers@L282; raw@L283; header@L285; expanded@L286; expanded@L290; chunk@L292; expanded@L293; headers@L296; number@L298; size@L300; kind@L301; limit@L303; remaining@L305; chunk@L307; remaining@L311; expanded@L312; expanded@L319; seen@L319; stream@L320; package@L324; index@L325; member@L325; name@L326; parts@L329; c@L337; part@L338; key@L341; expanded@L346; selected@L348; source@L353; measured@L355; output@L356; chunk@L357; measured@L358; resources@L363; metadata@L364; value@L366; worker@L382; service@L387; config@L394; response@L416; output@L428; deadline@L439; source@L441; body@L442; matches@L444; origin@L447; candidate@L448; health@L456; key@L488; value@L488; history@L491; snapshot@L504; attempt@L518; actual@L519; key@L526; alias@L539; password@L539; key@L540; client@L544; stored@L549; language@L550; project@L552; revision@L556; identifier@L557; job@L563; deadline@L565; result@L567; exported@L574; digest@L580; client@L609; history@L617; row@L620; row@L623; result@L624; exported@L625; failed@L649; service@L650; failed@L660; service@L661; parser@L672; args@L676; output@L677; evidence@L678; temporary@L679; services@L681; client@L683; payload@L684; sidecar@L684; archive@L688; checksum@L688; digest@L689; executable@L696; data@L698; first@L700; private@L703; second@L706; row@L715 +# parameters: condition@L72; message@L72; body@L78; items@L81; _value@L86; output@L92; output@L120; self@L120; self@L134; public@L141; self@L141; stage@L141; maximum@L147; response@L147; client@L156; path@L156; release@L164; tag@L164; initial@L216; url@L216; client@L240; row@L240; target@L240; stream@L280; archive@L316; destination@L316; home@L392; nonce@L392; token@L392; client@L410; data@L410; method@L410; path@L410; data@L424; executable@L424; home@L424; self@L424; self@L437; self@L471; exported@L484; result@L484; revision@L484; data@L536; evidence@L536; service@L536; data@L607; private@L607; service@L607; value@L640; services@L647; temporary@L647 +# + +"""Jason Stys: hosted-only published0.4.2 frozen-service -> Docker -> sealed-history acceptance. + +The CLI guards network/process work before starting. Helpers have independent +byte/protocol tests; no test invokes local Docker. Locations: docs/code-index.md. +No credential, recovery value, account path, raw service log or report body is emitted. +""" + +import argparse +import base64 +import gzip +import hashlib +import json +import os +import re +import secrets +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +from pathlib import Path, PurePosixPath +from urllib.parse import quote, urljoin, urlsplit + +import httpx +import psutil + +from examples.practical_projects import practical_project +from scripts.compiler_acceptance import validate_positive +from studio.models import LANGUAGES + +SOURCE = "eb9b1cd0c3e3fd1e70e397edeffb059073464f74" +WORKER_SHA256 = "750b665e545b7cf0806d4c493db86174f7f4e7ab2218aa239718e652ccdce158" +VERSION = "0.4.2" +RELEASE_ID = 440504623 +REPOSITORY = "JasonStys/code-flow-studio" +API = "https://api.github.com/repos/" + REPOSITORY +PAYLOAD = "CodeFlowStudio-0.4.2-linux-x64.tar.gz" +PAYLOAD_ID = 731850461 +PAYLOAD_BYTES = 188603375 +PAYLOAD_SHA256 = "1e75085bae4ab2611ead61a513b60f444941495608840cf33875d940cf963ebe" +MAX_ARCHIVE = 512 * 1024 * 1024 +MAX_EXPANDED = 2 * 1024 * 1024 * 1024 +MAX_MEMBER = 512 * 1024 * 1024 +MAX_RESPONSE = 2 * 1024 * 1024 +EXPECTED_LANGUAGES = { + "python", + "javascript", + "typescript", + "cpp", + "c", + "java", + "csharp", + "rust", + "php", + "bash", + "sql", + "assembly", +} + + +def require(condition: bool, message: str) -> None: + """Raise a fixed authored error without relying on removable Python assertions.""" + if not condition: + raise ValueError(message) + + +def strict_json(body: bytes): + """Reject duplicate/nonfinite JSON, without echoing the original data.""" + + def pairs(items): + result = dict(items) + require(len(result) == len(items), "Duplicate JSON field") + return result + + def constant(_value): + raise ValueError("Nonfinite JSON value") + + return json.loads(body, object_pairs_hook=pairs, parse_constant=constant) + + +def hosted_output(output: Path) -> Path: + """Refuse every execution path before network/process work outside the disposable hosted Linux job.""" + require( + sys.platform == "linux" + and os.environ.get("GITHUB_ACTIONS") == "true" + and os.environ.get("RUNNER_OS") == "Linux" + and os.environ.get("RUNNER_ENVIRONMENT") == "github-hosted" + and os.environ.get("GITHUB_REPOSITORY") == REPOSITORY, + "Published service probe requires the owned GitHub-hosted Linux job", + ) + workspace = Path(os.environ.get("GITHUB_WORKSPACE", "")).resolve() + require(workspace.is_dir() and bool(os.environ.get("GITHUB_WORKSPACE")), "Hosted workspace unavailable") + selected = output.resolve() + require( + selected.is_relative_to(workspace) and selected != workspace, + "Receipt must stay inside the owned workspace", + ) + require( + re.fullmatch(r"[a-f0-9]{40}", os.environ.get("GITHUB_SHA", "")) is not None, + "Verifier source identity unavailable", + ) + require(not selected.exists() or selected.is_dir(), "Receipt output must be a directory") + return selected / "released-service-probe.json" + + +class Evidence: + """Atomically preserve only public provenance, controlled stages, counts and hashes.""" + + def __init__(self, output: Path): + self.output = output + self.record = { + "schema": 1, + "version": VERSION, + "released_source": SOURCE, + "verifier_source": os.environ.get("GITHUB_SHA"), + "status": "running", + "stage": "starting", + "languages": [], + } + output.parent.mkdir(parents=True, exist_ok=True) + self.save() + + def save(self): + temporary = self.output.with_name(self.output.name + ".tmp") + temporary.write_text( + json.dumps(self.record, sort_keys=True, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + temporary.replace(self.output) + + def stage(self, stage: str, **public): + self.record.update(public) + self.record["stage"] = stage + self.save() + + +def bounded_body(response: httpx.Response, maximum: int) -> bytes: + """Bound actual response bytes, including decompressed JSON, before parsing.""" + body = bytearray() + for chunk in response.iter_bytes(8192): + body.extend(chunk) + require(len(body) <= maximum, "Response byte budget exceeded") + return bytes(body) + + +def api_json(client: httpx.Client, path: str): + """Read only fixed public repository API endpoints with no token or redirect.""" + require(path in {"/releases/tags/v0.4.2", "/git/ref/tags/v0.4.2"}, "Unknown public metadata endpoint") + with client.stream("GET", API + path, headers={"Accept": "application/vnd.github+json"}) as response: + require(response.status_code == 200, "Public release metadata unavailable") + return strict_json(bounded_body(response, 256 * 1024)) + + +def release_assets(release: dict, tag: dict) -> tuple[dict, dict]: + """Bind the actual published release, immutable tag and unique known Linux assets.""" + require( + release.get("id") == RELEASE_ID + and type(release.get("id")) is int + and release.get("tag_name") == "v" + VERSION + and release.get("draft") is False + and release.get("prerelease") is False + and release.get("target_commitish") == SOURCE + and bool(release.get("published_at")), + "Published release identity mismatch", + ) + require( + tag.get("ref") == "refs/tags/v" + VERSION + and tag.get("object", {}).get("type") == "commit" + and tag["object"].get("sha") == SOURCE, + "Published tag source mismatch", + ) + rows = release.get("assets") + require( + isinstance(rows, list) and len(rows) == 12 and all(isinstance(row, dict) for row in rows), + "Published asset inventory mismatch", + ) + require( + len({row.get("name") for row in rows}) == 12 + and all(type(row.get("id")) is int and row["id"] > 0 for row in rows) + and len({row["id"] for row in rows}) == 12, + "Ambiguous published assets", + ) + found = {row["name"]: row for row in rows} + require(PAYLOAD in found and PAYLOAD + ".sha256" in found, "Published Linux payload/sidecar missing") + require( + found[PAYLOAD].get("id") == PAYLOAD_ID + and found[PAYLOAD].get("size") == PAYLOAD_BYTES + and found[PAYLOAD].get("digest") == "sha256:" + PAYLOAD_SHA256, + "Accepted published payload was replaced", + ) + for name, maximum in [(PAYLOAD, MAX_ARCHIVE), (PAYLOAD + ".sha256", 4096)]: + row = found[name] + require( + type(row.get("size")) is int + and 0 < row["size"] <= maximum + and row.get("state") == "uploaded" + and isinstance(row.get("digest"), str) + and re.fullmatch(r"sha256:[a-f0-9]{64}", row["digest"]), + "Published asset size/digest mismatch", + ) + expected = "https://github.com/" + REPOSITORY + "/releases/download/v" + VERSION + "/" + quote(name) + require(row.get("browser_download_url") == expected, "Published asset URL mismatch") + return found[PAYLOAD], found[PAYLOAD + ".sha256"] + + +def download_url(url: str, initial: str) -> None: + """Allow canonical public GitHub URL or bounded signed release-CDN redirects only.""" + require(isinstance(url, str) and len(url) <= 16384, "Download URL outside budget") + prefix = "https://github.com/" + REPOSITORY + "/releases/download/v" + VERSION + "/" + require(initial in {prefix + PAYLOAD, prefix + PAYLOAD + ".sha256"}, "Unknown canonical release download") + parsed = urlsplit(url) + require( + parsed.scheme == "https" + and not parsed.username + and not parsed.password + and parsed.port is None + and not parsed.fragment, + "Unsafe download destination", + ) + if parsed.hostname == "github.com": + require(url == initial, "Unexpected repository download redirect") + else: + require( + parsed.hostname in {"release-assets.githubusercontent.com", "objects.githubusercontent.com"} + and re.match(r"^/github-production-release-asset(?:-[a-f0-9]+)?/", parsed.path), + "Foreign download redirect", + ) + + +def download(client: httpx.Client, row: dict, target: Path) -> str: + """Stream exactly the provider-declared bytes, enforce a total deadline and verify its SHA256.""" + initial = row["browser_download_url"] + require( + row.get("name") in {PAYLOAD, PAYLOAD + ".sha256"} + and type(row.get("size")) is int + and 0 < row["size"] <= (4096 if row["name"].endswith(".sha256") else MAX_ARCHIVE) + and isinstance(row.get("digest"), str) + and re.fullmatch(r"sha256:[a-f0-9]{64}", row["digest"]), + "Invalid download size/digest", + ) + url = initial + started = time.monotonic() + for _ in range(5): + download_url(url, initial) + client.cookies.clear() + with client.stream("GET", url) as response: + if response.status_code in {301, 302, 303, 307, 308}: + url = urljoin(url, response.headers.get("location", "")) + require(time.monotonic() - started < 180, "Download deadline exceeded") + continue + require(response.status_code == 200, "Published asset download unavailable") + measured, digest = 0, hashlib.sha256() + with target.open("xb") as stream: + for chunk in response.iter_raw(65536): + measured += len(chunk) + require( + measured <= row["size"] and time.monotonic() - started < 180, + "Download length/deadline exceeded", + ) + stream.write(chunk) + digest.update(chunk) + require( + measured == row["size"] and "sha256:" + digest.hexdigest() == row["digest"], + "Downloaded bytes fail provider digest", + ) + return digest.hexdigest() + raise ValueError("Download redirect budget exceeded") + + +def preflight_tar(stream) -> None: + """Bound physical gzip expansion and tar extension sizes before tarfile interprets PAX/long names.""" + expanded, headers = 0, 0 + with gzip.GzipFile(fileobj=stream) as raw: + while True: + header = raw.read(512) + expanded += len(header) + require(expanded <= MAX_EXPANDED and len(header) == 512, "Truncated/oversize tar stream") + if not any(header): + require(raw.read(512) == bytes(512), "Tar end marker mismatch") + expanded += 512 + require(expanded <= MAX_EXPANDED, "Archive expansion exceeded") + while chunk := raw.read(65536): + expanded += len(chunk) + require(expanded <= MAX_EXPANDED and not any(chunk), "Tar trailer/expansion mismatch") + return + headers += 1 + require(headers <= 30000, "Tar member count exceeded") + number = header[124:136].strip(b"\0 ") or b"0" + require(re.fullmatch(rb"[0-7]{1,12}", number) is not None, "Unsupported tar size encoding") + size = int(number, 8) + kind = header[156:157] + require(kind in {b"0", b"\0", b"5", b"x", b"g", b"L", b"K"}, "Unsupported archive member type") + limit = 65536 if kind in {b"x", b"g", b"L", b"K"} else MAX_MEMBER + require(size <= limit and (kind != b"5" or size == 0), "Tar member budget exceeded") + remaining = ((size + 511) // 512) * 512 + while remaining: + chunk = raw.read(min(65536, remaining)) + require(bool(chunk), "Truncated tar member") + if kind in {b"x", b"g"}: + require(b"GNU.sparse" not in chunk, "Sparse archive metadata unsupported") + remaining -= len(chunk) + expanded += len(chunk) + require(expanded <= MAX_EXPANDED, "Archive expansion exceeded") + + +def unpack(archive: Path, destination: Path) -> Path: + """Extract only bounded regular files/directories into a fresh private tree; no links or devices.""" + destination.mkdir(mode=0o700) + seen, expanded = set(), 0 + with archive.open("rb") as stream: + require(stat.S_ISREG(os.fstat(stream.fileno()).st_mode), "Archive must be a regular file") + preflight_tar(stream) + stream.seek(0) + with tarfile.open(fileobj=stream, mode="r:gz") as package: + for index, member in enumerate(package): + name = member.name.removeprefix("./") + if name in {"", "."} and member.isdir(): + continue + parts = PurePosixPath(name).parts + require( + index < 15000 + and len(name) <= 512 + and name not in seen + and not name.startswith("/") + and "\\" not in name + and ":" not in name + and not any(ord(c) < 32 for c in name) + and all(part not in {".", "..", ""} for part in name.split("/")) + and (member.isfile() or member.isdir()) + and not member.linkname + and not any(key.startswith("GNU.sparse") for key in member.pax_headers), + "Unsafe or duplicate archive path/member", + ) + seen.add(name) + require(0 <= member.size <= MAX_MEMBER, "Archive member size exceeded") + expanded += member.size + require(expanded <= MAX_EXPANDED, "Archive extracted size exceeded") + selected = destination.joinpath(*parts) + if member.isdir(): + selected.mkdir(parents=True, exist_ok=True, mode=0o700) + continue + selected.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + source = package.extractfile(member) + require(source is not None, "Missing archive file data") + measured = 0 + with source, selected.open("xb") as output: + while chunk := source.read(65536): + measured += len(chunk) + require(measured <= member.size, "Archive actual-read size exceeded") + output.write(chunk) + require(measured == member.size, "Archive file data truncated") + selected.chmod(0o700 if member.mode & 0o111 else 0o600) + resources = destination / "resources" + metadata = resources / "metadata/source-release.json" + require(metadata.is_file() and metadata.stat().st_size <= 4096, "Embedded release metadata missing") + value = strict_json(metadata.read_bytes()) + require( + isinstance(value, dict) + and type(value.get("schema")) is int + and type(value.get("source_dirty")) is bool + and value + == { + "schema": 1, + "product": "code-flow-studio", + "version": VERSION, + "source": SOURCE, + "source_dirty": False, + "worker_sha256": WORKER_SHA256, + }, + "Embedded release provenance differs", + ) + worker = resources / "runner/node_checks.cjs" + require(worker.is_file() and worker.stat().st_size <= 1024 * 1024, "Embedded worker missing") + require( + hashlib.sha256(worker.read_bytes()).hexdigest() == WORKER_SHA256, "Embedded worker digest differs" + ) + service = resources / "service/code-flow-service" + require(service.is_file() and os.access(service, os.X_OK), "Frozen released service unavailable") + return service + + +def service_environment(home: Path, token: str, nonce: str) -> dict: + """Pass no provider/GitHub/cloud credentials to the frozen service or Docker CLI.""" + config = home / "docker-config" + config.mkdir(mode=0o700) + return { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(home), + "TMPDIR": str(home), + "LANG": "C.UTF-8", + "DOCKER_CONFIG": str(config), + "DOCKER_HOST": "unix:///var/run/docker.sock", + "CFS_LAUNCH_TOKEN": token, + "CFS_LAUNCH_NONCE": nonce, + "CFS_PARENT_PID": str(os.getpid()), + "CFS_PARENT_STARTED": str(psutil.Process().create_time()), + } + + +def request_json(client: httpx.Client, method: str, path: str, data=None): + """Use the owned loopback session only, with bounded no-redirect JSON responses.""" + require( + path.startswith(("/private/", "/health", "/launcher/shutdown")) and ".." not in path, + "Unexpected service route", + ) + with client.stream(method, path, json=data) as response: + require(response.status_code == 200, "Released service request failed") + return strict_json(bounded_body(response, MAX_RESPONSE)) + + +class Service: + """Own one frozen service and nonce-bound listener, never adopt an existing server.""" + + def __init__(self, executable: Path, data: Path, home: Path): + self.token, self.nonce = secrets.token_hex(32), secrets.token_hex(32) + home.mkdir(mode=0o700) + self.log = home / "service-output.txt" + with self.log.open("wb") as output: + self.process = subprocess.Popen( # noqa: S603 - digest-verified executable/fixed argv, hosted guard in main. + [str(executable), "--no-browser", "--port", "0", "--data", str(data)], + env=service_environment(home, self.token, self.nonce), + stdout=output, + stderr=subprocess.STDOUT, + ) + self.client = None + + def ready(self): + """Wait only after the caller has recorded ownership of this child process.""" + deadline = time.monotonic() + 45 + while time.monotonic() < deadline and self.process.poll() is None: + with self.log.open("rb") as source: + body = source.read(65537) + require(len(body) <= 65536, "Service startup output exceeded") + matches = re.findall(rb"Code Flow Studio - Jason Stys: http://127\.0\.0\.1:([0-9]{1,5})/", body) + if matches: + require(len(matches) == 1 and 1024 <= int(matches[0]) <= 65535, "Ambiguous service readiness") + origin = "http://127.0.0.1:" + matches[0].decode() + candidate = httpx.Client( + base_url=origin, + trust_env=False, + follow_redirects=False, + timeout=15, + headers={"X-CFS-Request": "v1", "Content-Type": "application/json", "Origin": origin}, + ) + try: + health = request_json(candidate, "GET", "/health") + require( + health.get("launcher_nonce") == self.nonce + and health.get("version") == VERSION + and health.get("mode") == "private-local-accounts", + "Owned released service identity mismatch", + ) + except Exception: + candidate.close() + raise + self.client = candidate + return + time.sleep(0.1) + raise ValueError("Released service startup deadline/exit") + + def stop(self): + """Request the shipping authenticated graceful shutdown and require actual owned process exit.""" + require(self.client is not None, "Owned service was not ready") + self.client.headers["Authorization"] = "Bearer " + self.token + require( + request_json(self.client, "POST", "/launcher/shutdown", {}) == {"stopping": True}, + "Graceful shutdown unacknowledged", + ) + self.process.wait(timeout=35) + self.client.close() + require(self.process.returncode == 0, "Released service exit failed") + + +def validate_export(exported: dict, result: dict, revision: dict) -> None: + """Bind the shipping Store.timeline object to this exact saved project, revision and run.""" + require( + isinstance(exported, dict) + and {key: value for key, value in exported.items() if key != "iteration_history"} == result, + "JSON export differs from actual run", + ) + history = exported.get("iteration_history") + require( + isinstance(history, dict) + and set(history) == {"project_id", "revisions", "truncated", "scope"} + and isinstance(revision.get("project_id"), str) + and history["project_id"] == revision["project_id"] + and history["truncated"] is False + and history["scope"] + == "Latest 50 revisions; latest 10 attempts each. Full source/logs remain in their individual exports." + and isinstance(history["revisions"], list) + and len(history["revisions"]) == 1, + "JSON export timeline schema/project differs", + ) + snapshot = history["revisions"][0] + require( + isinstance(snapshot, dict) + and snapshot.get("id") == revision["id"] + and snapshot.get("parent_id") is None + and snapshot.get("snapshot_sha256") + == hashlib.sha256(json.dumps(revision["project"]).encode()).hexdigest() + and snapshot.get("priorities") == revision["project"]["priorities"] + and isinstance(snapshot.get("created"), str) + and bool(snapshot["created"]) + and isinstance(snapshot.get("attempts"), list) + and len(snapshot["attempts"]) == 1, + "JSON export timeline revision differs", + ) + attempt = snapshot["attempts"][0] + actual = result["result"] + require( + isinstance(attempt, dict) + and attempt.get("id") == result["id"] + and attempt.get("status") == "completed" + and isinstance(attempt.get("created"), str) + and bool(attempt["created"]) + and attempt.get("checks") == {key: actual.get("checks", {}).get(key) for key in ("passed", "failed")} + and attempt.get("benchmark") == actual.get("benchmark", {}) + and attempt.get("coverage") == actual.get("coverage", {}) + and attempt.get("mutation_score") == actual.get("mutation", {}).get("score") + and attempt.get("fuzz_status") == actual.get("fuzz", {}).get("status") + and attempt.get("generated_tests_exit") == actual.get("generated_tests", {}).get("exit_code"), + "JSON export timeline attempt differs", + ) + + +def exercise(service: Service, data: Path, evidence: Evidence) -> dict: + """Run real literal-oracle projects through HTTP, then save/seal before a full process restart.""" + require(set(LANGUAGES) == EXPECTED_LANGUAGES and len(LANGUAGES) == 12, "Language inventory changed") + alias, password = "hosted_release042", secrets.token_urlsafe(30) + key = { + "key": base64.b64encode(secrets.token_bytes(32)).decode(), + "salt": base64.b64encode(secrets.token_bytes(16)).decode(), + } + client = service.client + request_json(client, "POST", "/private/register", {"alias": alias, "password": password}) + require( + request_json(client, "POST", "/private/unlock", key).get("unlocked") is True, "Vault did not unlock" + ) + stored = {} + for language in LANGUAGES: + evidence.stage("executing-" + language) + project = practical_project(language) + require( + project.language == language and len(project.cases) >= 4, "Practical fixture lacks approved cases" + ) + revision = request_json(client, "POST", "/private/api/revisions", project.model_dump()) + identifier = revision.get("id") + require( + isinstance(identifier, str) and re.fullmatch(r"[a-f0-9-]{36}", identifier), + "Invalid revision identifier", + ) + require(revision.get("project") == project.model_dump(), "Saved source/oracles differ") + job = request_json(client, "POST", f"/private/api/revisions/{identifier}/run", {}).get("id") + require(isinstance(job, str) and re.fullmatch(r"[a-f0-9-]{36}", job), "Invalid job identifier") + deadline = time.monotonic() + 200 + while True: + result = request_json(client, "GET", "/private/api/runs/" + job) + if result.get("status") not in {"queued", "running"}: + break + require(time.monotonic() < deadline, "Released worker job deadline exceeded") + time.sleep(0.25) + require(result.get("status") == "completed", "Released worker did not complete") + validate_positive(result.get("result", {}), project) + exported = request_json(client, "GET", f"/private/api/runs/{job}/report?format=json") + require( + result.get("id") == job and result.get("revision_id") == identifier, + "Run/revision identity differs", + ) + validate_export(exported, result, revision) + digest = canonical_digest(result["result"]) + stored[language] = { + "revision": identifier, + "job": job, + "digest": digest, + "run_digest": canonical_digest(result), + "export_digest": canonical_digest(exported), + } + evidence.record["languages"].append( + { + "language": language, + "approved_cases": len(project.cases), + "status": "passed", + "result_sha256": digest, + } + ) + evidence.save() + request_json(client, "POST", "/private/save", {}) + require( + request_json(client, "POST", "/private/lock", {}) == {"locked": True} + and not (data / "active").exists() + and len(list((data / "encrypted").glob("*.cfsvault"))) == 1, + "Vault sealing evidence incomplete", + ) + return {"alias": alias, "password": password, "key": key, "stored": stored} + + +def restored(service: Service, data: Path, private: dict) -> None: + """Require a new login/key and byte-identical durable reports after restarting the actual binary.""" + client = service.client + request_json( + client, "POST", "/private/login", {"alias": private["alias"], "password": private["password"]} + ) + require( + request_json(client, "POST", "/private/unlock", private["key"]).get("unlocked") is True, + "Restarted vault did not unlock", + ) + history = request_json(client, "GET", "/private/api/revisions") + require( + isinstance(history, list) + and {row.get("id") for row in history} == {row["revision"] for row in private["stored"].values()}, + "Reopened revision history differs", + ) + for row in private["stored"].values(): + result = request_json(client, "GET", "/private/api/runs/" + row["job"]) + exported = request_json(client, "GET", f"/private/api/runs/{row['job']}/report?format=json") + require( + result.get("status") == "completed" + and canonical_digest(result.get("result")) == row["digest"] + and canonical_digest(result) == row["run_digest"] + and canonical_digest(exported) == row["export_digest"], + "Reopened run or JSON export differs", + ) + require( + request_json(client, "POST", "/private/lock", {}) == {"locked": True} + and not (data / "active").exists(), + "Restarted vault failed to seal", + ) + + +def canonical_digest(value) -> str: + """Bind every JSON result field without retaining its body in the public receipt.""" + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + + +def cleanup_owned(services: list, temporary: Path) -> bool: + """Stop only recorded children; retain private recovery data if a child cannot stop normally.""" + failed = False + for service in services: + if service.process.poll() is None: + try: + if service.client is not None: + service.stop() + else: + # No account operation began; SIGTERM targets only this recorded hosted child. + service.process.terminate() + service.process.wait(timeout=10) + except Exception: + failed = True + if all(service.process.poll() is not None for service in services): + require( + temporary.name.startswith("cfs-released042-") and not temporary.is_symlink(), + "Invalid owned temporary directory", + ) + shutil.rmtree(temporary) + return not failed + + +def main(): + """Perform only this fixed hosted acceptance, retaining sanitized incremental failure evidence.""" + parser = argparse.ArgumentParser( + description="Validate the actual published0.4.2 Linux service with hosted Docker" + ) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + output = hosted_output(args.output) + evidence = Evidence(output) + temporary = Path(tempfile.mkdtemp(prefix="cfs-released042-", dir=os.environ.get("RUNNER_TEMP"))) + temporary.chmod(0o700) + services = [] + try: + with httpx.Client(trust_env=False, follow_redirects=False, timeout=30) as client: + payload, sidecar = release_assets( + api_json(client, "/releases/tags/v0.4.2"), api_json(client, "/git/ref/tags/v0.4.2") + ) + evidence.stage("release-verified", release_id=RELEASE_ID) + archive, checksum = temporary / PAYLOAD, temporary / (PAYLOAD + ".sha256") + digest = download(client, payload, archive) + download(client, sidecar, checksum) + require( + checksum.read_bytes() == f"{digest} {PAYLOAD}\n".encode("ascii"), + "Published checksum record differs", + ) + evidence.stage("download-verified", payload_sha256=digest, payload_bytes=payload["size"]) + executable = unpack(archive, temporary / "package") + evidence.stage("embedded-provenance-verified", worker_sha256=WORKER_SHA256) + data = temporary / "accounts" + evidence.stage("starting-released-service") + first = Service(executable, data, temporary / "first-service") + services.append(first) + first.ready() + private = exercise(first, data, evidence) + first.stop() + evidence.stage("first-service-sealed-and-exited") + second = Service(executable, data, temporary / "second-service") + services.append(second) + second.ready() + restored(second, data, private) + second.stop() + evidence.stage( + "complete", + status="passed", + restored_reports=len(private["stored"]), + approved_cases=sum(row["approved_cases"] for row in evidence.record["languages"]), + scope="Published Linux service with a fresh source-bound0.4.2 hosted Docker worker; not a local PC compiler run or historical image-byte proof", + ) + except Exception as error: + evidence.stage( + "failed", status="failed", failed_stage=evidence.record["stage"], error_type=type(error).__name__ + ) + raise SystemExit( + "Published service acceptance failed; inspect the bounded sanitized receipt" + ) from None + finally: + if not cleanup_owned(services, temporary): + evidence.stage("cleanup-failed", status="failed", failed_stage=evidence.record["stage"]) + raise SystemExit( + "Owned released service did not stop normally; private ephemeral recovery data was retained" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/compiler-reports-042.json b/tests/fixtures/compiler-reports-042.json new file mode 100644 index 0000000..e9eda97 --- /dev/null +++ b/tests/fixtures/compiler-reports-042.json @@ -0,0 +1,1440 @@ +{ + "schema": 1, + "run_id": 42869521158, + "source": "eb9b1cd0c3e3fd1e70e397edeffb059073464f74", + "scope": "Observed synthetic reports from the initial hosted run, including legitimate partial coverage; replay tests are not live execution.", + "reports": [ + { + "language": "python", + "kind": "positive", + "acceptance": "failed", + "project": { + "title": "Practical python standard-library CLI validation", + "language": "python", + "entry": "main.py", + "files": [ + { + "name": "main.py", + "content": "\"\"\"Summarize at most 64 comma-separated, bounded integer telemetry samples.\"\"\"\nimport sys\n\ndef summarize(text):\n \"\"\"Return count/sum/min/max; reject malformed or out-of-range samples.\"\"\"\n if not text.strip():\n return \"0|0|NA|NA\"\n values = [int(part.strip()) for part in text.strip().split(\",\")]\n if len(values) > 64 or any(value < -1000 or value > 1000 for value in values):\n raise ValueError(\"Invalid telemetry range\")\n return f\"{len(values)}|{sum(values)}|{min(values)}|{max(values)}\"\n\ntry:\n print(summarize(sys.stdin.read()))\nexcept ValueError:\n raise SystemExit(2) from None\n", + "encoding": "text" + } + ], + "cases": [ + { + "name": "telemetry-window", + "stdin": "20,30,10\n", + "stdout": "3|60|10|30", + "exit_code": 0 + }, + { + "name": "zero-readings", + "stdin": "0,0\n", + "stdout": "2|0|0|0", + "exit_code": 0 + }, + { + "name": "signed-readings", + "stdin": "-5,5\n", + "stdout": "2|0|-5|5", + "exit_code": 0 + }, + { + "name": "empty-window", + "stdin": "", + "stdout": "0|0|NA|NA", + "exit_code": 0 + }, + { + "name": "range-boundaries", + "stdin": "-1000,1000\n", + "stdout": "2|0|-1000|1000", + "exit_code": 0 + }, + { + "name": "malformed-reading", + "stdin": "12,bad\n", + "stdout": "", + "exit_code": 2 + } + ], + "requirements": "Synthetic bounded CLI demonstration. Preserve the approved stdout and exit-code oracles, reject invalid input explicitly, and use no network or external packages.", + "tests": "", + "fuzz_methods": [ + "boundary", + "mutation" + ], + "fuzz_cases": 0, + "seed": 42, + "benchmark_repeats": 1, + "mutation": false, + "fuzz_engine": "blackbox", + "fuzz_harness": "", + "priorities": { + "runtime": 0.4, + "memory": 0.2, + "readability": 0.25, + "footprint": 0.15, + "scrutiny": "strict" + } + }, + "result": { + "tool_status": "completed", + "compilation": { + "exit_code": 0, + "stdout": "", + "stderr": "", + "timeout": false, + "ms": 114.19804199999817, + "output_truncated": false + }, + "checks": { + "passed": 6, + "failed": 0, + "cases": [ + { + "name": "telemetry-window", + "passed": true, + "exit_code": 0, + "stdout": "3|60|10|30\n", + "stderr": "", + "timeout": false, + "ms": 15.847777000004726, + "output_truncated": false + }, + { + "name": "zero-readings", + "passed": true, + "exit_code": 0, + "stdout": "2|0|0|0\n", + "stderr": "", + "timeout": false, + "ms": 15.694094999986419, + "output_truncated": false + }, + { + "name": "signed-readings", + "passed": true, + "exit_code": 0, + "stdout": "2|0|-5|5\n", + "stderr": "", + "timeout": false, + "ms": 15.70573300000433, + "output_truncated": false + }, + { + "name": "empty-window", + "passed": true, + "exit_code": 0, + "stdout": "0|0|NA|NA\n", + "stderr": "", + "timeout": false, + "ms": 15.704260999996222, + "output_truncated": false + }, + { + "name": "range-boundaries", + "passed": true, + "exit_code": 0, + "stdout": "2|0|-1000|1000\n", + "stderr": "", + "timeout": false, + "ms": 15.74163699999076, + "output_truncated": false + }, + { + "name": "malformed-reading", + "passed": true, + "exit_code": 2, + "stdout": "", + "stderr": "", + "timeout": false, + "ms": 15.763164999981427, + "output_truncated": false + } + ] + }, + "oracle_hash": "86148a99da058d72efdc9f46a73480644e16525c78dd8def5b64be36b25a837c", + "environment": { + "worker": "polyglot-v3-language-evidence", + "python": "3.12.12", + "language": "python", + "machine": "x86_64", + "benchmark_repeats": 1 + }, + "generated_tests": { + "status": "not-provided" + }, + "benchmark": { + "samples_ms": [ + 15.75225300001648, + 15.668045999973401, + 15.69523700001696, + 15.691530999987435, + 15.757335000017747, + 15.914937000019336 + ], + "median_ms": 15.72374500001672, + "max_rss_kib": 24068, + "scope": "Process startup + workload; peak child RSS, not per-function memory. Exploratory measurements." + }, + "coverage": { + "status": "measured", + "entry": { + "covered_lines": 11, + "num_statements": 12, + "percent_covered": 87.5, + "percent_covered_display": "88", + "missing_lines": 1, + "excluded_lines": 0, + "num_branches": 4, + "num_partial_branches": 1, + "covered_branches": 3, + "missing_branches": 1 + }, + "files": { + "main.py": { + "status": "measured", + "summary": { + "covered_lines": 11, + "num_statements": 12, + "percent_covered": 87.5, + "percent_covered_display": "88", + "missing_lines": 1, + "excluded_lines": 0, + "num_branches": 4, + "num_partial_branches": 1, + "covered_branches": 3, + "missing_branches": 1 + }, + "source_sha256": "661139d62e78982c04de13dc3bf66d6ab7366e52b573ad2f21c709ae42d46a95", + "truncated": false, + "missing_lines": [ + 10 + ], + "missing_branches": [ + [ + 9, + 10 + ] + ], + "executed_lines": [ + 1, + 2, + 4, + 6, + 7, + 8, + 9, + 11, + 13, + 14, + 15, + 16 + ], + "excluded_lines": [] + } + }, + "locations_truncated": false, + "scope": "Supplied Python sources under approved CLI cases only, not generated pytest tests; line/branch, not condition/MC/DC. At most 20,000 location entries, missing before executed; negative branch destinations denote function exits." + }, + "mutation": { + "status": "not-selected" + }, + "fuzz": { + "status": "completed", + "engine": "blackbox", + "executed": 0, + "seed": 42, + "findings": [], + "nonzero_policy": "Ordinary nonzero exits are not automatically bugs; approved expected behavior is required", + "blocked_methods": [], + "coverage_guided": false + }, + "diagnostics": [], + "diagnostics_truncated": false, + "unlocated_failures": [], + "location_caveat": "Tool-reported references, not proven root causes. Output may be incomplete, misleading or truncated; unrecognized paths remain in raw logs.", + "static": { + "sha256": "661139d62e78982c04de13dc3bf66d6ab7366e52b573ad2f21c709ae42d46a95", + "source_bytes": 614, + "lines": 16, + "long_lines": 0, + "function_count": 1, + "branch_nodes": 3, + "findings": [], + "files": { + "main.py": { + "source_sha256": "661139d62e78982c04de13dc3bf66d6ab7366e52b573ad2f21c709ae42d46a95", + "parsed": true, + "function_count": 1, + "branch_nodes": 3 + } + }, + "findings_truncated": false, + "caveat": "Python syntax and conservative unused/unreachable candidates; other text uses risk patterns only. Unused is not uncovered. No inferred Big-O, complete bug detection or security certification." + }, + "isolation": { + "network": "none", + "memory_mib": 1024, + "cpus": 2, + "pids": 128, + "read_only": true + } + }, + "error": "Coverage counts disagree with retained locations" + }, + { + "language": "csharp", + "kind": "positive", + "acceptance": "failed", + "project": { + "title": "Practical csharp standard-library CLI validation", + "language": "csharp", + "entry": "Program.cs", + "files": [ + { + "name": "Program.cs", + "content": "// Decimal invoice; invariant formatting and explicit half-away-from-zero currency rounding.\nusing System.Globalization;\nvar fields = Console.In.ReadToEnd().Split((char[])null, StringSplitOptions.RemoveEmptyEntries);\nvar culture = CultureInfo.InvariantCulture;\nif (fields.Length != 3 ||\n !int.TryParse(fields[0], NumberStyles.None, culture, out var quantity) ||\n !decimal.TryParse(fields[1], NumberStyles.AllowDecimalPoint, culture, out var unitPrice) ||\n !decimal.TryParse(fields[2], NumberStyles.AllowDecimalPoint, culture, out var taxPercent) ||\n quantity > 1000 || unitPrice > 1000000m || taxPercent > 100m) return 2;\nvar subtotal = decimal.Round(quantity * unitPrice, 2, MidpointRounding.AwayFromZero);\nvar tax = decimal.Round(subtotal * taxPercent / 100m, 2, MidpointRounding.AwayFromZero);\nConsole.WriteLine(subtotal.ToString(\"F2\", culture) + \"|\" + tax.ToString(\"F2\", culture) +\n \"|\" + (subtotal + tax).ToString(\"F2\", culture));\nreturn 0;\n", + "encoding": "text" + } + ], + "cases": [ + { + "name": "taxed-invoice", + "stdin": "3 19.99 8.25\n", + "stdout": "59.97|4.95|64.92", + "exit_code": 0 + }, + { + "name": "zero-quantity", + "stdin": "0 19.99 8.25\n", + "stdout": "0.00|0.00|0.00", + "exit_code": 0 + }, + { + "name": "half-cent-tax", + "stdin": "1 0.05 10\n", + "stdout": "0.05|0.01|0.06", + "exit_code": 0 + }, + { + "name": "tax-free-invoice", + "stdin": "2 7.50 0\n", + "stdout": "15.00|0.00|15.00", + "exit_code": 0 + }, + { + "name": "bounded-maximum", + "stdin": "1 1000000 100\n", + "stdout": "1000000.00|1000000.00|2000000.00", + "exit_code": 0 + }, + { + "name": "malformed-price", + "stdin": "1 money 8\n", + "stdout": "", + "exit_code": 2 + } + ], + "requirements": "Synthetic bounded CLI demonstration. Preserve the approved stdout and exit-code oracles, reject invalid input explicitly, and use no network or external packages.", + "tests": "", + "fuzz_methods": [ + "boundary", + "mutation" + ], + "fuzz_cases": 0, + "seed": 42, + "benchmark_repeats": 1, + "mutation": false, + "fuzz_engine": "blackbox", + "fuzz_harness": "", + "priorities": { + "runtime": 0.4, + "memory": 0.2, + "readability": 0.25, + "footprint": 0.15, + "scrutiny": "strict" + } + }, + "result": { + "tool_status": "completed", + "compilation": { + "exit_code": 0, + "stdout": "\nWelcome to .NET 8.0!\r\n---------------------\r\nSDK Version: 8.0.427\n\n----------------\r\nInstalled an ASP.NET Core HTTPS development certificate.\r\nTo trust the certificate, view the instructions: https://aka.ms/dotnet-https-linux\n\n----------------\r\nWrite your first app: https://aka.ms/dotnet-hello-world\r\nFind out what's new: https://aka.ms/dotnet-whats-new\r\nExplore documentation: https://aka.ms/dotnet-docs\r\nReport issues and find source on GitHub: https://github.com/dotnet/core\r\nUse 'dotnet --help' to see available commands or visit: https://aka.ms/dotnet-cli\r\n--------------------------------------------------------------------------------------\nAn issue was encountered verifying workloads. For more information, run \"dotnet workload update\".\n\nBuild succeeded.\n 0 Warning(s)\n 0 Error(s)\n\nTime Elapsed 00:00:02.16\n", + "stderr": "", + "timeout": false, + "ms": 2770.765936999993, + "output_truncated": false + }, + "checks": { + "passed": 6, + "failed": 0, + "cases": [ + { + "name": "taxed-invoice", + "passed": true, + "exit_code": 0, + "stdout": "59.97|4.95|64.92\n", + "stderr": "", + "timeout": false, + "ms": 64.18759599998225, + "output_truncated": false + }, + { + "name": "zero-quantity", + "passed": true, + "exit_code": 0, + "stdout": "0.00|0.00|0.00\n", + "stderr": "", + "timeout": false, + "ms": 64.07732599998894, + "output_truncated": false + }, + { + "name": "half-cent-tax", + "passed": true, + "exit_code": 0, + "stdout": "0.05|0.01|0.06\n", + "stderr": "", + "timeout": false, + "ms": 64.07525200000919, + "output_truncated": false + }, + { + "name": "tax-free-invoice", + "passed": true, + "exit_code": 0, + "stdout": "15.00|0.00|15.00\n", + "stderr": "", + "timeout": false, + "ms": 64.0945810000062, + "output_truncated": false + }, + { + "name": "bounded-maximum", + "passed": true, + "exit_code": 0, + "stdout": "1000000.00|1000000.00|2000000.00\n", + "stderr": "", + "timeout": false, + "ms": 64.11351500000251, + "output_truncated": false + }, + { + "name": "malformed-price", + "passed": true, + "exit_code": 2, + "stdout": "", + "stderr": "", + "timeout": false, + "ms": 64.1329990000088, + "output_truncated": false + } + ] + }, + "oracle_hash": "212c8e6419ec44ccd921ed1e7b2900b2408ba3bfc805993afa71db6f1bef062c", + "environment": { + "worker": "polyglot-v3-language-evidence", + "python": "3.12.12", + "language": "csharp", + "machine": "x86_64", + "benchmark_repeats": 1 + }, + "generated_tests": { + "status": "not-provided" + }, + "benchmark": { + "samples_ms": [ + 64.12505700001248, + 64.10751100000311, + 64.10314299998277, + 64.05358999998612, + 64.15890399998148, + 64.11448100001849 + ], + "median_ms": 64.1109960000108, + "max_rss_kib": 171844, + "scope": "Process startup + workload; peak child RSS, not per-function memory. Exploratory measurements." + }, + "coverage": { + "status": "partial", + "tool": "Coverlet 8.0.1", + "metric": "portable-pdb-sequence-point-line", + "files": { + "Program.cs": { + "status": "measured", + "metric": "portable-pdb-sequence-point-line", + "summary": { + "covered_lines": 12, + "num_statements": 12, + "missing_lines": 0, + "percent_covered": 100.0 + }, + "source_sha256": "6d63cfb4c471b31a81d2eb920411a5778b95dea0c5763c129492eb5c520708c5", + "missing_lines": [], + "executed_lines": [ + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "excluded_lines": [], + "missing_branches": [], + "branch_points": [ + { + "line": 5, + "branch_id": "28", + "case_index": 0, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "42", + "case_index": 0, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "57", + "case_index": 0, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "72", + "case_index": 0, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "80", + "case_index": 0, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "98", + "case_index": 0, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "121", + "case_index": 0, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "28", + "case_index": 1, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "42", + "case_index": 1, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "57", + "case_index": 1, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "72", + "case_index": 1, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "80", + "case_index": 1, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "98", + "case_index": 1, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "121", + "case_index": 1, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "28", + "case_index": 2, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "42", + "case_index": 2, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "57", + "case_index": 2, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "72", + "case_index": 2, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "80", + "case_index": 2, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "98", + "case_index": 2, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "121", + "case_index": 2, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "28", + "case_index": 3, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "42", + "case_index": 3, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "57", + "case_index": 3, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "72", + "case_index": 3, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "80", + "case_index": 3, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "98", + "case_index": 3, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "121", + "case_index": 3, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "28", + "case_index": 4, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "42", + "case_index": 4, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "57", + "case_index": 4, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "72", + "case_index": 4, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "80", + "case_index": 4, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "98", + "case_index": 4, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "121", + "case_index": 4, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "28", + "case_index": 5, + "coverage": "100%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "42", + "case_index": 5, + "coverage": "100%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "57", + "case_index": 5, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "72", + "case_index": 5, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "80", + "case_index": 5, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "98", + "case_index": 5, + "coverage": "50%", + "kind": "coverlet-condition" + }, + { + "line": 5, + "branch_id": "121", + "case_index": 5, + "coverage": "50%", + "kind": "coverlet-condition" + } + ], + "truncated": false + } + }, + "entry": { + "covered_lines": 12, + "num_statements": 12, + "missing_lines": 0, + "percent_covered": 100.0 + }, + "cases": [ + { + "name": "taxed-invoice", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false, + "collector_messages": "59.97|4.95|64.92\n\nCalculating coverage result...\n Generating report '/tmp/project/.analysis/case-0.json'\n+--------+------+--------+--------+\n| Module | Line | Branch | Method |\n+--------+------+--------+--------+\n| App | 100% | 50% | 100% |\n+--------+------+--------+--------+\n\n+---------+------+--------+--------+\n| | Line | Branch | Method |\n+---------+------+--------+--------+\n| Total | 100% | 50% | 100% |\n+---------+------+--------+--------+\n| Average | 100% | 50% | 100% |\n+---------+------+--------+--------+\n\n" + }, + { + "name": "zero-quantity", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false, + "collector_messages": "0.00|0.00|0.00\n\nCalculating coverage result...\n Generating report '/tmp/project/.analysis/case-1.json'\n+--------+------+--------+--------+\n| Module | Line | Branch | Method |\n+--------+------+--------+--------+\n| App | 100% | 50% | 100% |\n+--------+------+--------+--------+\n\n+---------+------+--------+--------+\n| | Line | Branch | Method |\n+---------+------+--------+--------+\n| Total | 100% | 50% | 100% |\n+---------+------+--------+--------+\n| Average | 100% | 50% | 100% |\n+---------+------+--------+--------+\n\n" + }, + { + "name": "half-cent-tax", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false, + "collector_messages": "0.05|0.01|0.06\n\nCalculating coverage result...\n Generating report '/tmp/project/.analysis/case-2.json'\n+--------+------+--------+--------+\n| Module | Line | Branch | Method |\n+--------+------+--------+--------+\n| App | 100% | 50% | 100% |\n+--------+------+--------+--------+\n\n+---------+------+--------+--------+\n| | Line | Branch | Method |\n+---------+------+--------+--------+\n| Total | 100% | 50% | 100% |\n+---------+------+--------+--------+\n| Average | 100% | 50% | 100% |\n+---------+------+--------+--------+\n\n" + }, + { + "name": "tax-free-invoice", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false, + "collector_messages": "15.00|0.00|15.00\n\nCalculating coverage result...\n Generating report '/tmp/project/.analysis/case-3.json'\n+--------+------+--------+--------+\n| Module | Line | Branch | Method |\n+--------+------+--------+--------+\n| App | 100% | 50% | 100% |\n+--------+------+--------+--------+\n\n+---------+------+--------+--------+\n| | Line | Branch | Method |\n+---------+------+--------+--------+\n| Total | 100% | 50% | 100% |\n+---------+------+--------+--------+\n| Average | 100% | 50% | 100% |\n+---------+------+--------+--------+\n\n" + }, + { + "name": "bounded-maximum", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false, + "collector_messages": "1000000.00|1000000.00|2000000.00\n\nCalculating coverage result...\n Generating report '/tmp/project/.analysis/case-4.json'\n+--------+------+--------+--------+\n| Module | Line | Branch | Method |\n+--------+------+--------+--------+\n| App | 100% | 50% | 100% |\n+--------+------+--------+--------+\n\n+---------+------+--------+--------+\n| | Line | Branch | Method |\n+---------+------+--------+--------+\n| Total | 100% | 50% | 100% |\n+---------+------+--------+--------+\n| Average | 100% | 50% | 100% |\n+---------+------+--------+--------+\n\n" + }, + { + "name": "malformed-price", + "exit_code": 1, + "expected_exit_code": 2, + "timeout": false, + "collector_messages": "\nCalculating coverage result...\n Generating report '/tmp/project/.analysis/case-5.json'\n+--------+--------+--------+--------+\n| Module | Line | Branch | Method |\n+--------+--------+--------+--------+\n| App | 58.33% | 64.28% | 100% |\n+--------+--------+--------+--------+\n\n+---------+--------+--------+--------+\n| | Line | Branch | Method |\n+---------+--------+--------+--------+\n| Total | 58.33% | 64.28% | 100% |\n+---------+--------+--------+--------+\n| Average | 58.33% | 64.28% | 100% |\n+---------+--------+--------+--------+\n\n" + } + ], + "requested_cases": 6, + "locations_truncated": false, + "scope": "Instrumented approved CLI reruns only, separate from timing/RSS and generated pytest tests. Unknown/unloaded sources remain not-measured. Branch points retain tool IDs, not invented destination lines. Shared-worker interference remains possible." + }, + "unused": { + "status": "completed", + "reason": null, + "tool": "C# compiler unused locals/private fields", + "process": { + "exit_code": 0, + "stdout": "\nBuild succeeded.\n 0 Warning(s)\n 0 Error(s)\n\nTime Elapsed 00:00:00.84\n", + "stderr": "", + "timeout": false, + "ms": 1016.2306930000113, + "output_truncated": false + }, + "findings": [], + "findings_truncated": false, + "scope": "Review-only language-native unused candidates; public APIs, side effects and dynamic references require human review." + }, + "mutation": { + "status": "not-selected" + }, + "fuzz": { + "status": "completed", + "engine": "blackbox", + "executed": 0, + "seed": 42, + "findings": [], + "nonzero_policy": "Ordinary nonzero exits are not automatically bugs; approved expected behavior is required", + "blocked_methods": [], + "coverage_guided": false + }, + "diagnostics": [], + "diagnostics_truncated": false, + "unlocated_failures": [], + "location_caveat": "Tool-reported references, not proven root causes. Output may be incomplete, misleading or truncated; unrecognized paths remain in raw logs.", + "static": { + "sha256": "6d63cfb4c471b31a81d2eb920411a5778b95dea0c5763c129492eb5c520708c5", + "source_bytes": 975, + "lines": 14, + "long_lines": 0, + "function_count": 0, + "branch_nodes": 0, + "findings": [], + "files": { + "Program.cs": { + "source_sha256": "6d63cfb4c471b31a81d2eb920411a5778b95dea0c5763c129492eb5c520708c5" + } + }, + "findings_truncated": false, + "caveat": "Python syntax and conservative unused/unreachable candidates; other text uses risk patterns only. Unused is not uncovered. No inferred Big-O, complete bug detection or security certification." + }, + "isolation": { + "network": "none", + "memory_mib": 1024, + "cpus": 2, + "pids": 128, + "read_only": true + } + }, + "error": "Coverage was not measured" + }, + { + "language": "php", + "kind": "positive", + "acceptance": "failed", + "project": { + "title": "Practical php standard-library CLI validation", + "language": "php", + "entry": "main.php", + "files": [ + { + "name": "main.php", + "content": " 1000 || $unitCents > 100000 || $shippingCents > 100000) { exit(2); }\n$subtotal = $quantity * $unitCents;\n$discount = $quantity >= 10 ? intdiv($subtotal * 5 + 50, 100) : 0;\n$total = $subtotal - $discount + ($quantity > 0 ? $shippingCents : 0);\necho $subtotal . '|' . $discount . '|' . $total . \"\\n\";\n", + "encoding": "text" + } + ], + "cases": [ + { + "name": "ordinary-quote", + "stdin": "3 1200 500\n", + "stdout": "3600|0|4100", + "exit_code": 0 + }, + { + "name": "bulk-discount", + "stdin": "10 1000 500\n", + "stdout": "10000|500|10000", + "exit_code": 0 + }, + { + "name": "zero-order", + "stdin": "0 1000 500\n", + "stdout": "0|0|0", + "exit_code": 0 + }, + { + "name": "half-cent-discount", + "stdin": "10 1 0\n", + "stdout": "10|1|9", + "exit_code": 0 + }, + { + "name": "bounded-maximum", + "stdin": "1000 100000 100000\n", + "stdout": "100000000|5000000|95100000", + "exit_code": 0 + }, + { + "name": "malformed-price", + "stdin": "2 nope 1\n", + "stdout": "", + "exit_code": 2 + } + ], + "requirements": "Synthetic bounded CLI demonstration. Preserve the approved stdout and exit-code oracles, reject invalid input explicitly, and use no network or external packages.", + "tests": "", + "fuzz_methods": [ + "boundary", + "mutation" + ], + "fuzz_cases": 0, + "seed": 42, + "benchmark_repeats": 1, + "mutation": false, + "fuzz_engine": "blackbox", + "fuzz_harness": "", + "priorities": { + "runtime": 0.4, + "memory": 0.2, + "readability": 0.25, + "footprint": 0.15, + "scrutiny": "strict" + } + }, + "result": { + "tool_status": "completed", + "compilation": { + "exit_code": 0, + "stdout": "No syntax errors detected in main.php\n", + "stderr": "", + "timeout": false, + "ms": 32.08741299999929, + "output_truncated": false + }, + "checks": { + "passed": 6, + "failed": 0, + "cases": [ + { + "name": "ordinary-quote", + "passed": true, + "exit_code": 0, + "stdout": "3600|0|4100\n", + "stderr": "", + "timeout": false, + "ms": 31.925749000009773, + "output_truncated": false + }, + { + "name": "bulk-discount", + "passed": true, + "exit_code": 0, + "stdout": "10000|500|10000\n", + "stderr": "", + "timeout": false, + "ms": 31.962915000008252, + "output_truncated": false + }, + { + "name": "zero-order", + "passed": true, + "exit_code": 0, + "stdout": "0|0|0\n", + "stderr": "", + "timeout": false, + "ms": 31.916494999990164, + "output_truncated": false + }, + { + "name": "half-cent-discount", + "passed": true, + "exit_code": 0, + "stdout": "10|1|9\n", + "stderr": "", + "timeout": false, + "ms": 31.922033000000738, + "output_truncated": false + }, + { + "name": "bounded-maximum", + "passed": true, + "exit_code": 0, + "stdout": "100000000|5000000|95100000\n", + "stderr": "", + "timeout": false, + "ms": 31.91297000000759, + "output_truncated": false + }, + { + "name": "malformed-price", + "passed": true, + "exit_code": 2, + "stdout": "", + "stderr": "", + "timeout": false, + "ms": 31.93240899997818, + "output_truncated": false + } + ] + }, + "oracle_hash": "e6675841c9c281c8a8ebdabb069d189dba8989d43bbb56a09e9d608903f886ab", + "environment": { + "worker": "polyglot-v3-language-evidence", + "python": "3.12.12", + "language": "php", + "machine": "x86_64", + "benchmark_repeats": 1 + }, + "generated_tests": { + "status": "not-provided" + }, + "benchmark": { + "samples_ms": [ + 31.945880000023408, + 31.87698599998612, + 31.94118199999707, + 31.875693999978694, + 31.906910999992988, + 31.92115300001319 + ], + "median_ms": 31.91403200000309, + "max_rss_kib": 24252, + "scope": "Process startup + workload; peak child RSS, not per-function memory. Exploratory measurements." + }, + "coverage": { + "status": "measured", + "tool": "PHP Xdebug", + "metric": "php-opcode-line", + "files": { + "main.php": { + "status": "measured", + "metric": "php-opcode-line", + "summary": { + "covered_lines": 11, + "num_statements": 11, + "missing_lines": 0, + "percent_covered": 100.0 + }, + "source_sha256": "4a32eb1188ca5630bdefdce166bbba6904981125c8a5b7c46c526e9115990c53", + "missing_lines": [], + "executed_lines": [ + 3, + 4, + 5, + 6, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "excluded_lines": [], + "missing_branches": [], + "branch_points": [], + "truncated": false + } + }, + "entry": { + "covered_lines": 11, + "num_statements": 11, + "missing_lines": 0, + "percent_covered": 100.0 + }, + "cases": [ + { + "name": "ordinary-quote", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false + }, + { + "name": "bulk-discount", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false + }, + { + "name": "zero-order", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false + }, + { + "name": "half-cent-discount", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false + }, + { + "name": "bounded-maximum", + "exit_code": 0, + "expected_exit_code": 0, + "timeout": false + }, + { + "name": "malformed-price", + "exit_code": 2, + "expected_exit_code": 2, + "timeout": false + } + ], + "requested_cases": 6, + "locations_truncated": false, + "scope": "Instrumented approved CLI reruns only, separate from timing/RSS and generated pytest tests. Unknown/unloaded sources remain not-measured. Branch points retain tool IDs, not invented destination lines. Shared-worker interference remains possible." + }, + "unused": { + "status": "completed", + "reason": null, + "tool": "PHPMD unusedcode", + "process": { + "exit_code": 0, + "stdout": "{\n \"version\": \"3.0.0\",\n \"package\": \"phpmd\",\n \"timestamp\": \"2026-11-10T17:56:34+00:00\",\n \"files\": []\n}\n", + "stderr": " 0/1 [\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591] 0%\n 1/1 [\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593\u2593] 100%\n", + "timeout": false, + "ms": 114.11100300000498, + "output_truncated": false + }, + "findings": [], + "findings_truncated": false, + "scope": "Review-only language-native unused candidates; public APIs, side effects and dynamic references require human review." + }, + "mutation": { + "status": "not-selected" + }, + "fuzz": { + "status": "completed", + "engine": "blackbox", + "executed": 0, + "seed": 42, + "findings": [], + "nonzero_policy": "Ordinary nonzero exits are not automatically bugs; approved expected behavior is required", + "blocked_methods": [], + "coverage_guided": false + }, + "diagnostics": [], + "diagnostics_truncated": false, + "unlocated_failures": [], + "location_caveat": "Tool-reported references, not proven root causes. Output may be incomplete, misleading or truncated; unrecognized paths remain in raw logs.", + "static": { + "sha256": "4a32eb1188ca5630bdefdce166bbba6904981125c8a5b7c46c526e9115990c53", + "source_bytes": 669, + "lines": 13, + "long_lines": 0, + "function_count": 0, + "branch_nodes": 0, + "findings": [], + "files": { + "main.php": { + "source_sha256": "4a32eb1188ca5630bdefdce166bbba6904981125c8a5b7c46c526e9115990c53" + } + }, + "findings_truncated": false, + "caveat": "Python syntax and conservative unused/unreachable candidates; other text uses risk patterns only. Unused is not uncovered. No inferred Big-O, complete bug detection or security certification." + }, + "isolation": { + "network": "none", + "memory_mib": 1024, + "cpus": 2, + "pids": 128, + "read_only": true + } + }, + "error": "Coverage line out of range" + }, + { + "language": "php", + "kind": "native-location", + "acceptance": "failed", + "project": { + "title": "php compiler contract", + "language": "php", + "entry": "main.php", + "files": [ + { + "name": "main.php", + "content": " +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# functions: complete_result(project)@L24; test_practical_matrix_is_complete_and_independent()@L80; test_positive_validator_rejects_incomplete_or_fabricated_evidence(defect)@L124; test_exact_compiler_location_and_nonexecution_are_required()@L178; test_runtime_failure_is_distinct_from_incorrect_oracle()@L211; test_real_execution_guard_refuses_this_pc_before_subprocess(monkeypatch)@L247; test_actual_first_hosted_reports_retain_their_real_tool_semantics(language, kind)@L258 +# variables: content@L26; case@L42; case@L46; case@L63; language@L84; project@L85; case@L87; duplicate@L91; project@L126; result@L127; project@L180; result@L181; defect@L199; changed@L200; project@L213; result@L214; wrong@L236; observed@L262; item@L265; row@L265; project@L266; changed@L271; changed@L275 +# parameters: project@L24; defect@L124; monkeypatch@L247; kind@L258; language@L258 +# + +"""Jason Stys: reject fabricated/incomplete compiler evidence without using local Docker.""" + +import copy +import hashlib +import json +import math +from pathlib import Path + +import pytest + +from examples.practical_projects import CASES, PROGRAMS, RUNTIME_FAILURES, SYNTAX_FAILURES, practical_project +from scripts.compiler_acceptance import negative_project, provenance, validate_negative, validate_positive +from studio.models import LANGUAGES + + +def complete_result(project): + """A minimal synthetic evidence record for validator mutation tests, never toolchain proof.""" + content = project.files[0].content + return { + "tool_status": "completed", + "compilation": {"exit_code": 0}, + "checks": { + "passed": len(project.cases), + "failed": 0, + "cases": [ + { + "name": case.name, + "stdout": case.stdout, + "exit_code": case.exit_code, + "passed": True, + "timeout": False, + "output_truncated": False, + } + for case in project.cases + ], + }, + "oracle_hash": hashlib.sha256( + json.dumps([case.model_dump() for case in project.cases], sort_keys=True).encode() + ).hexdigest(), + "environment": {"language": project.language}, + "benchmark": { + "samples_ms": [1.0] * len(project.cases) * project.benchmark_repeats, + "max_rss_kib": 1024, + }, + "coverage": { + "status": "measured", + "requested_cases": len(project.cases), + "cases": [ + { + "name": case.name, + "expected_exit_code": case.exit_code, + "exit_code": case.exit_code, + "timeout": False, + } + for case in project.cases + ], + "files": { + project.entry: { + "status": "measured", + "source_sha256": hashlib.sha256(content.encode()).hexdigest(), + "executed_lines": [1], + "missing_lines": [], + "summary": {"covered_lines": 1, "missing_lines": 0, "num_statements": 1}, + } + }, + }, + "unused": {"status": "completed"}, + "static": {"files": {project.entry: {"parsed": True}}}, + } + + +def test_practical_matrix_is_complete_and_independent(): + """Every executable adapter has several explicit independent oracles and both control sources.""" + assert set(PROGRAMS) == set(CASES) == set(RUNTIME_FAILURES) == set(LANGUAGES) + assert set(SYNTAX_FAILURES) == set(LANGUAGES) - {"sql"} + for language in LANGUAGES: + project = practical_project(language) + assert project.language == language and len(project.cases) == 6 + assert len({case.name for case in project.cases}) == 6 + assert project.cases[0].exit_code == 0 + assert project.fuzz_cases == 0 and project.benchmark_repeats == 1 + validate_positive(complete_result(project), project) + duplicate = practical_project(language) + duplicate.cases[0].stdout = "changed only this instance" + assert practical_project(language).cases[0].stdout != duplicate.cases[0].stdout + + +@pytest.mark.parametrize( + "defect", + [ + "blocked", + "compile", + "missing-case", + "wrong-output", + "wrong-exit", + "false-pass", + "timeout", + "truncated-output", + "wrong-oracle-hash", + "wrong-adapter", + "missing-sample", + "nan-time", + "missing-rss", + "partial-coverage", + "wrong-source", + "unmeasured-entry", + "bad-line", + "missing-unused", + "truncated-locations", + "overlapping-coverage", + "duplicate-coverage", + "inconsistent-coverage", + "truncated-compilation", + ], +) +def test_positive_validator_rejects_incomplete_or_fabricated_evidence(defect): + """Mutate concrete report fields so a green workflow cannot mean just 'worker returned JSON'.""" + project = practical_project("typescript") + result = complete_result(project) + if defect == "blocked": + result["tool_status"] = "blocked" + elif defect == "compile": + result["compilation"]["exit_code"] = 2 + elif defect == "missing-case": + result["checks"]["cases"].pop() + elif defect == "wrong-output": + result["checks"]["cases"][0]["stdout"] = "incorrect" + elif defect == "wrong-exit": + result["checks"]["cases"][0]["exit_code"] = 9 + elif defect == "false-pass": + result["checks"]["cases"][0]["passed"] = False + elif defect == "timeout": + result["checks"]["cases"][0]["timeout"] = True + elif defect == "truncated-output": + result["checks"]["cases"][0]["output_truncated"] = True + elif defect == "wrong-oracle-hash": + result["oracle_hash"] = "0" * 64 + elif defect == "wrong-adapter": + result["environment"]["language"] = "python" + elif defect == "missing-sample": + result["benchmark"]["samples_ms"].pop() + elif defect == "nan-time": + result["benchmark"]["samples_ms"][0] = math.nan + elif defect == "missing-rss": + result["benchmark"]["max_rss_kib"] = None + elif defect == "partial-coverage": + result["coverage"]["status"] = "partial" + elif defect == "wrong-source": + result["coverage"]["files"][project.entry]["source_sha256"] = "0" * 64 + elif defect == "unmeasured-entry": + result["coverage"]["files"][project.entry]["executed_lines"] = [] + elif defect == "bad-line": + result["coverage"]["files"][project.entry]["missing_lines"] = [999999] + elif defect == "missing-unused": + result["unused"]["status"] = "blocked" + elif defect == "truncated-locations": + result["diagnostics_truncated"] = True + elif defect == "overlapping-coverage": + result["coverage"]["files"][project.entry]["missing_lines"] = [1] + elif defect == "duplicate-coverage": + result["coverage"]["files"][project.entry]["executed_lines"] = [1, 1] + elif defect == "inconsistent-coverage": + result["coverage"]["files"][project.entry]["summary"]["num_statements"] = 99 + elif defect == "truncated-compilation": + result["compilation"]["output_truncated"] = True + with pytest.raises(ValueError): + validate_positive(result, project) + + +def test_exact_compiler_location_and_nonexecution_are_required(): + """A malformed C++ target must fail compilation at its authored line before any case executes.""" + project = negative_project(practical_project("cpp"), "syntax-error") + result = { + "tool_status": "compile-error", + "compilation": {"exit_code": 1}, + "checks": {"passed": 0, "failed": 0, "cases": []}, + "coverage": {"status": "blocked"}, + "unused": {"status": "blocked"}, + "environment": {"language": project.language}, + "oracle_hash": complete_result(project)["oracle_hash"], + "diagnostics": [ + { + "file": project.entry, + "line": SYNTAX_FAILURES["cpp"]["line"], + "source_sha256": hashlib.sha256(project.files[0].content.encode()).hexdigest(), + "stage": "compilation", + } + ], + } + validate_negative(result, project, "syntax-error") + for defect in ("missing-location", "executed-uncompiled", "invented-coverage"): + changed = copy.deepcopy(result) + if defect == "missing-location": + changed["diagnostics"] = [] + elif defect == "executed-uncompiled": + changed["checks"]["cases"] = [{"passed": False}] + else: + changed["coverage"]["status"] = "measured" + with pytest.raises(ValueError): + validate_negative(changed, project, "syntax-error") + + +def test_runtime_failure_is_distinct_from_incorrect_oracle(): + """A failing expected-output comparison alone cannot prove a program runtime fault.""" + project = negative_project(practical_project("python"), "runtime-error") + result = { + "tool_status": "completed", + "compilation": {"exit_code": 0}, + "environment": {"language": project.language}, + "oracle_hash": complete_result(project)["oracle_hash"], + "checks": { + "passed": 0, + "failed": 1, + "cases": [ + { + "name": "runtime-error", + "passed": False, + "exit_code": 1, + "stderr": "RuntimeError: Synthetic runtime failure", + } + ], + }, + } + validate_negative(result, project, "runtime-error") + result["checks"]["cases"][0]["exit_code"] = 0 + with pytest.raises(ValueError): + validate_negative(result, project, "runtime-error") + wrong = negative_project(practical_project("python"), "wrong-oracle") + result["oracle_hash"] = complete_result(wrong)["oracle_hash"] + result["checks"]["cases"][0].update( + {"name": "wrong-oracle", "stdout": practical_project("python").cases[0].stdout} + ) + validate_negative(result, wrong, "wrong-oracle") + result["checks"]["cases"][0]["stdout"] = "A second independent implementation bug" + with pytest.raises(ValueError, match="implementation/output fault"): + validate_negative(result, wrong, "wrong-oracle") + + +def test_real_execution_guard_refuses_this_pc_before_subprocess(monkeypatch): + """No local Docker command runs when the hosted acceptance entry point is invoked accidentally.""" + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + with pytest.raises(ValueError, match="GitHub-hosted Linux"): + provenance(Path.cwd(), "0" * 40, "release-0.4.2") + + +@pytest.mark.parametrize( + "language,kind", + [("python", "positive"), ("php", "positive"), ("php", "native-location"), ("csharp", "positive")], +) +def test_actual_first_hosted_reports_retain_their_real_tool_semantics(language, kind): + """Replay observed source-bound reports; these regressions are not fresh compiler execution.""" + from studio.models import Project + + observed = json.loads( + (Path(__file__).parent / "fixtures/compiler-reports-042.json").read_text(encoding="utf-8") + ) + row = next(item for item in observed["reports"] if item["language"] == language and item["kind"] == kind) + project = Project.model_validate(row["project"]) + validate_positive(row["result"], project) + if language == "csharp": + # Do not disguise the collector's expected nonzero wrapper result as full coverage. + assert row["result"]["coverage"]["status"] == "partial" + changed = copy.deepcopy(row["result"]) + changed["coverage"]["status"] = "measured" + with pytest.raises(ValueError, match="collector exits"): + validate_positive(changed, project) + changed = copy.deepcopy(row["result"]) + changed["coverage"]["cases"][0]["exit_code"] = 1 + with pytest.raises(ValueError, match="Unexpected instrumented"): + validate_positive(changed, project) diff --git a/tests/test_released_service_probe.py b/tests/test_released_service_probe.py new file mode 100644 index 0000000..cebbcda --- /dev/null +++ b/tests/test_released_service_probe.py @@ -0,0 +1,351 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# functions: test_hosted_guard_refuses_user_pc_before_any_work(monkeypatch, tmp_path)@L27; test_hosted_output_is_confined_and_verifier_source_explicit(monkeypatch, tmp_path)@L36; metadata()@L56; test_publication_or_replaced_asset_never_authorizes_execution(defect)@L95; test_download_binds_canonical_hosts_actual_bytes_and_digest(tmp_path)@L117; transport(request)@L129; archive(path, monkeypatch, extra, bad_schema)@L153; test_real_tar_provenance_and_no_setuid_preservation(monkeypatch, tmp_path)@L180; test_actual_tar_negative_members_fail_closed(monkeypatch, tmp_path, kind)@L194; test_service_environment_cannot_inherit_provider_cloud_or_github_keys(monkeypatch, tmp_path)@L219; test_failed_startup_cleanup_targets_only_owned_child_and_preserves_live_recovery(tmp_path, can_exit)@L230; terminate()@L238; wait(timeout)@L241; test_actual_store_timeline_identity_and_shape_fail_closed(tmp_path, defect)@L255; test_incremental_receipt_and_full_restored_export_equality(monkeypatch, tmp_path)@L283; request(_client, method, path, body)@L291; validate(result, project)@L331 +# variables: name@L39; value@L39; prefix@L58; rows@L59; i@L77; rows@L77; release@L97; tag@L97; body@L119; initial@L120; row@L121; seen@L127; client@L143; digest@L144; worker@L155; value@L157; output@L165; body@L166; name@L166; member@L171; content@L176; member@L176; source@L181; service@L183; source@L195; member@L196; content@L199; content@L204; result@L223; key@L225; temporary@L232; calls@L235; state@L236; process@L248; store@L257; revision@L258; identifier@L259; result@L263; exported@L264; timeline@L266; data@L285; store@L287; validations@L288; tamper@L289; revision@L300; identifier@L301; project@L302; result@L303; identifier@L311; result@L312; evidence@L337; service@L338; private@L339; row@L341; text@L342 +# parameters: monkeypatch@L27; tmp_path@L27; monkeypatch@L36; tmp_path@L36; defect@L95; tmp_path@L117; request@L129; bad_schema@L153; extra@L153; monkeypatch@L153; path@L153; monkeypatch@L180; tmp_path@L180; kind@L194; monkeypatch@L194; tmp_path@L194; monkeypatch@L219; tmp_path@L219; can_exit@L230; tmp_path@L230; timeout@L241; defect@L255; tmp_path@L255; monkeypatch@L283; tmp_path@L283; _client@L291; body@L291; method@L291; path@L291; project@L331; result@L331 +# + +"""Jason Stys: published-package/ownership/protocol negatives; no local Docker, network or service runs.""" + +import copy +import gzip +import hashlib +import io +import json +import os +import tarfile +from types import SimpleNamespace + +import httpx +import pytest + +from scripts import released_service_probe as probe +from studio.store import Store + + +def test_hosted_guard_refuses_user_pc_before_any_work(monkeypatch, tmp_path): + """Even spoofed Actions variables cannot make a Windows machine an allowed Linux runner.""" + monkeypatch.setattr(probe.sys, "platform", "win32") + monkeypatch.setenv("GITHUB_ACTIONS", "true") + with pytest.raises(ValueError, match="GitHub-hosted Linux"): + probe.hosted_output(tmp_path / "evidence") + assert not (tmp_path / "evidence").exists() + + +def test_hosted_output_is_confined_and_verifier_source_explicit(monkeypatch, tmp_path): + """A valid disposable context still cannot select a report outside its workspace.""" + monkeypatch.setattr(probe.sys, "platform", "linux") + for name, value in { + "GITHUB_ACTIONS": "true", + "RUNNER_OS": "Linux", + "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_REPOSITORY": probe.REPOSITORY, + "GITHUB_WORKSPACE": str(tmp_path), + "GITHUB_SHA": "a" * 40, + }.items(): + monkeypatch.setenv(name, value) + assert probe.hosted_output(tmp_path / "artifacts") == tmp_path / "artifacts/released-service-probe.json" + with pytest.raises(ValueError, match="inside"): + probe.hosted_output(tmp_path.parent / "outside-receipt") + monkeypatch.setenv("GITHUB_SHA", "main") + with pytest.raises(ValueError, match="identity"): + probe.hosted_output(tmp_path / "artifacts") + + +def metadata(): + """Independent real-shaped public metadata; unrelated assets are never downloaded.""" + prefix = "https://github.com/" + probe.REPOSITORY + "/releases/download/v0.4.2/" + rows = [ + { + "id": probe.PAYLOAD_ID, + "name": probe.PAYLOAD, + "size": probe.PAYLOAD_BYTES, + "digest": "sha256:" + probe.PAYLOAD_SHA256, + "state": "uploaded", + "browser_download_url": prefix + probe.PAYLOAD, + }, + { + "id": 2, + "name": probe.PAYLOAD + ".sha256", + "size": 104, + "digest": "sha256:" + "b" * 64, + "state": "uploaded", + "browser_download_url": prefix + probe.PAYLOAD + ".sha256", + }, + ] + rows += [{"id": i + 10, "name": "other-accepted-asset-" + str(i)} for i in range(10)] + return ( + { + "id": probe.RELEASE_ID, + "tag_name": "v0.4.2", + "draft": False, + "prerelease": False, + "target_commitish": probe.SOURCE, + "published_at": "2026-11-10T15:00:00Z", + "assets": rows, + }, + {"ref": "refs/tags/v0.4.2", "object": {"type": "commit", "sha": probe.SOURCE}}, + ) + + +@pytest.mark.parametrize( + "defect", ["draft", "source", "tag", "replaced", "duplicate", "large-sidecar", "foreign-url"] +) +def test_publication_or_replaced_asset_never_authorizes_execution(defect): + """Valid tag/version strings alone cannot authorize a different draft, source or binary.""" + release, tag = metadata() + probe.release_assets(release, tag) + if defect == "draft": + release["draft"] = True + elif defect == "source": + release["target_commitish"] = "main" + elif defect == "tag": + tag["object"]["sha"] = "c" * 40 + elif defect == "replaced": + release["assets"][0]["digest"] = "sha256:" + "c" * 64 + elif defect == "duplicate": + release["assets"][-1] = copy.deepcopy(release["assets"][0]) + elif defect == "large-sidecar": + release["assets"][1]["size"] = 4097 + else: + release["assets"][0]["browser_download_url"] = "https://example.invalid/payload" + with pytest.raises(ValueError): + probe.release_assets(release, tag) + + +def test_download_binds_canonical_hosts_actual_bytes_and_digest(tmp_path): + """A real streamed mock response exercises redirect scope and content checks, not the internet.""" + body = b"synthetic archive bytes" + initial = metadata()[0]["assets"][0]["browser_download_url"] + row = { + "name": probe.PAYLOAD, + "size": len(body), + "digest": "sha256:" + hashlib.sha256(body).hexdigest(), + "browser_download_url": initial, + } + seen = [] + + def transport(request): + seen.append(request) + assert "authorization" not in request.headers and "cookie" not in request.headers + if request.url.host == "github.com": + return httpx.Response( + 302, + headers={ + "location": "https://release-assets.githubusercontent.com/github-production-release-asset/owned/file?signature=synthetic" + }, + ) + return httpx.Response(200, stream=httpx.ByteStream(body)) + + with httpx.Client( + transport=httpx.MockTransport(transport), trust_env=False, follow_redirects=False + ) as client: + digest = probe.download(client, row, tmp_path / "payload") + assert len(seen) == 2 and digest == hashlib.sha256(body).hexdigest() + assert (tmp_path / "payload").read_bytes() == body + with pytest.raises(ValueError, match="Foreign"): + probe.download_url("https://example.invalid/payload", initial) + with pytest.raises(ValueError, match="Unknown canonical"): + probe.download_url(initial, "https://github.com/other/repo/releases/download/v1/file") + + +def archive(path, monkeypatch, extra=None, bad_schema=False): + """Create actual small compressed tar bytes; the inert service is never launched.""" + worker = b"synthetic worker bytes" + monkeypatch.setattr(probe, "WORKER_SHA256", hashlib.sha256(worker).hexdigest()) + value = { + "schema": True if bad_schema else 1, + "product": "code-flow-studio", + "version": "0.4.2", + "source": probe.SOURCE, + "source_dirty": False, + "worker_sha256": probe.WORKER_SHA256, + } + with tarfile.open(path, "w:gz") as output: + for name, body in { + "resources/metadata/source-release.json": json.dumps(value).encode(), + "resources/runner/node_checks.cjs": worker, + "resources/service/code-flow-service": b"inert-not-executable-code", + }.items(): + member = tarfile.TarInfo(name) + member.size = len(body) + member.mode = 0o755 + output.addfile(member, io.BytesIO(body)) + if extra: + member, content = extra + output.addfile(member, io.BytesIO(content)) + + +def test_real_tar_provenance_and_no_setuid_preservation(monkeypatch, tmp_path): + source = tmp_path / "archive.tar.gz" + archive(source, monkeypatch) + service = probe.unpack(source, tmp_path / "unpacked") + assert service.read_bytes() == b"inert-not-executable-code" + if os.name == "posix": + assert service.stat().st_mode & 0o7777 == 0o700 + else: + assert service.stat().st_mode & 0o7000 == 0 + + +@pytest.mark.parametrize( + "kind", ["traversal", "absolute", "symlink", "hardlink", "duplicate", "schema-bool", "oversize-header"] +) +def test_actual_tar_negative_members_fail_closed(monkeypatch, tmp_path, kind): + source = tmp_path / "archive.tar.gz" + member = tarfile.TarInfo( + "../outside" if kind == "traversal" else "/outside" if kind == "absolute" else "extra" + ) + content = b"x" + member.size = 1 + if kind in {"symlink", "hardlink"}: + member.type = tarfile.SYMTYPE if kind == "symlink" else tarfile.LNKTYPE + member.linkname, member.size = "../../outside", 0 + content = b"" + if kind == "duplicate": + member.name = "resources/runner/node_checks.cjs" + if kind == "oversize-header": + member.size = probe.MAX_MEMBER + 1 + source.write_bytes(gzip.compress(member.tobuf() + bytes(1024))) + else: + archive( + source, monkeypatch, (member, content) if kind != "schema-bool" else None, kind == "schema-bool" + ) + with pytest.raises((ValueError, tarfile.TarError)): + probe.unpack(source, tmp_path / "unpacked") + assert not (tmp_path / "outside").exists() + + +def test_service_environment_cannot_inherit_provider_cloud_or_github_keys(monkeypatch, tmp_path): + monkeypatch.setenv("STUDIO_OPENAI_KEY", "synthetic-private-provider") + monkeypatch.setenv("GH_TOKEN", "synthetic-private-github") + monkeypatch.setenv("DOCKER_CONTEXT", "unexpected-remote") + result = probe.service_environment(tmp_path, "synthetic-launch-token", "synthetic-nonce") + assert result["DOCKER_HOST"] == "unix:///var/run/docker.sock" + assert not any(key.startswith(("STUDIO_", "GH_", "GITHUB_", "AWS_", "AZURE_")) for key in result) + assert "DOCKER_CONTEXT" not in result + + +@pytest.mark.parametrize("can_exit", [True, False]) +def test_failed_startup_cleanup_targets_only_owned_child_and_preserves_live_recovery(tmp_path, can_exit): + """A failed pre-auth launch cannot orphan a stopped child or delete data used by a live child.""" + temporary = tmp_path / "cfs-released042-owned-test" + temporary.mkdir() + (temporary / "private-recovery").write_bytes(b"synthetic-only") + calls = [] + state = {"alive": True} + + def terminate(): + calls.append("terminate-owned") + + def wait(timeout): + assert timeout == 10 + calls.append("wait-owned") + if not can_exit: + raise TimeoutError("synthetic failure") + state["alive"] = False + + process = SimpleNamespace(poll=lambda: None if state["alive"] else 0, terminate=terminate, wait=wait) + assert probe.cleanup_owned([SimpleNamespace(process=process, client=None)], temporary) is can_exit + assert calls == ["terminate-owned", "wait-owned"] + assert temporary.exists() is not can_exit + + +@pytest.mark.parametrize("defect", ["list", "project", "revision", "run", "checks", "truncated"]) +def test_actual_store_timeline_identity_and_shape_fail_closed(tmp_path, defect): + """Use real SQLite timeline output, then alter one independent identity or evidence field.""" + store = Store(tmp_path / "timeline.sqlite") + revision = store.create(probe.practical_project("python").model_dump()) + identifier = store.queue(revision["id"]) + store.finish( + identifier, "completed", {"checks": {"passed": 6, "failed": 0}, "coverage": {"status": "measured"}} + ) + result = store.run(identifier) + exported = {**result, "iteration_history": store.timeline(revision["id"])} + probe.validate_export(exported, result, revision) + timeline = exported["iteration_history"] + if defect == "list": + exported["iteration_history"] = timeline["revisions"] + elif defect == "project": + timeline["project_id"] = "another-project" + elif defect == "revision": + timeline["revisions"][0]["id"] = "another-revision" + elif defect == "run": + timeline["revisions"][0]["attempts"][0]["id"] = "another-run" + elif defect == "checks": + timeline["revisions"][0]["attempts"][0]["checks"]["failed"] = 1 + else: + timeline["truncated"] = True + with pytest.raises(ValueError, match="JSON export timeline"): + probe.validate_export(exported, result, revision) + + +def test_incremental_receipt_and_full_restored_export_equality(monkeypatch, tmp_path): + """Mock only HTTP/compilation for this lifecycle unit; real frozen/worker proof remains hosted.""" + data = tmp_path / "account-data" + data.mkdir() + store = Store(tmp_path / "actual-store.sqlite") + validations = [] + tamper = {"export": False} + + def request(_client, method, path, body=None): + if path in {"/private/register", "/private/login"}: + return {"recovery": "synthetic-do-not-log-recovery"} + if path == "/private/unlock": + (data / "active").mkdir(exist_ok=True) + return {"unlocked": True} + if path == "/private/api/revisions" and method == "POST": + return store.create(body) + if path.endswith("/run"): + revision = path.split("/")[-2] + identifier = store.queue(revision) + project = store.revision(revision)["project"] + result = { + "compilation": {"exit_code": 0}, + "checks": {"passed": len(project["cases"]), "failed": 0}, + "coverage": {"status": "measured"}, + } + store.finish(identifier, "completed", result) + return {"id": identifier} + if path.startswith("/private/api/runs/"): + identifier = path.split("/")[4] + result = store.run(identifier) + if "?format=json" in path: + result["iteration_history"] = store.timeline(result["revision_id"]) + if tamper["export"]: + result["iteration_history"]["revisions"][0]["attempts"][0]["coverage"] = { + "status": "changed" + } + return result + if path == "/private/api/revisions": + return store.history() + if path == "/private/lock": + (data / "active").rmdir() + (data / "encrypted").mkdir(exist_ok=True) + (data / "encrypted/synthetic.cfsvault").write_bytes(b"synthetic-sealed-fixture") + return {"locked": True} + if path == "/private/save": + return {"saved": True} + raise AssertionError("Unexpected route") + + def validate(result, project): + assert result["compilation"]["exit_code"] == 0 and result["checks"]["passed"] == len(project.cases) + validations.append(project.language) + + monkeypatch.setattr(probe, "request_json", request) + monkeypatch.setattr(probe, "validate_positive", validate) + evidence = probe.Evidence(tmp_path / "receipts/report.json") + service = SimpleNamespace(client=object()) + private = probe.exercise(service, data, evidence) + assert len(validations) == len(store.history()) == 12 + assert sum(row["approved_cases"] for row in evidence.record["languages"]) == 72 + text = evidence.output.read_text() + assert ( + private["password"] not in text + and private["key"]["key"] not in text + and "synthetic-do-not-log-recovery" not in text + ) + probe.restored(service, data, private) + tamper["export"] = True + with pytest.raises(ValueError, match="export differs"): + probe.restored(service, data, private)