diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index 3766a166a14..f19dafd13fc 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -850,6 +850,65 @@ jobs: echo "::notice::Created metadata file: $METADATA_FILE (py${PYTHON_VERSION}, ${LIBRARY}==${LIBRARY_VERSION})" echo "::notice::Implementation verified present, metadata committed" + # ======================================================================== + # Adaptability smoke run (record-only, Python only) + # ======================================================================== + # docs/concepts/adaptable-code.md, "Perturbation smoke run" and "The adapt + # route": the new source runs on perturbed tables (core/smoke_run.py run); + # on a regeneration the predecessor from origin/main is rendered in this + # same runner next to a fresh render of the new source (pair: both + # themes, `python -P {library}.py` with the theme env, as the generator's + # Step 3 renders), so impl-review's gate can measure render identity on + # raw renders from one environment, before "Process plot images" + # quantizes anything. Results land in $RUNNER_TEMP/adapt and "Upload to + # GCS Staging" stages them privately under adapt/. Record-only: no exit + # code gates, continue-on-error keeps a crash or a timeout from failing + # the job, and a missing artifact reads as `missing` in impl-review (no + # repair trigger, design "No repair trigger"). The other languages have + # no runner yet (design "Rollout" step 6), so the step skips them. + - name: Adaptability smoke run (record-only) + id: adapt_smoke + if: steps.inputs.outputs.language == 'python' + continue-on-error: true + timeout-minutes: 20 + env: + SPEC_ID: ${{ steps.inputs.outputs.specification_id }} + LIBRARY: ${{ steps.inputs.outputs.library }} + run: | + OUT="$RUNNER_TEMP/adapt"; TOOLS="$RUNNER_TEMP/adapt-tools" + rm -rf "$OUT" "$TOOLS"; mkdir -p "$OUT/prev-src" "$TOOLS" + # The runner comes from this workflow's ref, never from the branch the + # generator wrote (the pattern of the regen_gate_header copy in "Create + # library metadata file"). Without it nothing runs and nothing is staged. + if ! git archive "$GITHUB_SHA" core | tar -x -C "$TOOLS" || [ ! -f "$TOOLS/core/smoke_run.py" ]; then + echo "::warning::adapt_smoke could not copy core/ from ${GITHUB_SHA}; no adaptability record is staged" + rm -rf "$OUT" + exit 0 + fi + RUN=(env PYTHONPATH="$TOOLS" .venv/bin/python -P -m core.smoke_run) + IMPL="plots/${SPEC_ID}/implementations/python/${LIBRARY}.py" + SPEC="plots/${SPEC_ID}/specification.md" + "${RUN[@]}" run "$IMPL" --spec "$SPEC" --spec-id "$SPEC_ID" --library "$LIBRARY" \ + --out "$OUT/new-smoke.json" || true + # A regeneration is the implementation file on origin/main, the test + # impl-review uses. The predecessor is written under its library's file + # name, so the runner applies that library's setup calls. + if git cat-file -e "origin/main:${IMPL}" 2>/dev/null; then + git show "origin/main:${IMPL}" > "$OUT/prev-src/${LIBRARY}.py" + # New first, then the predecessor: the generated code has exited + # before the predecessor's PNGs exist. + "${RUN[@]}" pair --new "$IMPL" --prev "$OUT/prev-src/${LIBRARY}.py" --library "$LIBRARY" \ + --out-dir "$OUT" --out "$OUT/pair.json" || true + "${RUN[@]}" run "$OUT/prev-src/${LIBRARY}.py" --spec "$SPEC" --spec-id "$SPEC_ID" --library "$LIBRARY" \ + --skip-if-static-fails --out "$OUT/prev-smoke.json" || true + fi + # The generator's Step 3c attempt log, when it ran the self-check. + if [ -f /tmp/anyplot-selfcheck.jsonl ]; then + cp /tmp/anyplot-selfcheck.jsonl "$OUT/selfcheck.jsonl" + fi + rm -rf "$OUT/prev-src" + echo "::notice::adapt_smoke spec=${SPEC_ID} lib=${LIBRARY} staged=$(cd "$OUT" && find . -type f | sed 's|^\./||' | sort | tr '\n' ' ')" + # ======================================================================== # Process images: optimize + thumbnail # ======================================================================== @@ -1014,6 +1073,23 @@ jobs: STAGING_PATH="gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}" PUBLIC_URL="https://storage.googleapis.com/anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}" + # Adaptability artifacts from "Adaptability smoke run (record-only)". + # An earlier run of this pair is cleared first, unconditionally, so a + # kept regeneration's prev-* and pair.json never outlive it. They stay + # private (no AllUsers ACL) and impl-merge never promotes them. + # rsync, not `cp -r dir/*`: copying a directory into a destination + # subdirectory that does not exist yet drops the directory's name + # (gsutil's naming rule), so new/plot-light.png and + # prev/plot-light.png could both land as adapt/plot-light.png. + # rsync names every object by its path under $ADAPT. + gsutil -m rm -r "${STAGING_PATH}/adapt" 2>/dev/null || true + ADAPT="$RUNNER_TEMP/adapt" + rm -rf "$ADAPT/prev-src" # left behind only when the smoke step was cut off + if [ -d "$ADAPT" ] && [ -n "$(ls -A "$ADAPT")" ]; then + gsutil -m -h "Cache-Control:no-store" rsync -r "$ADAPT" "${STAGING_PATH}/adapt" \ + || echo "::warning::adapt artifacts not staged; impl-review records them as missing" + fi + # Require both theme renders if [ ! -f "$IMPL_DIR/plot-light.png" ] || [ ! -f "$IMPL_DIR/plot-dark.png" ]; then echo "::error::Missing plot-light.png and/or plot-dark.png — cannot upload" diff --git a/.github/workflows/impl-merge.yml b/.github/workflows/impl-merge.yml index 896e0a35c08..2311b7f453e 100644 --- a/.github/workflows/impl-merge.yml +++ b/.github/workflows/impl-merge.yml @@ -413,8 +413,12 @@ jobs: STAGING="gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}" PRODUCTION="gs://anyplot-images/plots/${SPEC_ID}/${LANGUAGE}/${LIBRARY}" - # Copy from staging to production - gsutil -m -h "Cache-Control:public, max-age=604800" cp -r "${STAGING}/*" "${PRODUCTION}/" 2>/dev/null || echo "No staging files to promote" + # Copy from staging to production: the renders only (`plot-*`, no + # recursion). staging/.../adapt/ holds the smoke runs and the + # predecessor's raw renders that impl-generate staged for + # impl-review; they stay private, and the cleanup below removes + # them with the rest. + gsutil -m -h "Cache-Control:public, max-age=604800" cp "${STAGING}/plot-*" "${PRODUCTION}/" 2>/dev/null || echo "No staging files to promote" # Make production files public gsutil -m acl ch -r -u AllUsers:R "${PRODUCTION}/" 2>/dev/null || true diff --git a/.github/workflows/impl-repair.yml b/.github/workflows/impl-repair.yml index 52a47f0c17b..c91dc9810a0 100644 --- a/.github/workflows/impl-repair.yml +++ b/.github/workflows/impl-repair.yml @@ -293,6 +293,33 @@ jobs: echo "::notice::Processed both themes after repair (light + dark)" ls -la "$IMPL_DIR/" + # The perturbation smoke run of the repaired file (core/smoke_run.py run), + # as impl-generate runs it on the generated one; "Upload repaired plot to + # GCS staging" replaces the staged adapt/new-smoke.json with it. No pair + # and no predecessor run: a gated regeneration never enters repair. + # Record-only, like impl-generate's step: no exit code gates, and + # continue-on-error keeps a crash or a timeout from failing the repair. + - name: Adaptability smoke run (record-only) + id: adapt_smoke + if: success() && steps.lang.outputs.language == 'python' + continue-on-error: true + timeout-minutes: 20 + env: + SPEC_ID: ${{ inputs.specification_id }} + LIBRARY: ${{ inputs.library }} + run: | + OUT="$RUNNER_TEMP/adapt"; TOOLS="$RUNNER_TEMP/adapt-tools" + rm -rf "$OUT" "$TOOLS"; mkdir -p "$OUT" "$TOOLS" + # The runner comes from this workflow's ref, never from the PR branch. + if ! git archive "$GITHUB_SHA" core | tar -x -C "$TOOLS" || [ ! -f "$TOOLS/core/smoke_run.py" ]; then + echo "::warning::adapt_smoke could not copy core/ from ${GITHUB_SHA}; no adaptability record is staged" + exit 0 + fi + IMPL="plots/${SPEC_ID}/implementations/python/${LIBRARY}.py" + env PYTHONPATH="$TOOLS" .venv/bin/python -P -m core.smoke_run run "$IMPL" \ + --spec "plots/${SPEC_ID}/specification.md" --spec-id "$SPEC_ID" --library "$LIBRARY" \ + --out "$OUT/new-smoke.json" || true + - name: Authenticate to GCP if: success() uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 @@ -331,6 +358,20 @@ jobs: fi done + # The smoke run of the repaired file replaces the staged one (private, + # never promoted). Without a new one the stale record from before the + # repair goes, so impl-review reads `missing`, never a result for code + # that no longer exists. + if [ "$LANGUAGE" = "python" ]; then + SMOKE="$RUNNER_TEMP/adapt/new-smoke.json" + if [ -s "$SMOKE" ] && gsutil -h "Cache-Control:no-store" cp "$SMOKE" "${STAGING_PATH}/adapt/new-smoke.json"; then + echo "::notice::Staged the repaired file's adapt/new-smoke.json" + else + [ -s "$SMOKE" ] && echo "::warning::adapt/new-smoke.json not staged; impl-review records it as missing" + gsutil rm "${STAGING_PATH}/adapt/new-smoke.json" 2>/dev/null || true + fi + fi + rm -f /tmp/gcs-key.json - name: Post repair status diff --git a/.github/workflows/impl-review.yml b/.github/workflows/impl-review.yml index 86a8db827b4..863e1c07b2a 100644 --- a/.github/workflows/impl-review.yml +++ b/.github/workflows/impl-review.yml @@ -53,6 +53,16 @@ jobs: with: fetch-depth: 0 + # The regen tools (regen_gate.py, review_provenance.py, the adaptability + # checker mirror) carry PEP 758 `except A, B:` since the move to Python + # 3.14 (#12247), which an older interpreter cannot parse; the runner + # image's python3 is not pinned. From here on every `python3` and `pip` + # in this job is the version in this ref's .python-version. + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: '.python-version' + - name: Extract PR info id: pr env: @@ -244,9 +254,12 @@ jobs: # Keep the regen helpers from THIS workflow's ref before switching to # the PR head: the gate's parser must match the workflow that calls # it, and a PR branched off an older main does not carry them. + # The adaptability checker and its record builder join them for the + # same reason: a PR head must not choose the rules it is checked by. mkdir -p "$RUNNER_TEMP/regen-tools" cp automation/scripts/regen_gate.py automation/scripts/close_issue_if_complete.py \ automation/scripts/review_provenance.py automation/scripts/regen_writeback.py \ + automation/scripts/adaptability_check.py automation/scripts/adapt_record.py \ "$RUNNER_TEMP/regen-tools/" # The reviewer's step-10 self-check (`check-feedback`) runs this # same workflow-ref copy of the gate. @@ -426,7 +439,11 @@ jobs: # someone re-dispatched the review by hand (#11360 on 2026-09-05, # #11678 on 2026-09-09; staging was complete both times). Keep stderr: # a transfer blip and a missing staging folder need different answers. - SRC="gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}/*" + # `plot-*` only: everything the reviewer needs starts with `plot-`, + # and the adapt/ artifacts beside it (the predecessor's raw renders + # among them) must never land in the folder the blind review reads. + # "Adaptability check (record-only)" fetches those on its own. + SRC="gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}/plot-*" for attempt in 1 2 3; do if gsutil -m cp "$SRC" plot_images/ 2> /tmp/gsutil.err \ && [ -f plot_images/plot-light.png ] && [ -f plot_images/plot-dark.png ]; then @@ -613,6 +630,184 @@ jobs: print(f"::warning::Canvas gate FAILED — wrote synthetic weakness to /tmp/anyplot-canvas-gate.txt: {weakness}") PY + # Record-only (docs/concepts/adaptable-code.md, "Pipeline changes"). The + # checker mirror runs on the PR's implementation and, on a regeneration, + # on its predecessor from origin/main; with the smoke runs impl-generate + # staged under staging/.../adapt/, each side becomes one record: + # /tmp/anyplot-adapt-new.json and /tmp/anyplot-adapt-prev.json. Only the + # regen gate reads them (the adapt route and the adapt regression); the + # review prompt never mentions them, and the staged predecessor renders + # stay under $RUNNER_TEMP, outside the reviewer's /tmp/* reads. Never a + # raw `exit 1` (see "Canvas dimension gate"): a failure here only means + # no record, and the gate then decides as it did before. + # On a regeneration the step also asks the gate whether the adapt route + # holds before any review (`adapt-route`), and skips the review only when + # the route holds, the owner's ADAPT_REVIEW_FREE variable is `true`, and + # nothing else needs a reviewer: the stored score is an integer above 0, + # the regen context is complete, no change request is pending and the + # canvas gate passed. review_skipped is the step's last write, so a step + # that fails or times out before it leaves the review running. + # The mirror parses the PR's source with `ast` on the pinned python3 + # ("Set up Python"): ruff writes PEP 758 `except A, B:` for py314, which + # an older interpreter would report as `layout: does not parse`. + - name: Adaptability check (record-only) + id: adapt + if: steps.lang.outputs.language == 'python' + continue-on-error: true + timeout-minutes: 5 + env: + SPEC_ID: ${{ steps.pr.outputs.specification_id }} + LIBRARY: ${{ steps.pr.outputs.library }} + LANGUAGE: ${{ steps.lang.outputs.language }} + IS_REGEN: ${{ steps.regen.outputs.is_regen }} + GCS_OK: ${{ steps.gcs.outcome == 'success' }} + # Off unless the owner sets the repository variable to exactly `true`. + ADAPT_REVIEW_FREE: ${{ vars.ADAPT_REVIEW_FREE || 'false' }} + CONTEXT_OUTCOME: ${{ steps.regen_ctx.outcome }} + PREV_STORED: ${{ steps.regen_ctx.outputs.prev_stored || 'n/a' }} + CHANGE_REQUEST_PRESENT: ${{ steps.regen_ctx.outputs.change_request_present || 'false' }} + run: | + OUT="$RUNNER_TEMP/adapt" + IMPL="plots/${SPEC_ID}/implementations/python/${LIBRARY}.py" + SPEC="plots/${SPEC_ID}/specification.md" + NEW_REC=/tmp/anyplot-adapt-new.json + PREV_REC=/tmp/anyplot-adapt-prev.json + + # 1. A stale record must never stand in for this run's. + rm -f "$NEW_REC" "$PREV_REC" + rm -rf "$OUT" + mkdir -p "$OUT" + + # 2. What impl-generate staged: new-smoke.json, the Step 3c log and, + # on a regeneration, prev-smoke.json, pair.json and both sources' raw + # renders. Best effort: a file that is not there reads as missing. + if [ "$GCS_OK" = "true" ]; then + SRC="gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}/adapt/*" + for attempt in 1 2 3; do + if gsutil -m cp -r "$SRC" "$OUT/" 2> "$RUNNER_TEMP/adapt-gsutil.err"; then + break + fi + if grep -q "No URLs matched" "$RUNNER_TEMP/adapt-gsutil.err"; then + echo "::notice::no adapt artifacts staged for ${SPEC_ID}/${LIBRARY}" + break + fi + echo "::warning::adapt artifact download failed (attempt ${attempt}/3): $(head -c 300 "$RUNNER_TEMP/adapt-gsutil.err" | tr '\n' ' ')" + [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) + done + else + echo "::warning::GCP auth failed — no adapt artifacts; the smoke runs read as missing" + fi + # Only the mirror writes these two, and only this step writes the source copy. + rm -rf "$OUT/prev-src" "$OUT/static-new.json" "$OUT/static-prev.json" + + # 3. The checker mirror on the PR's file (exit 0 pass or excluded, + # 1 fail, 2 unreadable input), then its record. + NEW_EXIT=0 + python3 "$RUNNER_TEMP/regen-tools/adaptability_check.py" check "$IMPL" \ + --spec "$SPEC" --spec-id "$SPEC_ID" --out "$OUT/static-new.json" \ + > /dev/null && NEW_EXIT=0 || NEW_EXIT=$? + python3 "$RUNNER_TEMP/regen-tools/adapt_record.py" record --side new \ + --spec "$SPEC_ID" --lib "$LIBRARY" --language "$LANGUAGE" --source "$IMPL" \ + --static "$OUT/static-new.json" --static-exit "$NEW_EXIT" \ + --smoke "$OUT/new-smoke.json" --selfcheck "$OUT/selfcheck.jsonl" --out "$NEW_REC" \ + || echo "::warning::adapt record (new) not written — the gate sees no adapt record" + + PREV_STATUS=n/a + ROUTE=n/a + WHY=n/a + if [ "$IS_REGEN" = "true" ]; then + # 4. The predecessor, under its library's file name: the checker + # infers the library, and so its setup-call allowlist, from the + # stem (never /tmp/anyplot-prev-impl.py). Its header holds the + # stored score, so the copy lives under $RUNNER_TEMP and goes + # right after the check. + mkdir -p "$OUT/prev-src" + PREV_SRC="$OUT/prev-src/${LIBRARY}.py" + PREV_EXIT=0 + if git show "origin/main:${IMPL}" > "$PREV_SRC" 2>/dev/null; then + python3 "$RUNNER_TEMP/regen-tools/adaptability_check.py" check "$PREV_SRC" \ + --spec "$SPEC" --spec-id "$SPEC_ID" --out "$OUT/static-prev.json" \ + > /dev/null && PREV_EXIT=0 || PREV_EXIT=$? + else + echo "::warning::${IMPL} is not readable on origin/main — the predecessor's record reads unknown" + rm -f "$PREV_SRC" + PREV_EXIT=2 + fi + python3 "$RUNNER_TEMP/regen-tools/adapt_record.py" record --side prev \ + --spec "$SPEC_ID" --lib "$LIBRARY" --language "$LANGUAGE" --source "$PREV_SRC" \ + --static "$OUT/static-prev.json" --static-exit "$PREV_EXIT" \ + --smoke "$OUT/prev-smoke.json" --out "$PREV_REC" \ + || echo "::warning::adapt record (prev) not written — the gate sees no adapt record" + rm -rf "$OUT/prev-src" + + # 5. The adapt route's conditions that need no review. adapt-route + # always exits 0 (2 on a usage error) and writes its adapt_* keys + # to the file it is given, which this step turns into its own + # outputs. Anything it does not say clearly is "no route": a + # failed call, an unknown value, or `true` without `why=route`. + ROUTE_OUT="$RUNNER_TEMP/adapt-route.out" + : > "$ROUTE_OUT" + ROUTE_FLAGS=() + [ -f /tmp/anyplot-canvas-gate.txt ] && ROUTE_FLAGS+=(--canvas-failed) + if GITHUB_OUTPUT="$ROUTE_OUT" python3 "$RUNNER_TEMP/regen-tools/regen_gate.py" adapt-route \ + --spec-id "$SPEC_ID" --library "$LIBRARY" \ + --adapt-new "$NEW_REC" --adapt-prev "$PREV_REC" --adapt-pair "$OUT/pair.json" \ + --prev-impl /tmp/anyplot-prev-impl.py --new-impl "$IMPL" "${ROUTE_FLAGS[@]}"; then + ROUTE=$(sed -n 's/^adapt_route=//p' "$ROUTE_OUT" | tail -n 1) + WHY=$(sed -n 's/^adapt_why=//p' "$ROUTE_OUT" | tail -n 1) + else + echo "::warning::regen_gate.py adapt-route failed — no adapt route, the review runs" + fi + if ! [[ "$ROUTE" =~ ^(true|false)$ ]] || ! [[ "$WHY" =~ ^[a-z_]{1,40}$ ]]; then + ROUTE=false + WHY=route_error + elif [ "$ROUTE" = "true" ] && [ "$WHY" != "route" ]; then + ROUTE=false + fi + PREV_STATUS=$(jq -r '.status' "$PREV_REC" 2>/dev/null) || PREV_STATUS=n/a + fi + NEW_STATUS=$(jq -r '.status' "$NEW_REC" 2>/dev/null) || NEW_STATUS=n/a + for side in NEW_STATUS PREV_STATUS; do + case "${!side}" in + pass|fail|excluded|unknown|n/a) ;; + *) printf -v "$side" '%s' n/a ;; + esac + done + + # 6. Review or not. The fail-safe direction is to review. + REVIEW_SKIPPED=false + if [ "$ADAPT_REVIEW_FREE" != "true" ]; then + REVIEW_WHY=switch_off + elif [ "$IS_REGEN" != "true" ]; then + REVIEW_WHY=first_generation + elif [ "$ROUTE" != "true" ]; then + REVIEW_WHY=no_route + elif [ "$CONTEXT_OUTCOME" != "success" ]; then + REVIEW_WHY=context_failed + elif ! [[ "$PREV_STORED" =~ ^[1-9][0-9]{0,2}$ ]] || [ "$PREV_STORED" -gt 100 ]; then + REVIEW_WHY=no_stored_score + elif [ "$CHANGE_REQUEST_PRESENT" = "true" ]; then + REVIEW_WHY=change_request + elif [ -f /tmp/anyplot-canvas-gate.txt ]; then + REVIEW_WHY=canvas_failed + else + REVIEW_SKIPPED=true + REVIEW_WHY=route + fi + SWITCH=off + [ "$ADAPT_REVIEW_FREE" = "true" ] && SWITCH=on + REVIEW=run + [ "$REVIEW_SKIPPED" = "true" ] && REVIEW=skipped + { + echo "new_status=${NEW_STATUS}" + echo "prev_status=${PREV_STATUS}" + echo "route=${ROUTE}" + echo "why=${WHY}" + } >> "$GITHUB_OUTPUT" + echo "::notice::adapt_review spec=${SPEC_ID} lib=${LIBRARY} switch=${SWITCH} route=${ROUTE} review=${REVIEW} why=${REVIEW_WHY}" + # Last write of the step (see above). + echo "review_skipped=${REVIEW_SKIPPED}" >> "$GITHUB_OUTPUT" + - name: React with eyes emoji env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -677,8 +872,12 @@ jobs: fi echo "::notice::blind review: header $(grep -m1 -o 'Quality: [a-z0-9]*' "$IMPL_FILE" || echo 'Quality: n/a'), stored review hidden=$([ -f "$META_FILE" ] && echo false || echo true)" + # Skipped only on a review-free adapt merge ("Adaptability check + # (record-only)"); every step keyed on this one's success then skips by + # itself, and the steps the review-free path needs say so in their `if`. - name: Run AI Quality Review id: review + if: steps.adapt.outputs.review_skipped != 'true' continue-on-error: true timeout-minutes: 30 uses: anthropics/claude-code-action@db77d4d3c98f3f6380e7cd45ffba2342a3dbde3c # v1 @@ -995,9 +1194,14 @@ jobs: # verdict is keep (fail closed). The gate record (scores, counts, reason # code, model and rules provenance — no model-written text) goes into the # PR comment on both paths below, where it outlives this run's log. + # The adapt records of "Adaptability check (record-only)" and the staged + # pair.json feed the adapt route and the adapt regression; without them + # the gate decides as before. On a review-free adapt merge the gate runs + # without a score (`--score ""` reads as none) and with + # --review-skipped, which only a counted adapt route turns into a merge. - name: Regen gate id: gate - if: steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' && steps.regen.outputs.is_regen == 'true' + if: steps.regen.outputs.is_regen == 'true' && ((steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true') || steps.adapt.outputs.review_skipped == 'true') env: PR_NUM: ${{ steps.pr.outputs.pr_number }} SPEC_ID: ${{ steps.pr.outputs.specification_id }} @@ -1014,6 +1218,11 @@ jobs: PREV_CRITERIA_VERSION: ${{ steps.regen_ctx.outputs.prev_criteria_version || 'n/a' }} LANGUAGE: ${{ steps.lang.outputs.language }} EXT: ${{ steps.lang.outputs.ext }} + REVIEW_SKIPPED: ${{ steps.adapt.outputs.review_skipped }} + # What adapt-route said before the review (the record's + # adapt.route_pre_review; a held route the review turns off is + # `adapt_overruled`). + ADAPT_ROUTE: ${{ steps.adapt.outputs.route }} run: | RECORD=/tmp/anyplot-regen-record.json rm -f "$RECORD" @@ -1021,6 +1230,12 @@ jobs: [ -f /tmp/anyplot-canvas-gate.txt ] && FLAGS+=(--canvas-failed) [ "$CHANGE_REQUEST_PRESENT" = "true" ] && FLAGS+=(--change-request-present) [ "$CONTEXT_OUTCOME" = "success" ] || FLAGS+=(--context-failed) + ADAPT=() + [ -s /tmp/anyplot-adapt-new.json ] && ADAPT+=(--adapt-new /tmp/anyplot-adapt-new.json) + [ -s /tmp/anyplot-adapt-prev.json ] && ADAPT+=(--adapt-prev /tmp/anyplot-adapt-prev.json) + [ -s "$RUNNER_TEMP/adapt/pair.json" ] && ADAPT+=(--adapt-pair "$RUNNER_TEMP/adapt/pair.json") + [ "$REVIEW_SKIPPED" = "true" ] && ADAPT+=(--review-skipped) + [ "$ADAPT_ROUTE" = "true" ] && ADAPT+=(--adapt-route-held) if ! python3 "$RUNNER_TEMP/regen-tools/regen_gate.py" decide \ --spec-id "$SPEC_ID" --library "$LIBRARY" \ --score "$SCORE" --prev-stored "$PREV_STORED" \ @@ -1035,8 +1250,8 @@ jobs: --record-out "$RECORD" \ --prev-impl "/tmp/anyplot-prev-impl${EXT}" \ --new-impl "plots/${SPEC_ID}/implementations/${LANGUAGE}/${LIBRARY}${EXT}" \ - "${FLAGS[@]}"; then - echo "::notice::regen_gate spec=${SPEC_ID} lib=${LIBRARY} prev_stored=${PREV_STORED} prev_rescored=n/a new=${SCORE} verdict=keep code=script_crashed model=${MODEL_ID} criteria=${CRITERIA_VERSION} reason=gate script crashed" + "${FLAGS[@]}" "${ADAPT[@]}"; then + echo "::notice::regen_gate spec=${SPEC_ID} lib=${LIBRARY} prev_stored=${PREV_STORED} prev_rescored=n/a new=${SCORE:-n/a} verdict=keep code=script_crashed model=${MODEL_ID} criteria=${CRITERIA_VERSION} reason=gate script crashed" { echo "verdict=keep" echo "reason=gate script crashed" @@ -1057,10 +1272,12 @@ jobs: # and the gate record. The predecessor's production renders are # overwritten by the next merge, so this is the only copy. Staged into # one directory first (mixed /tmp + workspace paths would be rooted at - # `/`). Never gating. + # `/`). adapt/ holds both adapt records and what impl-generate staged for + # them (smoke runs, pair.json, the raw renders of both sources, the + # Step 3c log), which the weekly review samples. Never gating. - name: Stage regen pair id: pair - if: steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' && steps.regen.outputs.is_regen == 'true' + if: steps.regen.outputs.is_regen == 'true' && ((steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true') || steps.adapt.outputs.review_skipped == 'true') continue-on-error: true timeout-minutes: 5 env: @@ -1085,6 +1302,10 @@ jobs: cp review_*.json review_*.txt quality_score.txt "$DIR/" 2>/dev/null || true cp /tmp/anyplot-regen-record.json "$DIR/gate-record.json" 2>/dev/null || true cp /tmp/anyplot-regen-summary.md "$DIR/gate-summary.md" 2>/dev/null || true + mkdir -p "$DIR/adapt" + cp -r "$RUNNER_TEMP/adapt/." "$DIR/adapt/" 2>/dev/null || true + rm -rf "$DIR/adapt/prev-src" + cp /tmp/anyplot-adapt-new.json /tmp/anyplot-adapt-prev.json "$DIR/adapt/" 2>/dev/null || true MAIN_SHA=$(git rev-parse -q --verify origin/main 2>/dev/null || echo "") jq -n --arg pr "$PR_NUM" --arg head "$HEAD_SHA" --arg main "$MAIN_SHA" --arg spec "$SPEC_ID" \ --arg lib "$LIBRARY" --arg lang "$LANGUAGE" --arg ext "$EXT" --arg model "$MODEL_ID" \ @@ -1105,9 +1326,65 @@ jobs: retention-days: 60 if-no-files-found: warn + # Review-free adapt merge only: no review ran, so the stored score that + # carries over ("Update metadata and implementation header") is the PR's + # score label, ahead of the verdict labels (quality:N, regen:improved, + # regen:adapt, ai-approved). impl-merge reads it for the issue comment. + # A failure stops the run before any verdict label, and "Re-dispatch + # review after a post-score failure" re-dispatches this review once + # (the shared `review-retry` marker); after that the PR carries + # `ai-review-failed` and the watchdog only flags it (a regeneration). + - name: Add carried quality score label + if: steps.adapt.outputs.review_skipped == 'true' && steps.gate.outputs.verdict == 'merge' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUM: ${{ steps.pr.outputs.pr_number }} + SCORE: ${{ steps.regen_ctx.outputs.prev_stored }} + run: | + if ! [[ "$SCORE" =~ ^[1-9][0-9]{0,2}$ ]] || [ "$SCORE" -gt 100 ]; then + echo "::error::no stored score to carry over (prev_stored='${SCORE}')" + exit 1 + fi + LABEL="quality:${SCORE}" + gh_retry() { + local what="$1"; shift + local attempt + for attempt in 1 2 3 4; do + if "$@" 2> /tmp/quality-label.err; then + return 0 + fi + echo "::warning::${what} failed (attempt ${attempt}/4): $(head -c 500 /tmp/quality-label.err | tr '\n' ' ')" + [ "$attempt" -lt 4 ] && sleep $((attempt * 10)) + done + echo "::error::${what} failed after 4 attempts for PR #${PR_NUM}" + return 1 + } + gh_retry "quality label creation" gh label create "$LABEL" --force --color "0e8a16" --description "Quality score ${SCORE}/100" \ + || echo "::warning::could not create ${LABEL} — the add below fails unless the label already exists" + # Exactly one quality:* label, as "Add quality score label" keeps it. + STALE="" + READ_OK=0 + for attempt in 1 2 3 4; do + if STALE=$(gh pr view "$PR_NUM" --json labels --jq '[.labels[].name | select(startswith("quality:")) | select(. != "'"$LABEL"'")] | join(",")' 2> /tmp/quality-label.err); then + READ_OK=1 + break + fi + echo "::warning::gh pr view (labels) failed (attempt ${attempt}/4): $(head -c 500 /tmp/quality-label.err | tr '\n' ' ')" + [ "$attempt" -lt 4 ] && sleep $((attempt * 10)) + done + if [ "$READ_OK" -ne 1 ]; then + echo "::error::Could not read labels for PR #${PR_NUM} after 4 attempts" + exit 1 + fi + if [ -n "$STALE" ]; then + gh_retry "stale quality label removal" gh pr edit "$PR_NUM" --remove-label "$STALE" + fi + gh_retry "quality label" gh pr edit "$PR_NUM" --add-label "$LABEL" + echo "::notice::carried quality label ${LABEL} (no review ran: adapt route)" + - name: Add preliminary verdict label (early) id: early_verdict - if: steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' + if: (steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true') || steps.adapt.outputs.review_skipped == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ steps.pr.outputs.pr_number }} @@ -1115,6 +1392,11 @@ jobs: ATTEMPT_COUNT: ${{ steps.attempts.outputs.count }} IS_REGEN: ${{ steps.regen.outputs.is_regen }} GATE_VERDICT: ${{ steps.gate.outputs.verdict }} + # The gate's adapt route held (regen_gate.py decide, adapt_counted). + ADAPT_COUNTED: ${{ steps.gate.outputs.adapt_counted }} + # It held before the review, and the review's floor turned it off + # (regen_gate.py decide, adapt_overruled). + ADAPT_OVERRULED: ${{ steps.gate.outputs.adapt_overruled }} run: | # Add verdict label early to ensure it's set even if later steps fail # This is idempotent - re-running won't cause issues @@ -1143,18 +1425,54 @@ jobs: # keep → `regen:kept` only. Never `ai-rejected`: that would start # the repair loop, which a regeneration never enters (one review, # then the gate). + # A merge on which the adapt route held also gets `regen:adapt`, + # between the two. A route that held before the review and that the + # review's floor turned off gets `regen:adapt-overruled`, on a merge + # or a keep, before the verdict label. Together they are the + # route-held PRs the owner reads before turning on + # ADAPT_REVIEW_FREE: the switch is safe only while the overruled + # share stays near zero. Best effort: a lost count label never + # holds up the verdict. + add_count_label() { + local label="$1" description="$2" attempt labelled=0 + gh label create "$label" --force --color "1d76db" --description "$description" 2>/dev/null \ + || echo "::warning::could not create ${label} — the add below fails unless the label exists" + for attempt in 1 2 3; do + if gh pr edit "$PR_NUM" --add-label "$label" 2> /tmp/label.err; then + labelled=1 + break + fi + echo "::warning::adding ${label} failed (attempt ${attempt}/3): $(head -c 300 /tmp/label.err | tr '\n' ' ')" + [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) + done + [ "$labelled" = "1" ] || echo "::warning::${label} not added to PR #${PR_NUM} — the verdict goes ahead; the adapt count misses this PR" + return 0 + } if [ "$IS_REGEN" = "true" ]; then gh label create "regen:improved" --color "0e8a16" --description "Regeneration replaced the live implementation (regen gate)" 2>/dev/null || true gh label create "regen:kept" --color "bfdadc" --description "Regeneration closed, live implementation kept (regen gate)" 2>/dev/null || true + OVERRULED_NOTE="" + [ "$ADAPT_OVERRULED" = "true" ] && OVERRULED_NOTE=" + regen:adapt-overruled" if [ "$GATE_VERDICT" = "merge" ]; then echo "approved=true" >> "$GITHUB_OUTPUT" add_label "regen:improved" || { echo "::error::regen:improved could not be added — not adding ai-approved"; exit 1; } + ADAPT_NOTE="" + if [ "$ADAPT_COUNTED" = "true" ]; then + add_count_label "regen:adapt" "Regeneration merged on the adapt route (render identity, adaptability check)" + ADAPT_NOTE=" + regen:adapt" + fi + if [ "$ADAPT_OVERRULED" = "true" ]; then + add_count_label "regen:adapt-overruled" "Adapt route held before the review, and the review overruled it" + fi add_label "ai-approved" || exit 1 - echo "::notice::Regen gate: merge (regen:improved + ai-approved)" + echo "::notice::Regen gate: merge (regen:improved${ADAPT_NOTE}${OVERRULED_NOTE} + ai-approved)" else echo "approved=false" >> "$GITHUB_OUTPUT" + if [ "$ADAPT_OVERRULED" = "true" ]; then + add_count_label "regen:adapt-overruled" "Adapt route held before the review, and the review overruled it" + fi add_label "regen:kept" || exit 1 - echo "::notice::Regen gate: keep (regen:kept)" + echo "::notice::Regen gate: keep (regen:kept${OVERRULED_NOTE})" fi exit 0 fi @@ -1204,7 +1522,7 @@ jobs: fi - name: Install Python dependencies for metadata update - if: steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' + if: (steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true') || steps.adapt.outputs.review_skipped == 'true' run: pip install pyyaml # Monitoring only: every weakness line is a defect (` (): …`) @@ -1230,16 +1548,23 @@ jobs: # Skipped on a kept regeneration: the PR is closed unmerged, so there is # nothing to record on its branch (the live implementation's re-score # goes to main through "Write back the re-score (regen keep)"). + # On a review-free adapt merge it runs in carried mode (REVIEW_CARRIED): + # impl-generate reset quality_score to null and emptied the review, and + # impl-merge merges the branch's metadata as it stands, so the stored + # score and review are written back unchanged, marked as carried, and + # the header gets the carried score with the measured versions. - name: Update metadata and implementation header - if: steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' && (steps.regen.outputs.is_regen != 'true' || steps.gate.outputs.verdict == 'merge') + if: (steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' && (steps.regen.outputs.is_regen != 'true' || steps.gate.outputs.verdict == 'merge')) || (steps.adapt.outputs.review_skipped == 'true' && steps.gate.outputs.verdict == 'merge') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SPEC_ID: ${{ steps.pr.outputs.specification_id }} LIBRARY: ${{ steps.pr.outputs.library }} LANGUAGE: ${{ steps.lang.outputs.language }} EXT: ${{ steps.lang.outputs.ext }} - SCORE: ${{ steps.score.outputs.score }} + SCORE: ${{ steps.adapt.outputs.review_skipped == 'true' && steps.regen_ctx.outputs.prev_stored || steps.score.outputs.score }} BRANCH: ${{ steps.pr.outputs.branch }} + REVIEW_CARRIED: ${{ steps.adapt.outputs.review_skipped == 'true' }} + CARRY_FROM: /tmp/anyplot-prev-metadata.yaml # Review provenance (not synced to the database: sync_to_postgres # picks review keys by name). An empty value removes the key. REVIEW_MODEL: ${{ steps.review_model.outputs.model_id }} @@ -1281,7 +1606,23 @@ jobs: score = int(sys.argv[2]) timestamp = sys.argv[3] - # Read existing review data files + # Carried mode, a review-free adapt merge only: no review ran and the + # render is identical to the predecessor's, so main's stored score + # and review (CARRY_FROM, the copy "Regen context" wrote) carry over + # unchanged and are marked `review.carried`. Nothing of the review_* + # files or this run's provenance is used; the measured versions + # impl-generate wrote stay as they are. + carried = os.environ.get('REVIEW_CARRIED', '').strip() == 'true' + prev = {} + if carried: + carry_from = os.environ.get('CARRY_FROM', '').strip() + with open(carry_from) as f: + prev = yaml.safe_load(f) or {} + prev_score = prev.get('quality_score') if isinstance(prev, dict) else None + if isinstance(prev_score, bool) or not isinstance(prev_score, int) or prev_score != score or not 0 < score <= 100: + sys.exit(f'carried review: {carry_from} stores quality_score {prev_score!r}, not {score}') + + # Read existing review data files (carried mode uses none of them) strengths = [] weaknesses = [] image_description = None @@ -1347,39 +1688,49 @@ jobs: if 'review' not in data: data['review'] = {} - # Update review section with all fields - data['review']['strengths'] = strengths - data['review']['weaknesses'] = weaknesses - - # Add extended review data (issue #2845) - if image_description: - data['review']['image_description'] = image_description - if criteria_checklist: - data['review']['criteria_checklist'] = criteria_checklist - if verdict: - data['review']['verdict'] = verdict - - # Review provenance: resolved model id, rules version, render time. - # Each is optional — a missing value never stops this script. A - # value this review could not determine removes the key: a repair - # review starts from the metadata the previous review wrote, whose - # model, rules or render time would otherwise pass for this one's. - review_model = os.environ.get('REVIEW_MODEL', '').strip() - criteria_version = os.environ.get('CRITERIA_VERSION', '').strip() - rendered_at = os.environ.get('RENDERED_AT', '').strip() - for key, value in ( - ('model', review_model), - ('criteria_version', criteria_version), - ('rendered_at', rendered_at), - ): - if value: - data['review'][key] = value - else: - data['review'].pop(key, None) - - # Add impl_tags (issue #2434) - if impl_tags: - data['impl_tags'] = impl_tags + if carried: + prev_review = prev.get('review') + data['review'] = dict(prev_review) if isinstance(prev_review, dict) else {} + data['review']['carried'] = True + # impl-generate rewrote the metadata without them. + if prev.get('impl_tags'): + data['impl_tags'] = prev['impl_tags'] + else: + # Update review section with all fields + data['review']['strengths'] = strengths + data['review']['weaknesses'] = weaknesses + + # Add extended review data (issue #2845) + if image_description: + data['review']['image_description'] = image_description + if criteria_checklist: + data['review']['criteria_checklist'] = criteria_checklist + if verdict: + data['review']['verdict'] = verdict + + # Review provenance: resolved model id, rules version, render time. + # Each is optional — a missing value never stops this script. A + # value this review could not determine removes the key: a repair + # review starts from the metadata the previous review wrote, whose + # model, rules or render time would otherwise pass for this one's. + review_model = os.environ.get('REVIEW_MODEL', '').strip() + criteria_version = os.environ.get('CRITERIA_VERSION', '').strip() + rendered_at = os.environ.get('RENDERED_AT', '').strip() + for key, value in ( + ('model', review_model), + ('criteria_version', criteria_version), + ('rendered_at', rendered_at), + ): + if value: + data['review'][key] = value + else: + data['review'].pop(key, None) + # This review ran: nothing in it is carried. + data['review'].pop('carried', None) + + # Add impl_tags (issue #2434) + if impl_tags: + data['impl_tags'] = impl_tags def str_representer(dumper, data): if isinstance(data, str) and data.endswith('Z') and 'T' in data: @@ -1396,7 +1747,11 @@ jobs: EOF python3 /tmp/update_metadata.py "$METADATA_FILE" "$SCORE" "$TIMESTAMP" - echo "::notice::Updated metadata with quality score ${SCORE} and extended review data" + if [ "$REVIEW_CARRIED" = "true" ]; then + echo "::notice::Carried the stored score ${SCORE} and its review over (no review ran: adapt route)" + else + echo "::notice::Updated metadata with quality score ${SCORE} and extended review data" + fi fi # Update implementation header with quality score @@ -1560,7 +1915,11 @@ jobs: if ! git diff --cached --quiet; then # No score in the subject: `git log` on the PR branch is the first # thing a later reviewer would see. - git commit -m "chore(${LIBRARY}): store the review for ${SPEC_ID}" + if [ "$REVIEW_CARRIED" = "true" ]; then + git commit -m "chore(${LIBRARY}): carry the stored review over for ${SPEC_ID}" + else + git commit -m "chore(${LIBRARY}): store the review for ${SPEC_ID}" + fi git push origin "$BRANCH" echo "::notice::Changes committed to ${BRANCH}" fi @@ -1570,7 +1929,11 @@ jobs: # would ever run. Run 36712916551 (PR #11984, 2026-09-30): "Add quality # score label" hit HTTP 502 twice and the PR sat without a verdict for # ~7 h. A crashed or empty review never gets here (has_output is false: - # "Validate review output" owns it). Once a verdict label is on the PR + # "Validate review output" owns it). The review-free adapt path + # (review_skipped, no review step at all) has the same window: a + # failure before its verdict label (the carried quality label, the + # early verdict) gets the same one re-dispatch, with the stored score + # in place of a review score. Once a verdict label is on the PR # the watchdog's merge/repair cases own the state, and a second review # could add a contradicting verdict, so this step then does nothing. # It shares the `review-retry` marker with "Validate review output": at @@ -1581,13 +1944,16 @@ jobs: # regeneration is only flagged). # The retry runs on this run's ref, so a branch smoke stays on its branch. - name: Re-dispatch review after a post-score failure - if: failure() && steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' && steps.early_verdict.conclusion != 'success' + if: failure() && steps.early_verdict.conclusion != 'success' && ((steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true') || steps.adapt.outputs.review_skipped == 'true') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ steps.pr.outputs.pr_number }} SPEC_ID: ${{ steps.pr.outputs.specification_id }} LIBRARY: ${{ steps.pr.outputs.library }} - SCORE: ${{ steps.score.outputs.score }} + # On the review-free path no review scored: the stored score that + # carries over, as "Update metadata and implementation header" uses. + SCORE: ${{ steps.adapt.outputs.review_skipped == 'true' && steps.regen_ctx.outputs.prev_stored || steps.score.outputs.score }} + REVIEW_SKIPPED: ${{ steps.adapt.outputs.review_skipped }} REPOSITORY: ${{ github.repository }} RUN_ID: ${{ github.run_id }} MODEL: ${{ steps.pr.outputs.model }} @@ -1596,6 +1962,15 @@ jobs: run: | RUN_URL="https://github.com/${REPOSITORY}/actions/runs/${RUN_ID}" MARKER="" + # What the comments say came before the failure: a review score, or + # on the review-free adapt path the stored score that carries over. + if [ "$REVIEW_SKIPPED" = "true" ]; then + SCORED="No review ran (the adapt route skipped it; stored score ${SCORE:-n/a}/100)" + RETRY="impl-review is dispatched once more, and decides again whether the review runs." + else + SCORED="The review scored ${SCORE}/100" + RETRY="A fresh review is dispatched once." + fi # Same shape as the verdict step's helper; one GitHub brown-out is # the likely reason this step runs at all. gh_retry() { @@ -1618,7 +1993,7 @@ jobs: echo "${MARKER}" echo "## :x: AI Review Failed After Scoring" echo "" - echo "The review scored ${SCORE}/100, but $1 No verdict was applied." + echo "${SCORED}, but $1 No verdict was applied." echo "" echo "**Manual rerun:**" echo "\`\`\`" @@ -1682,7 +2057,7 @@ jobs: echo "${MARKER}" echo "## :wrench: AI Review Failed After Scoring — Re-Reviewing" echo "" - echo "The review scored ${SCORE}/100, but a later step failed before a verdict was applied (see the run log). A fresh review is dispatched once." + echo "${SCORED}, but a later step failed before a verdict was applied (see the run log). ${RETRY}" echo "" echo "---" echo ":robot: *[impl-review](${RUN_URL})*" @@ -1914,7 +2289,7 @@ jobs: set_status opened "#${WB_PR}" - name: Add verdict label and take action - if: steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true' + if: (steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true') || steps.adapt.outputs.review_skipped == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ steps.pr.outputs.pr_number }} @@ -1939,6 +2314,9 @@ jobs: WB_PR: ${{ steps.writeback.outputs.pr }} WB_SCORE: ${{ steps.writeback.outputs.score }} NOTHING_TO_REPAIR: ${{ steps.early_verdict.outputs.nothing_to_repair }} + REVIEW_SKIPPED: ${{ steps.adapt.outputs.review_skipped }} + ADAPT_COUNTED: ${{ steps.gate.outputs.adapt_counted }} + ADAPT_OVERRULED: ${{ steps.gate.outputs.adapt_overruled }} run: | # Cascading thresholds: # Attempt 0 (Review 1): >= 90 @@ -1984,10 +2362,19 @@ jobs: if [ ! -s "$SUMMARY_FILE" ]; then printf '**Reason:** %s\n' "${GATE_REASON:-regen gate produced no verdict}" > "$SUMMARY_FILE" fi + # What the comments say about this run's review: its score, or + # that the adapt route skipped it. + if [ "$REVIEW_SKIPPED" = "true" ]; then + RENDER="this render (no review: the adapt route skipped it)" + else + RENDER="this render (score ${SCORE})" + fi # Keep only: what happened to the re-score of the live # implementation ("Write back the re-score", previous step). A - # missing status (the step was skipped or cut off) reads as failed. + # missing status (the step was skipped or cut off) reads as failed; + # without a review there is no re-score to write back. if [ "$GATE_VERDICT" != "merge" ]; then + [ "$REVIEW_SKIPPED" = "true" ] && WB_STATUS=no_rescore case "$WB_STATUS" in opened|unchanged|no_rescore|invalid|stale|failed) ;; *) WB_STATUS=failed ;; @@ -2003,6 +2390,12 @@ jobs: # for both comments; empty when the gate wrote none. GATE_RECORD_MARKER=$(python3 "$RUNNER_TEMP/regen-tools/regen_gate.py" marker \ --record /tmp/anyplot-regen-record.json 2>/dev/null) || GATE_RECORD_MARKER="" + # A route that held before the review and that the review turned + # off (the early step's `regen:adapt-overruled`), on either verdict. + OVERRULED_TEXT="" + if [ "$ADAPT_OVERRULED" = "true" ]; then + OVERRULED_TEXT="The adapt route held before the review, but the review overruled it (see **Adapt route** below), so this PR is labelled \`regen:adapt-overruled\`. The review-free switch (\`ADAPT_REVIEW_FREE\`) is turned on only while such PRs stay near zero among the route-held ones." + fi if [ "$GATE_VERDICT" = "merge" ]; then echo "::notice::Regen gate approved the replacement (${GATE_REASON}), triggering merge" @@ -2015,7 +2408,19 @@ jobs: { echo "## :arrows_counterclockwise: Regeneration replaces the live implementation" echo "" - echo "The regen gate compared this render (score ${SCORE}) with the implementation on main and approved the replacement; the merge follows." + if [ "$REVIEW_SKIPPED" = "true" ]; then + echo "No review ran: the adapt route held. The implementation on main fails the adaptability check, this one passes it and its smoke run, and both render alike within the identity bound in one runner. The stored score (${PREV_STORED}) and its review carry over unchanged; the merge follows." + else + echo "The regen gate compared ${RENDER} with the implementation on main and approved the replacement; the merge follows." + if [ "$ADAPT_COUNTED" = "true" ]; then + echo "" + echo "The adapt route held as well, so this PR is labelled \`regen:adapt\`: one more reviewed conversion in the count that the review-free switch (\`ADAPT_REVIEW_FREE\`) waits for." + fi + if [ -n "$OVERRULED_TEXT" ]; then + echo "" + echo "$OVERRULED_TEXT" + fi + fi echo "" cat "$SUMMARY_FILE" echo "" @@ -2051,7 +2456,11 @@ jobs: echo "" echo "## :leftwards_arrow_with_hook: Regeneration kept the live implementation" echo "" - echo "The regen gate compared this render (score ${SCORE}) with the implementation on main and kept the latter. This PR is closed without merging; the live code and the production images are unchanged." + echo "The regen gate compared ${RENDER} with the implementation on main and kept the latter. This PR is closed without merging; the live code and the production images are unchanged." + if [ -n "$OVERRULED_TEXT" ]; then + echo "" + echo "$OVERRULED_TEXT" + fi echo "" cat "$SUMMARY_FILE" echo "" diff --git a/agentic/commands/regen.md b/agentic/commands/regen.md index 9dcaf428d3c..24cfc8351ef 100644 --- a/agentic/commands/regen.md +++ b/agentic/commands/regen.md @@ -181,7 +181,8 @@ CHANGE_REQUEST=$(echo "$NEXT" | cut -f4) # may be empty 4. `plots/{SPEC_ID}/metadata/python/{LIBRARY}.yaml` — `review.strengths`, `review.weaknesses`, `review.criteria_checklist` 5. `prompts/library/{LIBRARY}.md` (CRITICAL — follow exactly) 6. `prompts/plot-generator.md` -7. `prompts/quality-criteria.md` +7. `prompts/adaptable-code.md` — the Data block convention the file follows +8. `prompts/quality-criteria.md` Skip Context7 by default — only consult it if step 2d reveals a clear library-API issue. diff --git a/agentic/commands/update.md b/agentic/commands/update.md index af4ce57e016..f970b89daff 100644 --- a/agentic/commands/update.md +++ b/agentic/commands/update.md @@ -521,8 +521,9 @@ Read these files to understand what you're working with: - `quality_score` — current score to beat 4. `prompts/library/{LIBRARY}.md` — library-specific rules (**CRITICAL**: follow these exactly) 5. `prompts/plot-generator.md` — base generation rules -6. `prompts/quality-criteria.md` — quality scoring criteria -7. **Context7 library documentation** — Query up-to-date library docs for idiomatic patterns: +6. `prompts/adaptable-code.md` — the Data block convention the file follows +7. `prompts/quality-criteria.md` — quality scoring criteria +8. **Context7 library documentation** — Query up-to-date library docs for idiomatic patterns: - Call `resolve-library-id` with `libraryName: "{CONTEXT7_LIBRARY}"` and `query: "how to create {PLOT_TYPE} chart with {CONTEXT7_LIBRARY}"` - Call `query-docs` with the resolved library ID and `query: "idiomatic patterns for creating {SPEC_TITLE} ({PLOT_TYPE}) with {CONTEXT7_LIBRARY}, including best practices for styling and layout"` - Use the returned documentation **together with** (not instead of) the static library rules from step 4 diff --git a/agentic/docs/project-guide.md b/agentic/docs/project-guide.md index 03d7754144a..cee66a3cf93 100644 --- a/agentic/docs/project-guide.md +++ b/agentic/docs/project-guide.md @@ -547,6 +547,7 @@ The `prompts/` directory contains AI agent prompts for code generation, quality | File | Purpose | |------|---------| | `plot-generator.md` | Base rules for all plot implementations | +| `adaptable-code.md` | The Data block convention: one replaceable block ending in `df`, columns named once, plot code derived from `df` | | `default-style-guide.md` | Default visual style rules (colors, typography, layout) | | `library/*.md` | Library-specific rules (15 files, one per library) | | `quality-criteria.md` | Definition of code/visual quality | diff --git a/automation/scripts/README.md b/automation/scripts/README.md index 524a0cb8d0f..6a148ac9ffa 100644 --- a/automation/scripts/README.md +++ b/automation/scripts/README.md @@ -100,6 +100,47 @@ invalidates `review_regen.json`. The record's `improvements.code` counts the entries that held, the notice line gains `prev_lines=`/`new_lines=` when the flags are given, and `check-feedback` checks the entries. +The adapt route (`docs/concepts/adaptable-code.md`, "The adapt route"): +`adapt_assess` decides whether a conversion to the adaptable-code convention +merges on render identity. It reads the two adapt records that impl-review's +"Adaptability check (record-only)" step writes (`--adapt-new`, +`--adapt-prev`), and the render pair that `core.smoke_run pair` staged in +impl-generate (`--adapt-pair`). It never renders, so the gate stays stdlib-only. +The route counts when every one of these holds; the first that fails is the +record's `why`: + +- The new implementation is Python. +- Both records are valid for this pair, and each status follows from its + static and smoke statuses (`derived_adapt_status`, the table of + `adapt_record.py` `derive_status`); otherwise `record_invalid`. +- The predecessor's status is `fail` and the new one's `pass`. +- The pair names this library and rendered the two sources the records + checked (the same `code_sha256` on each side, else `pair_stale`). +- The pair rendered both themes at equal size, and each differs in at most + `IDENTITY_BOUND` (0.5 %, the value of `core.render_identity.DEFAULT_BOUND`) + of its pixels. The gate applies its own bound, whatever bound the pair file + names. +- The new source keeps `within_line_bound`: max(1.25 × prev, prev + 15) lines. +- The canvas gate passed. +- When a review ran, its score is above 0, it names no regression, the + scenario is unchanged, no encoding was added, and no VQ, SC, DQ, or CQ item + of `prev_checklist` scores lower in the new checklist. + +`decide` turns the keeps `no_visible_improvement`, `no_defect_improvement` and +`below_tolerance` into a merge with code `adapt_identity` when the route counts. +With `--review-skipped` and no score, it merges on the route before the score +checks. A merge whose predecessor passes and whose new source fails becomes keep +with `adapt_regression`, and so does one whose predecessor passes the static +check with an unknown smoke run while the new source fails the static check. `adapt-route` writes `adapt_route`, `adapt_why`, +`adapt_prev`, and `adapt_new` before the review, and always exits 0. impl-review +passes that result back to `decide` as `--adapt-route-held`. The record gains +`adapt` (`AdaptAssessment.to_record`, checked by `validate_record`), with +`route_pre_review` from that flag. The outputs gain `adapt_counted`, +`adapt_why`, and `adapt_overruled` (the route held before the review and the +review's floor turned it off, which impl-review labels `regen:adapt-overruled`), +and with an adapt flag the notice line gains them too, before `reason=`. Without the adapt flags, the route and the +regression are off, so older argument sets decide as before. + ### `regen_writeback.py` Review write-back on a kept regeneration (stdlib and PyYAML; imports `regen_gate` from its own directory when it runs as a script, so a workflow's @@ -114,9 +155,11 @@ implementation, written by the review as `review_prev.json` (step 8b of - `apply` — writes the score, the five review fields, `review.model`, and `review.criteria_version` into the metadata with the metadata writer's YAML settings (an empty provenance value removes the key), and with `--impl` the - score into the `Quality: N/100` header. Nothing else changes. + score into the `Quality: N/100` header, and removes `review.carried`. + Nothing else changes. - `verify-diff` — against the merge base: only the pair's metadata (as parsed - YAML, `quality_score` and those review keys) and its header number change. + YAML, `quality_score` and those review keys) and its header number change; + `review.carried` may only be removed. - `check-pr` — the PR is an open `review-writeback/{spec}/{library}/{kept PR}` PR with the `review-writeback` label by `app/github-actions` (otherwise a no-op), and its score equals the `prev_rescored` of the first gate record @@ -133,6 +176,19 @@ implementation, written by the review as `review_prev.json` (step 8b of `check-pr`, `verify-diff`, and `check-fresh` before every merge attempt, then merges with the admin token. +### `adapt_record.py` +Adapt records (stdlib). `impl-review.yml`'s "Adaptability check (record-only)" +runs `record` once per side of a Python pair: the PR head (`--side new`) and, on +a regeneration, the predecessor from `origin/main` (`--side prev`). It combines +the checker mirror's `--out` record (verbatim, plus its exit code) with the +smoke run that impl-generate staged under `staging/.../adapt/`. A missing staged +run reads as `missing`, and a run whose `code_sha256` differs from the source's +reads as `stale`. It derives the record's status (`pass`, `fail`, `excluded`, +or `unknown`) and prints one `::notice::adapt_check` line. The regen gate reads +`v`, `side`, `spec`, `lib`, `language`, `checker_version`, `status`, +`static.status`, and `smoke.status`. `code_sha256` is `core.smoke_run`'s rule, +and a test ties the two. + ### `review_provenance.py` Review provenance (stdlib): `criteria-version` prints the rules version — git blob ids of `prompts/quality-criteria.md`, `ai-quality-review.md`, diff --git a/automation/scripts/adapt_record.py b/automation/scripts/adapt_record.py new file mode 100644 index 00000000000..ec214ab970d --- /dev/null +++ b/automation/scripts/adapt_record.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Adapt records: impl-review's record-only adaptability check of one implementation. + +Standard library only. ``impl-review.yml`` "Adaptability check (record-only)" +runs it from the copy it takes at its own ref (``$RUNNER_TEMP/regen-tools/``), +once per side of a Python pair: + +- ``new``: the PR head's implementation. Always, for a Python library. +- ``prev``: the predecessor from ``origin/main``. A regeneration only. + +Each side combines two inputs into one JSON record +(``docs/concepts/adaptable-code.md``, "The checks"): + +- ``static``: what the checker mirror (``adaptability_check.py check --out``) + wrote, verbatim, plus its exit code. When the mirror exits 2 or writes + nothing, it is ``{"status": "error", "exit": N}``. +- ``smoke``: the perturbation smoke run impl-generate staged + (``core.smoke_run run``, ``new-smoke.json`` or ``prev-smoke.json``), trimmed + to its status, reason and failures. A missing or unreadable file is + ``missing``; a file whose ``code_sha256`` differs from the checked source's is + ``stale``, because it ran on another version of the code. + +CLI:: + + adapt_record.py record --side new --spec bar-grouped --lib matplotlib --language python \\ + --source plots/bar-grouped/implementations/python/matplotlib.py \\ + --static static-new.json --static-exit 0 --smoke new-smoke.json \\ + [--selfcheck selfcheck.jsonl] --out /tmp/anyplot-adapt-new.json + +``record`` writes the record and prints one ``::notice::adapt_check`` line. +It exits 0 once the record is written, 1 when it cannot be written, and 2 on +a usage error. The workflow never gates on it. + +The derived ``status`` (first matching rule wins; ``why`` is a token): + +========= ================================= ============================================ +status why rule +========= ================================= ============================================ +unknown static_error ``static.status`` is ``error`` +excluded excluded ``static.status`` is ``excluded`` (map specs) +fail static_fail ``static.status`` is ``fail`` +unknown smoke_missing, smoke_stale ``smoke.status`` is ``missing`` or ``stale`` +fail smoke_fail ``smoke.status`` is ``fail`` +pass smoke_na ``skipped`` for ``shape:generated`` or ``shape:network`` +unknown smoke_skipped any other ``skipped`` (``no_perturbations:*`` too) +pass ``-`` ``smoke.status`` is ``pass`` +========= ================================= ============================================ + +The regen gate (``regen_gate.py adapt-route`` and ``decide --adapt-new +--adapt-prev``) reads ``v``, ``side``, ``spec``, ``lib``, ``language``, +``checker_version``, ``status``, ``static.status``, ``smoke.status`` and +``smoke.why``. It cross-checks the derived status against the two blocks +with its own copy of this table (``regen_gate.derived_adapt_status``; a unit +test runs both over every combination) and rejects a record whose status +does not follow from them. Everything else is diagnostics for the regen-pair +artifact and the weekly review. + +``code_sha256`` is byte for byte the function of ``core.smoke_run``: the +SHA-256 of the source without its leading docstring header, which the +workflows rewrite (``Quality:``, ``Updated:``). A unit test pins the two. + +Every value the notice prints is reduced to a token first: the smoke results +come from the generation runner, where a model session had a shell. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + + +RECORD_VERSION = 1 +SIDES = ("new", "prev") +STATUSES = ("pass", "fail", "excluded", "unknown") +STATIC_STATUSES = ("pass", "fail", "excluded") +STAGED_SMOKE_STATUSES = ("pass", "fail", "skipped") +# A `skipped` smoke run whose shape has no table to perturb is not a gap: the +# static check is the whole check there. +SMOKE_NOT_APPLICABLE = ("shape:generated", "shape:network") +# core.smoke_run.SELFCHECK_SKIP_PREFIXES (tests/unit/automation/scripts/ +# test_adapt_record.py ties the two; this script is copied alone and imports no +# core): a skip with one of these reasons passes a self-check attempt, as +# `selfcheck` exits 0 on it. `no_perturbations:` still reads `unknown` in the +# record (derive_status): the smoke run checked nothing. +SELFCHECK_SKIP_PREFIXES = ("shape:", "no_perturbations:") +# core.adaptability.RULE_ORDER (tests/unit/automation/scripts/test_adapt_record.py +# ties the two): the notice lists rule classes in this order. +RULE_ORDER = ( + "layout", + "columns", + "interface", + "column-literal", + "sample-literal", + "data-coordinate", + "statistic-text", + "df-comparison", + "count-literal", + "palette", + "constants", +) +TOKEN_RE = re.compile(r"^[A-Za-z0-9_.:+-]{1,64}$") +MAX_ERROR_CHARS = 300 +MAX_RESULTS = 50 +NONE = "-" + + +def code_sha256(text: str) -> str: + """The SHA-256 of the source without its leading docstring header (core.smoke_run's rule).""" + body = text.lstrip() + if body.startswith('"""'): + end = body.find('"""', 3) + if end != -1: + body = body[end + 3 :] + return hashlib.sha256(body.strip().encode("utf-8")).hexdigest() + + +def token(value: Any, default: str = "n/a") -> str: + """``value`` when it is a short token safe for a notice line, else ``default``.""" + if isinstance(value, bool) or value is None: + return default + text = str(value) + return text if TOKEN_RE.fullmatch(text) else default + + +def _tokens(values: Any) -> list[str]: + if not isinstance(values, list): + return [] + return [str(v) for v in values if isinstance(v, str) and TOKEN_RE.fullmatch(v)] + + +def _int_or_none(value: Any) -> int | None: + if isinstance(value, bool) or not isinstance(value, int): + return None + return value + + +def load_json(path: Path | None) -> Any: + """The parsed file, or None when it is absent, unreadable or not JSON.""" + if path is None or not path.is_file(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + print(f"::warning::{path.name} is unreadable: {' '.join(str(exc).split())[:200]}") + return None + + +def static_block(path: Path | None, exit_code: int) -> dict[str, Any]: + """The mirror's record verbatim plus ``exit``, or ``{"status": "error", "exit": N}``.""" + data = load_json(path) + if exit_code not in (0, 1) or not isinstance(data, dict) or data.get("status") not in STATIC_STATUSES: + return {"status": "error", "exit": exit_code} + return {**data, "exit": exit_code} + + +def _empty_smoke(status: str, fresh: bool | None) -> dict[str, Any]: + return {"status": status, "why": NONE, "fresh": fresh, "failed": [], "advisory_failed": [], "results": []} + + +def _result(item: Any) -> dict[str, Any] | None: + if not isinstance(item, dict): + return None + error = item.get("error") + return { + "name": token(item.get("name"), default=NONE), + "status": token(item.get("status"), default=NONE), + "advisory": item.get("advisory") is True, + "line": _int_or_none(item.get("line")), + "error": " ".join(error.split())[:MAX_ERROR_CHARS] if isinstance(error, str) else None, + } + + +def smoke_block(path: Path | None, source_sha: str | None) -> dict[str, Any]: + """The staged smoke run, trimmed: ``pass``, ``fail``, ``skipped``, ``missing`` or ``stale``.""" + data = load_json(path) + if not isinstance(data, dict) or data.get("status") not in STAGED_SMOKE_STATUSES: + return _empty_smoke("missing", None) + staged_sha = data.get("code_sha256") + if source_sha is None or not isinstance(staged_sha, str) or staged_sha != source_sha: + return _empty_smoke("stale", False) + raw_results = data.get("results") + results = ( + [r for r in (_result(item) for item in raw_results[:MAX_RESULTS]) if r] if isinstance(raw_results, list) else [] + ) + return { + "status": data["status"], + "why": token(data.get("why"), default=NONE), + "fresh": True, + "failed": _tokens(data.get("failed")), + "advisory_failed": _tokens(data.get("advisory_failed")), + "results": results, + } + + +def selfcheck_block(path: Path | None) -> dict[str, Any] | None: + """``{"attempts": N, "first": "pass"|"fail"}`` from the generator's Step 3c log, or None. + + An attempt passes as ``selfcheck`` exits 0: its static check is ``pass`` + or ``excluded``, and its smoke run is ``pass`` or ``skipped`` for the + shape or for a sample with nothing to perturb (``smoke_why`` ``shape:*`` + or ``no_perturbations:*``, ``SELFCHECK_SKIP_PREFIXES``). A log line + without ``smoke_why`` reads any ``skipped`` as a pass. + """ + if path is None or not path.is_file(): + return None + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return None + attempts: list[dict[str, Any]] = [] + for line in lines: + try: + item = json.loads(line) + except ValueError: + continue + if isinstance(item, dict): + attempts.append(item) + if not attempts: + return None + first = attempts[0] + smoke, smoke_why = first.get("smoke"), first.get("smoke_why") + if isinstance(smoke_why, str): + smoke_ok = smoke == "pass" or (smoke == "skipped" and smoke_why.startswith(SELFCHECK_SKIP_PREFIXES)) + else: + smoke_ok = smoke in ("pass", "skipped") + passed = first.get("static") in ("pass", "excluded") and smoke_ok + return {"attempts": len(attempts), "first": "pass" if passed else "fail"} + + +def derive_status(static_status: str, smoke_status: str, smoke_why: str) -> tuple[str, str]: + """The record's ``(status, why)`` from the static and smoke statuses (module docstring table).""" + if static_status not in STATIC_STATUSES: + return "unknown", "static_error" + if static_status == "excluded": + return "excluded", "excluded" + if static_status == "fail": + return "fail", "static_fail" + if smoke_status in ("missing", "stale"): + return "unknown", f"smoke_{smoke_status}" + if smoke_status == "fail": + return "fail", "smoke_fail" + if smoke_status == "skipped": + if smoke_why in SMOKE_NOT_APPLICABLE: + return "pass", "smoke_na" + return "unknown", "smoke_skipped" + if smoke_status == "pass": + return "pass", NONE + return "unknown", "smoke_missing" + + +def build_record( + *, + side: str, + spec: str, + lib: str, + language: str, + source: Path | None, + static: Path | None, + static_exit: int, + smoke: Path | None, + selfcheck: Path | None = None, +) -> dict[str, Any]: + """One side's record (the format of the module docstring).""" + text: str | None = None + if source is not None and source.is_file(): + try: + text = source.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + print(f"::warning::{source} is unreadable: {' '.join(str(exc).split())[:200]}") + source_sha = code_sha256(text) if text is not None else None + static_rec = static_block(static, static_exit) + smoke_rec = smoke_block(smoke, source_sha) + status, why = derive_status(str(static_rec["status"]), smoke_rec["status"], smoke_rec["why"]) + record: dict[str, Any] = { + "v": RECORD_VERSION, + "side": side, + "spec": spec, + "lib": lib, + "language": language, + "checker_version": _int_or_none(static_rec.get("version")), + "status": status, + "why": why, + "lines": len(text.splitlines()) if text is not None else None, + "code_sha256": source_sha, + "static": static_rec, + "smoke": smoke_rec, + } + if side == "new": + record["selfcheck"] = selfcheck_block(selfcheck) + return record + + +def rule_classes(static: Mapping[str, Any]) -> tuple[list[str], list[str]]: + """The non-advisory and the advisory rule classes with a finding, each in ``RULE_ORDER``.""" + binding: set[str] = set() + advisory: set[str] = set() + findings = static.get("findings") + for finding in findings if isinstance(findings, list) else []: + if not isinstance(finding, dict) or finding.get("class") not in RULE_ORDER: + continue + (advisory if finding.get("advisory") is True else binding).add(finding["class"]) + return [c for c in RULE_ORDER if c in binding], [c for c in RULE_ORDER if c in advisory] + + +def _joined(values: Sequence[str]) -> str: + return ",".join(values) if values else NONE + + +def notice_line(record: Mapping[str, Any]) -> str: + """``::notice::adapt_check spec=... lib=... side=... status=... rules=...`` for one record.""" + static = record["static"] + smoke = record["smoke"] + binding, advisory = rule_classes(static) + fields = [ + ("spec", token(record["spec"])), + ("lib", token(record["lib"])), + ("side", token(record["side"])), + ("status", token(record["status"])), + ("static", token(static.get("status"))), + ("smoke", token(smoke.get("status"))), + ("rules", _joined(binding)), + ("advisory", _joined(advisory)), + ("perturbations", _joined(smoke.get("failed") or [])), + ("advisory_perturbations", _joined(smoke.get("advisory_failed") or [])), + ("version", token(record.get("checker_version"))), + ("lines", token(record.get("lines"))), + ] + if record["side"] == "new": + selfcheck = record.get("selfcheck") + if isinstance(selfcheck, dict): + fields += [ + ("selfcheck_attempts", token(selfcheck.get("attempts"))), + ("selfcheck_first", token(selfcheck.get("first"))), + ] + else: + fields += [("selfcheck_attempts", "n/a"), ("selfcheck_first", "n/a")] + fields.append(("why", token(record["why"]))) + return "::notice::adapt_check " + " ".join(f"{key}={value}" for key, value in fields) + + +def cmd_record(args: argparse.Namespace) -> int: + record = build_record( + side=args.side, + spec=args.spec, + lib=args.lib, + language=args.language, + source=args.source, + static=args.static, + static_exit=args.static_exit, + smoke=args.smoke, + selfcheck=args.selfcheck, + ) + try: + args.out.write_text(json.dumps(record, ensure_ascii=False) + "\n", encoding="utf-8") + except OSError as exc: + print(f"::warning::adapt record ({args.side}) not written: {' '.join(str(exc).split())[:200]}") + return 1 + print(notice_line(record)) + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Build impl-review's adapt records (record-only).") + sub = parser.add_subparsers(dest="command", required=True) + rec = sub.add_parser("record", help="write one side's adapt record and print its notice") + rec.add_argument("--side", required=True, choices=SIDES) + rec.add_argument("--spec", required=True) + rec.add_argument("--lib", required=True) + rec.add_argument("--language", required=True) + rec.add_argument("--source", type=Path, required=True, help="the checked implementation file") + rec.add_argument("--static", type=Path, required=True, help="the mirror's --out file") + rec.add_argument("--static-exit", type=int, required=True, help="the mirror's exit code") + rec.add_argument("--smoke", type=Path, default=None, help="the staged smoke-run JSON") + rec.add_argument("--selfcheck", type=Path, default=None, help="the staged Step 3c log (new side only)") + rec.add_argument("--out", type=Path, required=True) + args = parser.parse_args(argv) + return cmd_record(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/automation/scripts/regen_gate.py b/automation/scripts/regen_gate.py index 77c054817e4..c629a43e2e2 100644 --- a/automation/scripts/regen_gate.py +++ b/automation/scripts/regen_gate.py @@ -45,6 +45,25 @@ 7. no regressions -- on a ``*-basic`` spec a changed data scenario or added encodings count as regressions unless a change request asked for them. +The *adapt route* (``docs/concepts/adaptable-code.md``, "The adapt route"; +``adapt_assess``) merges a conversion to the adaptable-code convention on +render identity instead of a carrier: the predecessor fails the adaptability +check (static or smoke) and the new source passes both, the two sources' +renders from one impl-generate runner differ in at most ``IDENTITY_BOUND`` of +their pixels per theme at equal size, the new source has at most +``max(1.25 × prev, prev + 15)`` lines and the canvas gate passed; when a +review ran, it also needs a score above 0, no regression, an unchanged data +scenario, no added encoding and no technical item below its re-score. It +replaces the keeps ``no_visible_improvement``, ``no_defect_improvement`` and +``below_tolerance`` (the tolerance never applies to it), and with +``--review-skipped`` it merges before the score checks; both merges carry the +code ``adapt_identity``. The *adapt regression* -- a predecessor that passes +and a new source that fails -- turns any merge into keep +(``adapt_regression``). The gate reads the records impl-review writes and the +render pair impl-generate stages (``--adapt-new``, ``--adapt-prev``, +``--adapt-pair``) and never renders; without them the route and the +regression are off, so every older caller decides as before. + Anything missing or malformed fails closed to ``keep``. The classification keys (``prev_checklist``, ``prev_weaknesses``, ``rule``, ``kind``) never invalidate the file: a missing or malformed one only leaves fewer carriers. @@ -92,6 +111,12 @@ [--pr N] [--model ID] [--criteria-version V] [--prompts-tree SHA] \ [--prev-model ID] [--prev-criteria-version V] [--record-out FILE] \ [--prev-impl /tmp/anyplot-prev-impl.EXT --new-impl plots/S/implementations/L/B.EXT] + [--adapt-new /tmp/anyplot-adapt-new.json --adapt-prev /tmp/anyplot-adapt-prev.json \ + --adapt-pair $RUNNER_TEMP/adapt/pair.json] [--review-skipped] + + regen_gate.py adapt-route --spec-id S --library B --adapt-new FILE --adapt-prev FILE \ + --adapt-pair FILE --prev-impl /tmp/anyplot-prev-impl.py \ + --new-impl plots/S/implementations/python/B.py [--canvas-failed] regen_gate.py sanitize-source --source PREV_IMPL --out /tmp/anyplot-prev-impl.EXT [--pending] @@ -104,8 +129,10 @@ [--regen review_regen.json --prev-weaknesses /tmp/anyplot-prev-weaknesses.json \ --spec-file plots/S/specification.md [--prev-review review_prev.json]] [--warn-only] -``context`` and ``decide`` write ``key=value`` outputs to ``$GITHUB_OUTPUT`` -when it is set. ``decide`` reads the new render's ``review_checklist.json`` +``context``, ``decide`` and ``adapt-route`` write ``key=value`` outputs to +``$GITHUB_OUTPUT`` when it is set. ``adapt-route`` evaluates the adapt route's +conditions that need no review, before the review runs, and always exits 0 +(2 on a usage error). ``decide`` reads the new render's ``review_checklist.json`` from the directory of ``--regen-json``. ``check-feedback`` is the review's self-check (step 10 of the review prompt) and the workflow's warn-only format check: one line per problem, exit 1 when there is any (``--warn-only``: @@ -116,6 +143,7 @@ import argparse import json +import math import os import re import sys @@ -123,7 +151,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import Any +from typing import Any, TypeGuard MERGE = "merge" @@ -144,7 +172,9 @@ "no_visible_improvement", "no_defect_improvement", "below_tolerance", + "adapt_regression", "merge", + "adapt_identity", "script_crashed", ) @@ -793,6 +823,603 @@ def header_score(text: str) -> int | None: return None +# --------------------------------------------------------------------------- +# Adapt route (docs/concepts/adaptable-code.md, "The adapt route") +# --------------------------------------------------------------------------- + +# The gate never renders and imports no PIL: it reads three JSON files. The +# adapt records of both sources (``/tmp/anyplot-adapt-new.json`` and +# ``-prev.json``) come from impl-review's record step, which runs the checker +# mirror and reads the staged smoke runs. The gate reads their derived +# ``status`` and only cross-checks it against the static and smoke statuses +# (``derived_adapt_status``): a record whose status does not follow from them +# was not written by the record step, so it fails closed. The pair record +# (``$RUNNER_TEMP/adapt/pair.json``) comes from ``core.smoke_run pair`` in +# impl-generate, which renders both sources in one runner and writes one +# ``core.render_identity.RenderIdentity.to_dict()`` per theme. +ADAPT_IDENTITY = "adapt_identity" +ADAPT_REGRESSION = "adapt_regression" +# ``v`` of the adapt records and of the pair record. +ADAPT_RECORD_VERSION = 1 +ADAPT_STATUSES = ("pass", "fail", "unknown", "excluded") +ADAPT_STATIC_STATUSES = ("pass", "fail", "excluded", "error") +ADAPT_SMOKE_STATUSES = ("pass", "fail", "skipped", "missing", "stale") +# A ``skipped`` smoke run for a shape without a table to perturb is no gap: the +# static check is the whole check there (``adapt_record.SMOKE_NOT_APPLICABLE``; +# a unit test keeps the two equal). +ADAPT_SMOKE_NOT_APPLICABLE = ("shape:generated", "shape:network") +# The one language with a checker; any other reads as ``not_python``, never as a failure. +ADAPT_LANGUAGE = "python" +PAIR_STATUSES = ("identical", "different", "error") +ADAPT_THEMES = ("light", "dark") +# The largest share of differing pixels per theme. It equals +# ``core.render_identity.DEFAULT_BOUND`` (a unit test keeps them equal), and +# the gate applies it whatever bound the pair record names. +IDENTITY_BOUND = 0.005 +# The new source may grow to max(1.25 × prev, prev + 15) lines. +ADAPT_LINE_SLACK = 15 +# A staged file larger than this is not read (``unreadable``). +ADAPT_FILE_LIMIT = 1_000_000 +# Why a flag's file is not there: no flag, or a flag naming no file; a file that +# does not parse is ``unreadable``. +ABSENT = "absent" +MISSING = "missing" +UNREADABLE = "unreadable" +ADAPT_ROUTE = "route" +# The reasons only a review can give (``_adapt_review_why``, in its order): a +# route that held before the review and ends on one of these was overruled by +# the review (``AdaptAssessment.overruled``, the ``regen:adapt-overruled`` label). +ADAPT_REVIEW_WHY = ( + "score_zero", + "review_regression", + "scenario_changed", + "encodings_added", + "no_prev_checklist", + "technical_below", +) +# Every reason the route can give, in the order ``adapt_assess`` tests them; +# the first that applies wins, and ``route`` means none did. +ADAPT_WHY = ( + "not_python", + "no_record", + "record_invalid", + "excluded", + "new_unknown", + "prev_unknown", + "prev_passes", + "new_fails", + "pair_missing", + "pair_error", + "pair_stale", + "size_changed", + "identity_light", + "identity_dark", + "identity_both", + "no_sources", + "lines_over", + "canvas_failed", + # Only when a review ran: + *ADAPT_REVIEW_WHY, + ADAPT_ROUTE, +) +ADAPT_REVIEW_RAN = "ran" +ADAPT_REVIEW_SKIPPED = "skipped" +ADAPT_RECORD_KEYS = ("version", "prev", "new", "identity", "counted", "why", "review") +# Added after the block's first form; a block without it still validates. +ADAPT_RECORD_OPTIONAL_KEYS = ("route_pre_review",) +ADAPT_SIDE_KEYS = ("status", "static", "smoke") + + +@dataclass(frozen=True) +class AdaptInput: + """What the adapt route reads (``decide`` and ``adapt-route`` flags). + + ``new``, ``prev`` and ``pair`` are the parsed files of ``--adapt-new``, + ``--adapt-prev`` and ``--adapt-pair``; each ``*_error`` is None when its + file was read, else ``absent``, ``missing`` or ``unreadable`` + (``load_adapt_file``). ``language`` is ``source_language`` of + ``--new-impl`` (None without it). ``review_skipped`` is + ``--review-skipped``: impl-review skipped the review on the owner's switch. + ``route_held`` is ``--adapt-route-held``: impl-review's ``adapt-route``, + before the review, said the route holds. + """ + + library: str = "" + language: str | None = None + new: Any = None + prev: Any = None + pair: Any = None + new_error: str | None = ABSENT + prev_error: str | None = ABSENT + pair_error: str | None = ABSENT + review_skipped: bool = False + route_held: bool = False + + @property + def requested(self) -> bool: + """Any adapt flag was given (the notice line and the summary then report the route).""" + return ( + self.review_skipped + or self.route_held + or any(e != ABSENT for e in (self.new_error, self.prev_error, self.pair_error)) + ) + + +@dataclass(frozen=True) +class AdaptAssessment: + """The adapt route's verdict for one pair (``adapt_assess``). + + ``why`` is one of ``ADAPT_WHY``; the route counts exactly when it is + ``route``. ``prev``/``new`` summarize a valid record (``status``, + ``static``, ``smoke``), ``identity`` gives each theme's differing-pixel + fraction (None on a size change; the whole map None without a usable + pair), ``version`` is the new record's ``checker_version``. ``regression`` + is set whenever both records are valid, whatever ``why`` says, in two + cases (``adapt_regression``): a predecessor that passes and a new source + that fails, or a predecessor whose static check passes and whose smoke + run is not known (status ``unknown``) and a new source whose static check + fails. The second needs no smoke run: the static check alone already + shows that converted code was unconverted. ``route_pre_review`` is what + impl-review's ``adapt-route`` said before the review (``--adapt-route-held``): + true when it held, false when the gate had adapt input without the flag, + None without adapt input or for another language. + """ + + why: str + review: str = ADAPT_REVIEW_RAN + version: int | None = None + prev: Mapping[str, str | None] | None = None + new: Mapping[str, str | None] | None = None + identity: Mapping[str, float | None] | None = None + regression: bool = False + prev_lines: int | None = None + new_lines: int | None = None + requested: bool = False + route_pre_review: bool | None = None + + @property + def counted(self) -> bool: + return self.why == ADAPT_ROUTE + + @property + def overruled(self) -> bool: + """The route held before the review, and the review's floor turned it off. + + These PRs and the counted ones (``regen:adapt``) are the route-held + PRs the review-free switch (``ADAPT_REVIEW_FREE``) is read against. + """ + return self.route_pre_review is True and self.why in ADAPT_REVIEW_WHY + + @property + def static_regression(self) -> bool: + """The regression rests on the static check alone (the predecessor's smoke run is not known).""" + return self.regression and self.prev is not None and self.prev.get("status") != "pass" + + def to_record(self) -> dict[str, Any]: + """The gate record's ``adapt`` block: tokens, numbers, booleans and nulls only.""" + return { + "version": self.version, + "prev": dict(self.prev) if self.prev is not None else None, + "new": dict(self.new) if self.new is not None else None, + "identity": dict(self.identity) if self.identity is not None else None, + "counted": self.counted, + "why": self.why, + "review": self.review, + "route_pre_review": self.route_pre_review, + } + + +def load_adapt_file(path_text: str | None) -> tuple[Any, str | None]: + """A staged adapt JSON file and why it is not there (``absent``, ``missing``, ``unreadable``).""" + if not path_text: + return None, ABSENT + path = Path(path_text) + try: + if not path.is_file(): + return None, MISSING + if path.stat().st_size > ADAPT_FILE_LIMIT: + return None, UNREADABLE + return json.loads(path.read_text(encoding="utf-8")), None + except OSError, ValueError, RecursionError: + return None, UNREADABLE + + +def within_line_bound(prev_lines: int, new_lines: int) -> bool: + """The new source has at most max(1.25 × prev, prev + ``ADAPT_LINE_SLACK``) lines (integers only).""" + return new_lines * 4 <= prev_lines * 5 or new_lines <= prev_lines + ADAPT_LINE_SLACK + + +def _real(value: Any) -> TypeGuard[int | float]: + """A finite int or float, never a bool.""" + return isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value) + + +def derived_adapt_status(static: Any, smoke: Any, smoke_why: Any) -> str: + """The status an adapt record must carry for its static and smoke statuses. + + ``adapt_record.derive_status`` reimplemented for the single-file copy (a + unit test runs both over every combination): the first matching rule wins. + A static ``error`` is ``unknown``; ``excluded`` and ``fail`` decide alone; + then a ``missing`` or ``stale`` smoke run is ``unknown``, a failed one + ``fail``, a skip ``pass`` only for a shape without a table + (``ADAPT_SMOKE_NOT_APPLICABLE``) and ``unknown`` otherwise, and a passed + one ``pass``. + """ + if static not in ("pass", "fail", "excluded"): + return "unknown" + if static in ("excluded", "fail"): + return str(static) + if smoke in ("missing", "stale"): + return "unknown" + if smoke == "fail": + return "fail" + if smoke == "skipped": + return "pass" if smoke_why in ADAPT_SMOKE_NOT_APPLICABLE else "unknown" + return "pass" if smoke == "pass" else "unknown" + + +def adapt_record_problem(record: Any, side: str, spec_id: str, library: str) -> str | None: + """Why an adapt record cannot be read for this pair, or None when it can. + + The gate reads ``v``, ``side``, ``spec``, ``lib``, ``language``, + ``checker_version``, ``status``, ``static.status``, ``smoke.status`` and + ``smoke.why`` (contract: impl-review's record step owns the format and the + derivation). A ``status`` that does not follow from the two blocks + (``derived_adapt_status``) makes the record malformed: a crafted + ``pass`` over a missing smoke run must never open the route. + """ + if not isinstance(record, Mapping): + return "not a JSON object" + if not (_is_int(record.get("v")) and record["v"] == ADAPT_RECORD_VERSION): + return f"v is not {ADAPT_RECORD_VERSION}" + if record.get("side") != side: + return f"side is not {side}" + if record.get("spec") != spec_id or record.get("lib") != library: + return "spec or lib names another pair" + if record.get("language") != ADAPT_LANGUAGE: + return f"language is not {ADAPT_LANGUAGE}" + if record.get("status") not in ADAPT_STATUSES: + return "status is not pass, fail, unknown or excluded" + # The status is derived from these two; a record without them was not + # built by the record step, so it fails closed like any malformed one. + for key, allowed in (("static", ADAPT_STATIC_STATUSES), ("smoke", ADAPT_SMOKE_STATUSES)): + block = record.get(key) + if not isinstance(block, Mapping) or block.get("status") not in allowed: + return f"{key}.status is not one of {', '.join(allowed)}" + smoke = record["smoke"] + if record["status"] != derived_adapt_status(record["static"]["status"], smoke["status"], smoke.get("why")): + return "status does not follow from static and smoke" + return None + + +def _side_summary(record: Mapping[str, Any]) -> dict[str, str | None]: + """``status`` plus the static and smoke statuses (None when absent or unknown) of a valid record.""" + + def inner(key: str, allowed: tuple[str, ...]) -> str | None: + block = record.get(key) + value = block.get("status") if isinstance(block, Mapping) else None + return value if isinstance(value, str) and value in allowed else None + + return { + "status": record["status"], + "static": inner("static", ADAPT_STATIC_STATUSES), + "smoke": inner("smoke", ADAPT_SMOKE_STATUSES), + } + + +def _size(value: Any) -> tuple[int, int] | None: + if isinstance(value, list) and len(value) == 2 and all(_is_int(v) and v >= 0 for v in value): + return (value[0], value[1]) + return None + + +def pair_problem(pair: Any, library: str) -> str | None: + """Why a pair record cannot be read for this library, or None when it can. + + It needs ``v``, a known ``status``, a ``library`` equal to this one + (``core.smoke_run pair`` always writes it, so a record without one was + not written there) and, unless the status is ``error``, an ``identity`` + entry per theme with both sizes, a boolean ``passed`` and a ``fraction`` + that is a number or null. + """ + if not isinstance(pair, Mapping): + return "not a JSON object" + if not (_is_int(pair.get("v")) and pair["v"] == ADAPT_RECORD_VERSION): + return f"v is not {ADAPT_RECORD_VERSION}" + if pair.get("status") not in PAIR_STATUSES: + return "status is not identical, different or error" + if pair.get("library") != library: + return "library is missing or names another library" + if pair["status"] == "error": + return None + identity = pair.get("identity") + if not isinstance(identity, Mapping): + return "no identity block" + for theme in ADAPT_THEMES: + entry = identity.get(theme) + if not isinstance(entry, Mapping): + return f"no identity for {theme}" + if _size(entry.get("old_size")) is None or _size(entry.get("new_size")) is None: + return f"{theme} sizes are not two integers each" + if not isinstance(entry.get("passed"), bool): + return f"{theme} passed is not a boolean" + fraction = entry.get("fraction") + if fraction is not None and (isinstance(fraction, bool) or not isinstance(fraction, int | float)): + return f"{theme} fraction is not a number" + return None + + +def _pair_matches(pair: Mapping[str, Any], new: Any, prev: Any) -> bool: + """The pair rendered the very sources the two records checked: equal ``code_sha256`` on each side. + + Both hashes come from one rule (``core.smoke_run.code_sha256``, reimplemented + in ``adapt_record.py``). A pair rendered from an earlier head or another + predecessor measures other code, so it never counts, and neither does a + missing hash. + """ + for side, record in (("new", new), ("prev", prev)): + rendered = pair.get(side) + sha = rendered.get("code_sha256") if isinstance(rendered, Mapping) else None + checked = record.get("code_sha256") if isinstance(record, Mapping) else None + if not (isinstance(sha, str) and sha and sha == checked): + return False + return True + + +def _theme_holds(entry: Mapping[str, Any]) -> bool: + """Equal sizes, ``passed``, and a fraction from 0 to ``IDENTITY_BOUND`` (the gate's own bound).""" + fraction = entry.get("fraction") + return ( + _size(entry.get("old_size")) == _size(entry.get("new_size")) + and entry.get("passed") is True + and _real(fraction) + and 0 <= fraction <= IDENTITY_BOUND + ) + + +def _identity_fractions(pair: Mapping[str, Any]) -> dict[str, float | None]: + """Each theme's fraction for the record, rounded; None on a size change or a non-finite value.""" + out: dict[str, float | None] = {} + for theme in ADAPT_THEMES: + entry = pair["identity"][theme] + fraction = entry.get("fraction") + same = _size(entry.get("old_size")) == _size(entry.get("new_size")) + out[theme] = round(float(fraction), 6) if same and _real(fraction) else None + return out + + +def _encodings(payload: Mapping[str, Any]) -> list[str]: + raw = payload.get("encodings_added") + return [e.strip() for e in raw if isinstance(e, str) and e.strip()] if isinstance(raw, list) else [] + + +def review_regressions(inp: GateInput, payload: Mapping[str, Any]) -> list[dict[str, str]]: + """The review's regressions, plus a replaced scenario or added encodings on a ``-basic`` spec. + + On a ``-basic`` spec only a change request that exists and that the review + says it applied may swap the scenario or add encodings. ``_judge`` keeps on + any of these; ``adapt_assess`` turns the route off on them. + """ + raw = payload.get("regressions") + regressions = [ + {"what": item["what"].strip(), "where_visible": str(item.get("where_visible") or "").strip()} + for item in (raw if isinstance(raw, list) else []) + if isinstance(item, Mapping) and isinstance(item.get("what"), str) and item["what"].strip() + ] + change_request_applied = inp.change_request_present and payload.get("change_request_applied") is True + if inp.spec_id.endswith("-basic") and not change_request_applied: + if payload.get("scenario_changed") is True: + regressions.append( + {"what": "data scenario replaced on a -basic spec", "where_visible": "data and labels of both renders"} + ) + encodings = _encodings(payload) + if encodings: + regressions.append( + {"what": f"encodings added on a -basic spec: {', '.join(encodings)}", "where_visible": "both renders"} + ) + return regressions + + +def _adapt_review_why(inp: GateInput, review: Mapping[str, Any]) -> str | None: + """The first review-dependent condition that fails (the review's sanity floor), or None.""" + if inp.score is None or inp.score <= 0: + return "score_zero" + if review_regressions(inp, review): + return "review_regression" + if review.get("scenario_changed") is True: + return "scenario_changed" + if _encodings(review): + return "encodings_added" + prev_checklist = flat_scores(review.get("prev_checklist")) + if not prev_checklist: + return "no_prev_checklist" + for cid, prev in prev_checklist.items(): + new = inp.new_checklist.get(cid) + if cid[:2] in TECHNICAL_PREFIXES and (new is None or new < prev): + return "technical_below" + return None + + +def adapt_assess(inp: GateInput, review: Mapping[str, Any] | None) -> AdaptAssessment: + """Whether the adapt route counts for this pair, and why not (``ADAPT_WHY``, in order). + + ``review`` is None when no review ran (``adapt-route`` before the review, + or ``decide --review-skipped``); otherwise it is the validated, normalized + ``review_regen.json`` (an empty mapping when there is none), and the + review's floor applies: a score above 0, no regression (``_judge``'s list, + the ``-basic`` scenario and encoding rules included), an unchanged data + scenario and no added encoding on any spec, a ``prev_checklist``, and no + VQ, SC, DQ or CQ item of it missing from the new checklist or below its + value there. A missing or malformed record or pair, or a source the gate + did not read, turns the route off with its reason; nothing counts as a pass. + """ + adapt = inp.adapt + python = adapt.language in (None, ADAPT_LANGUAGE) + new_ok = adapt.new_error is None and adapt_record_problem(adapt.new, "new", inp.spec_id, adapt.library) is None + prev_ok = adapt.prev_error is None and adapt_record_problem(adapt.prev, "prev", inp.spec_id, adapt.library) is None + new = _side_summary(adapt.new) if python and new_ok else None + prev = _side_summary(adapt.prev) if python and prev_ok else None + version = adapt.new.get("checker_version") if new is not None else None + pair_ok = adapt.pair_error is None and pair_problem(adapt.pair, adapt.library) is None + usable_pair = python and pair_ok and adapt.pair["status"] != "error" + prev_lines = len(inp.prev_source.splitlines()) if inp.prev_source is not None else None + new_lines = len(inp.new_source.splitlines()) if inp.new_source is not None else None + + def why() -> str: + if not python: + return "not_python" + if adapt.new_error in (ABSENT, MISSING) or adapt.prev_error in (ABSENT, MISSING): + return "no_record" + if new is None or prev is None: + return "record_invalid" + if "excluded" in (new["status"], prev["status"]): + return "excluded" + if new["status"] == "unknown": + return "new_unknown" + if prev["status"] == "unknown": + return "prev_unknown" + if prev["status"] == "pass": + return "prev_passes" + if new["status"] == "fail": + return "new_fails" + if not pair_ok: + return "pair_missing" + if not usable_pair: + return "pair_error" + if not _pair_matches(adapt.pair, adapt.new, adapt.prev): + return "pair_stale" + identity = adapt.pair["identity"] + if any(_size(identity[t].get("old_size")) != _size(identity[t].get("new_size")) for t in ADAPT_THEMES): + return "size_changed" + failing = [t for t in ADAPT_THEMES if not _theme_holds(identity[t])] + if failing: + return "identity_both" if len(failing) == len(ADAPT_THEMES) else f"identity_{failing[0]}" + if prev_lines is None or new_lines is None: + return "no_sources" + if not within_line_bound(prev_lines, new_lines): + return "lines_over" + if inp.canvas_failed: + return "canvas_failed" + if review is not None: + return _adapt_review_why(inp, review) or ADAPT_ROUTE + return ADAPT_ROUTE + + return AdaptAssessment( + why=why(), + review=ADAPT_REVIEW_SKIPPED if review is None else ADAPT_REVIEW_RAN, + version=version if _is_int(version) else None, + prev=prev, + new=new, + identity=_identity_fractions(adapt.pair) if usable_pair else None, + regression=_adapt_regression(prev, new), + prev_lines=prev_lines, + new_lines=new_lines, + requested=adapt.requested, + route_pre_review=True if adapt.route_held else (False if python and adapt.requested else None), + ) + + +def _adapt_regression(prev: Mapping[str, str | None] | None, new: Mapping[str, str | None] | None) -> bool: + """Converted code that this regeneration unconverted, from two valid records' summaries. + + Either the predecessor passes and the new source fails, or the + predecessor's static check passes while its smoke run is not known + (``unknown``: missing, stale or skipped) and the new source's static check + fails. A predecessor whose own smoke run failed passes nothing, so a + failing new source is no regression from it. + """ + if prev is None or new is None or new["status"] != "fail": + return False + if prev["status"] == "pass": + return True + return prev["status"] == "unknown" and prev["static"] == "pass" and new["static"] == "fail" + + +def _review_skipped(inp: GateInput) -> bool: + """``--review-skipped`` with no score: a score means a review ran, and the stricter path applies.""" + return inp.adapt.review_skipped and inp.score is None + + +def _adapt_review(inp: GateInput) -> Mapping[str, Any] | None: + """The review the route's floor reads: None when it was skipped, ``{}`` when it left no valid file.""" + if _review_skipped(inp): + return None + if inp.regen is None: + return {} + payload, _ = normalize_regen(inp.regen) + if validate_regen(payload, inp.known_weakness_ids, inp.characteristic_count): + return {} + return payload if isinstance(payload, dict) else {} + + +def _percent(fraction: float | None) -> str: + return "n/a" if fraction is None else f"{fraction * 100:.2f} %" + + +def adapt_route_text(adapt: AdaptAssessment) -> str: + """One line on a counted route: both checks, both themes' identity and the line counts.""" + prev, identity = adapt.prev or {}, adapt.identity or {} + return ( + f"the predecessor fails the adaptability check (static {prev.get('static') or 'n/a'}, smoke " + f"{prev.get('smoke') or 'n/a'}) and the new source passes it; the renders differ in " + f"{_percent(identity.get('light'))} (light) and {_percent(identity.get('dark'))} (dark) of their pixels, " + f"within {_percent(IDENTITY_BOUND)}; {_score_text(adapt.prev_lines)} → {_score_text(adapt.new_lines)} lines" + ) + + +# What each reason means, for the PR comment's one-line summary of the route. +ADAPT_WHY_TEXT = { + "not_python": "only Python implementations have the adaptability check", + "no_record": "an adaptability record is missing", + "record_invalid": "an adaptability record is malformed or names another pair", + "excluded": "the spec is excluded from the adaptability check", + "new_unknown": "the new source's adaptability check is incomplete", + "prev_unknown": "the predecessor's adaptability check is incomplete", + "prev_passes": "the predecessor already passes the adaptability check, so this is no conversion", + "new_fails": "the new source fails the adaptability check", + "pair_missing": "the render pair from impl-generate is missing or malformed", + "pair_error": "a render of the pair failed in impl-generate", + "pair_stale": "the render pair was made from other source versions than the two checked here", + "size_changed": "the two sources' renders differ in size", + "identity_light": f"the light renders differ in more than {_percent(IDENTITY_BOUND)} of their pixels", + "identity_dark": f"the dark renders differ in more than {_percent(IDENTITY_BOUND)} of their pixels", + "identity_both": f"both themes' renders differ in more than {_percent(IDENTITY_BOUND)} of their pixels", + "no_sources": "a source was not read", + "lines_over": f"the new source is longer than max(1.25 × prev, prev + {ADAPT_LINE_SLACK}) lines", + "canvas_failed": "the canvas gate failed", + "score_zero": "the review gave no score above 0", + "review_regression": "the review names a regression", + "scenario_changed": "the review says the data scenario changed", + "encodings_added": "the review lists added encodings", + "no_prev_checklist": "the review wrote no re-score checklist", + "technical_below": "a VQ, SC, DQ or CQ item scores below its re-score", +} + + +def adapt_regression_text(adapt: AdaptAssessment) -> str: + """What the adapt regression rests on: both checks, or the static check alone.""" + if adapt.static_regression: + return ( + "the predecessor passes the static adaptability check (its smoke run is not known) " + "and the new source fails it" + ) + return "the predecessor passes the adaptability check and the new source fails it" + + +def adapt_summary_line(adapt: AdaptAssessment) -> str: + """``**Adapt route:** …`` for the PR comment; fixed text, tokens and numbers only.""" + if adapt.regression: + text = f"regression: {adapt_regression_text(adapt)}, so the live implementation stays" + elif adapt.counted: + review = "no review ran" if adapt.review == ADAPT_REVIEW_SKIPPED else "the review's floor holds" + text = f"counted: {adapt_route_text(adapt)}; {review}" + else: + text = f"not counted ({adapt.why}): {ADAPT_WHY_TEXT.get(adapt.why, adapt.why)}" + if adapt.overruled: + text += "; the route held before the review, so the review overruled it" + return f"**Adapt route:** {text}" + + # --------------------------------------------------------------------------- # Decision # --------------------------------------------------------------------------- @@ -826,6 +1453,10 @@ class GateInput: # The sources' language (``source_language`` of ``--new-impl``): what # ``call_count`` reads as a comment, a string and a definition. source_language: str = "python" + # The adapt route's records, render pair and review switch + # (``--adapt-new``/``--adapt-prev``/``--adapt-pair``/``--review-skipped``). + # The default has none, which turns the route and the regression off. + adapt: AdaptInput = field(default_factory=AdaptInput) @dataclass @@ -850,9 +1481,11 @@ class GateResult: code_improvements: list[dict[str, Any]] = field(default_factory=list) prev_lines: int | None = None new_lines: int | None = None + # The adapt route's assessment (``decide`` sets it on every result). + adapt: AdaptAssessment | None = None -def _is_int(value: Any) -> bool: +def _is_int(value: Any) -> TypeGuard[int]: return isinstance(value, int) and not isinstance(value, bool) @@ -1021,18 +1654,33 @@ def decide(inp: GateInput) -> GateResult: """Apply the regen gate. Every uncertain input resolves to ``keep``. The two sources' line counts are on every result when both were read, - so an early keep still reports them. + so an early keep still reports them. The adapt route is assessed once, + with the review the decision reads (``_adapt_review``), and its + assessment is on every result. Last, the adapt regression turns any merge + into keep (``adapt_regression``). """ - result = _decide(inp) + adapt = adapt_assess(inp, _adapt_review(inp)) + result = _decide(inp, adapt) + result.adapt = adapt if inp.prev_source is not None and inp.new_source is not None: result.prev_lines = len(inp.prev_source.splitlines()) result.new_lines = len(inp.new_source.splitlines()) + if result.verdict == MERGE and adapt.regression: + result.reason = ( + f"adapt regression: {adapt_regression_text(adapt)}, " + f"so the live implementation stays (without it: {result.code})" + ) + result.verdict, result.code = KEEP, ADAPT_REGRESSION return result -def _decide(inp: GateInput) -> GateResult: +def _decide(inp: GateInput, adapt: AdaptAssessment | None = None) -> GateResult: if inp.canvas_failed: return GateResult(KEEP, "canvas dimension gate failed", "canvas_failed") + # No review ran (the owner's switch): the route is the only way to merge, + # and a pair it does not carry is kept below for the lack of a score. + if _review_skipped(inp) and adapt is not None and adapt.counted: + return GateResult(MERGE, f"adapt route, no review: {adapt_route_text(adapt)}", ADAPT_IDENTITY) if inp.score is None: return GateResult(KEEP, "no valid review score", "no_score") if inp.score == 0: @@ -1051,7 +1699,7 @@ def _decide(inp: GateInput) -> GateResult: return GateResult(KEEP, f"review_regen.json {error}", code) regen, coerced = normalize_regen(inp.regen) - result = _judge(inp, regen) + result = _judge(inp, regen, adapt) if coerced: result.reason += f" (coerced: {'; '.join(coerced)})" result.coerced = True @@ -1365,7 +2013,17 @@ def classify_code_improvements(payload: Any, inp: GateInput) -> list[dict[str, A return out -def _judge(inp: GateInput, regen: Any) -> GateResult: +def _adapt_instead(result: GateResult, adapt: AdaptAssessment | None) -> GateResult: + """A no-carrier or below-tolerance keep becomes a merge when the adapt route counts.""" + if adapt is None or not adapt.counted: + return result + floor = "no technical item below its re-score" + result.reason = f"adapt route: {adapt_route_text(adapt)}; {floor} (without the route: {result.code})" + result.verdict, result.code = MERGE, ADAPT_IDENTITY + return result + + +def _judge(inp: GateInput, regen: Any, adapt: AdaptAssessment | None = None) -> GateResult: errors = validate_regen(regen, inp.known_weakness_ids, inp.characteristic_count) if errors: return GateResult(KEEP, "invalid review_regen.json: " + "; ".join(errors), "regen_json_invalid") @@ -1373,24 +2031,9 @@ def _judge(inp: GateInput, regen: Any) -> GateResult: payload: dict[str, Any] = regen prev_rescored: int = payload["prev_rescored"] improvements = classify_improvements(payload, inp) - regressions = [ - {"what": item["what"].strip(), "where_visible": str(item.get("where_visible") or "").strip()} - for item in payload["regressions"] - ] - # A change request is the only thing that may legitimately swap the # scenario or add encodings on a -basic spec — and only one that exists. - change_request_applied = inp.change_request_present and payload.get("change_request_applied") is True - if inp.spec_id.endswith("-basic") and not change_request_applied: - if payload["scenario_changed"]: - regressions.append( - {"what": "data scenario replaced on a -basic spec", "where_visible": "data and labels of both renders"} - ) - encodings = [e.strip() for e in payload["encodings_added"] if e.strip()] - if encodings: - regressions.append( - {"what": f"encodings added on a -basic spec: {', '.join(encodings)}", "where_visible": "both renders"} - ) + regressions = review_regressions(inp, payload) result = GateResult( KEEP, @@ -1441,7 +2084,7 @@ def _judge(inp: GateInput, regen: Any) -> GateResult: detail = " (every improvement needs a non-empty where_visible)" if counted or not (cited or obsolete) else "" result.code = "no_visible_improvement" result.reason = f"no visible improvement{detail}{note}{code_note}" - return result + return _adapt_instead(result, adapt) if not carriers and not code_counted: kinds = [ @@ -1461,12 +2104,16 @@ def _judge(inp: GateInput, regen: Any) -> GateResult: f"{len(visible)} visible improvement(s) ({breakdown}), none fixes a verified defect " f"or an 'A good version shows' property{note}{code_note}" ) - return result + return _adapt_instead(result, adapt) - if inp.score < prev_rescored - 1: + # The tolerance guards the carrier and the CQ-04 code paths; the adapt + # route's floor is the technical items, so it takes this keep over too. + # ``_decide`` keeps a missing score before ``_judge`` runs. + score = inp.score if inp.score is not None else -1 + if score < prev_rescored - 1: result.code = "below_tolerance" result.reason = f"new score {inp.score} < re-scored predecessor {prev_rescored} - 1{note}" - return result + return _adapt_instead(result, adapt) result.verdict = MERGE result.code = "merge" @@ -2042,6 +2689,9 @@ def render_summary(result: GateResult, prev_stored: str, score: int | None) -> s lines.append(f"- `{ref}` {_safe(item['what'])} — {where}{flag}") if result.prev_lines is not None and result.new_lines is not None: lines += ["", f"**Code size:** {result.prev_lines} → {result.new_lines} lines"] + adapt = result.adapt + if adapt is not None and (adapt.requested or result.code in (ADAPT_IDENTITY, ADAPT_REGRESSION)): + lines += ["", adapt_summary_line(adapt)] lines += ["", "**Regressions**"] if result.regressions: for item in result.regressions: @@ -2086,6 +2736,7 @@ def render_summary(result: GateResult, prev_stored: str, score: int | None) -> s "coerced", "at", "writeback", + "adapt", } ) # What happened to the re-score of the live implementation on a keep @@ -2184,7 +2835,12 @@ def build_record( (from P8) counts the code improvements that hold (``classify_code_improvements``): with no carrier they carried the merge (the code path), with one they rode along; records written before P8 - lack it. + lack it. ``adapt`` (from PR 2 of the adaptable-code rollout) is the + adapt route's assessment (``AdaptAssessment.to_record``); a result + without one records the route as off for want of a record. Records + written before it lack the key. Its ``route_pre_review`` (whether + impl-review's ``adapt-route`` held before the review) came after the + block's first form, so the validator accepts a block without it. """ kinds = {"W": 0, "P": 0, "C": 0, "new": 0} for item in result.improvements: @@ -2219,16 +2875,72 @@ def build_record( "encodings_added": result.encodings_added, "coerced": result.coerced, "at": at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "adapt": (result.adapt or AdaptAssessment(why="no_record")).to_record(), } +def adapt_record_errors(adapt: Any) -> list[str]: + """Schema check for a record's ``adapt`` block (``AdaptAssessment.to_record``). + + Known keys only; a ``version`` that is an integer or null; ``prev`` and + ``new`` null or the three statuses (``status`` one of ``ADAPT_STATUSES``, + ``static`` and ``smoke`` known or null); ``identity`` null or one finite + fraction from 0 to 1 (or null) per theme; a boolean ``counted`` that is + true exactly when ``why`` is ``route``; ``why`` one of ``ADAPT_WHY``; + ``review`` ``ran`` or ``skipped``; ``route_pre_review``, when present, + true, false or null. No lists anywhere. + """ + if not isinstance(adapt, dict): + return ["record.adapt is not an object"] + known = (*ADAPT_RECORD_KEYS, *ADAPT_RECORD_OPTIONAL_KEYS) + errors = [f"record.adapt has unknown key {key!r}" for key in adapt if key not in known] + errors += [f"record.adapt lacks {key!r}" for key in ADAPT_RECORD_KEYS if key not in adapt] + route_pre_review = adapt.get("route_pre_review") + if route_pre_review is not None and not isinstance(route_pre_review, bool): + errors.append("record.adapt.route_pre_review must be true, false or null") + version = adapt.get("version") + if version is not None and not _is_int(version): + errors.append("record.adapt.version must be an integer or null") + allowed = {"status": ADAPT_STATUSES, "static": ADAPT_STATIC_STATUSES, "smoke": ADAPT_SMOKE_STATUSES} + for side in ("prev", "new"): + value = adapt.get(side) + if value is None: + continue + if not isinstance(value, dict) or set(value) != set(ADAPT_SIDE_KEYS): + errors.append(f"record.adapt.{side} must be null or an object with {', '.join(ADAPT_SIDE_KEYS)}") + continue + for key, values in allowed.items(): + inner = value[key] + if not ((inner is None and key != "status") or (isinstance(inner, str) and inner in values)): + errors.append(f"record.adapt.{side}.{key} is not a known status") + identity = adapt.get("identity") + if identity is not None: + if not isinstance(identity, dict) or set(identity) != set(ADAPT_THEMES): + errors.append("record.adapt.identity must be null or an object with light and dark") + else: + for theme, fraction in identity.items(): + if fraction is not None and not (_real(fraction) and 0 <= fraction <= 1): + errors.append(f"record.adapt.identity.{theme} must be a fraction from 0 to 1 or null") + counted, why = adapt.get("counted"), adapt.get("why") + if not isinstance(counted, bool): + errors.append("record.adapt.counted must be true or false") + if not (isinstance(why, str) and why in ADAPT_WHY): + errors.append("record.adapt.why is not one of ADAPT_WHY") + elif isinstance(counted, bool) and counted != (why == ADAPT_ROUTE): + errors.append("record.adapt.counted must be true exactly when why is route") + if adapt.get("review") not in (ADAPT_REVIEW_RAN, ADAPT_REVIEW_SKIPPED): + errors.append("record.adapt.review must be ran or skipped") + return errors + + def validate_record(record: Any) -> list[str]: """Schema check for a gate record. Returns error strings. Known keys only, no free text anywhere, and a ``v`` / ``verdict`` / ``code`` this module can have written: ``RECORD_VERSION``, ``MERGE`` or ``KEEP``, and one of ``REASON_CODES``. ``writeback`` is optional, one of - ``WRITEBACK_CODES``, and only on a keep. + ``WRITEBACK_CODES``, and only on a keep. ``adapt`` is optional and has + exactly the keys of ``AdaptAssessment.to_record`` (``adapt_record_errors``). """ if not isinstance(record, dict): return ["record is not a JSON object"] @@ -2247,6 +2959,8 @@ def validate_record(record: Any) -> list[str]: errors.append("record.writeback is not one of WRITEBACK_CODES") elif record.get("verdict") != KEEP: errors.append("record.writeback belongs to a keep record only") + if "adapt" in record: + errors += adapt_record_errors(record["adapt"]) def walk(value: Any, where: str) -> None: if isinstance(value, dict): @@ -2332,6 +3046,83 @@ def _optional_int(raw: str | None) -> int | None: return int(raw) +def _sources(args: argparse.Namespace) -> tuple[str | None, str | None, str | None]: + """``(prev, new, language)`` of ``--prev-impl``/``--new-impl``. + + A suffix the gate cannot read as code, or two different languages, leaves + both sources unread, like an unreadable file. + """ + language = source_language(args.new_impl) + if language != source_language(args.prev_impl): + language = None + if not language: + return None, None, None + return load_source(args.prev_impl), load_source(args.new_impl), language + + +def _adapt_input(args: argparse.Namespace, review_skipped: bool = False, route_held: bool = False) -> AdaptInput: + new, new_error = load_adapt_file(args.adapt_new) + prev, prev_error = load_adapt_file(args.adapt_prev) + pair, pair_error = load_adapt_file(args.adapt_pair) + return AdaptInput( + library=args.library, + language=source_language(args.new_impl), + new=new, + prev=prev, + pair=pair, + new_error=new_error, + prev_error=prev_error, + pair_error=pair_error, + review_skipped=review_skipped, + route_held=route_held, + ) + + +def _bool_text(value: bool) -> str: + return "true" if value else "false" + + +def _fraction_text(value: Any) -> str: + return f"{value:.4f}" if _real(value) else "n/a" + + +def cmd_adapt_route(args: argparse.Namespace) -> int: + """The adapt route's review-free conditions, before the review runs; always exit 0. + + Any failure turns the route off (``record_invalid``) with a warning, so + impl-review's record step never reads a route that was not assessed. + """ + try: + prev_source, new_source, language = _sources(args) + inp = GateInput( + spec_id=args.spec_id, + score=None, + regen=None, + canvas_failed=args.canvas_failed, + prev_source=prev_source, + new_source=new_source, + source_language=language or "python", + adapt=_adapt_input(args), + ) + adapt = adapt_assess(inp, None) + except Exception as exc: + print(f"::warning::adapt-route could not assess the pair, so the route is off: {_one_line(str(exc))[:200]}") + adapt = AdaptAssessment(why="record_invalid") + prev = (adapt.prev or {}).get("status") or "n/a" + new = (adapt.new or {}).get("status") or "n/a" + identity = adapt.identity or {} + print( + f"::notice::adapt_route spec={args.spec_id} lib={args.library} prev={prev} new={new} " + f"light={_fraction_text(identity.get('light'))} dark={_fraction_text(identity.get('dark'))} " + f"lines={_score_text(adapt.prev_lines)}->{_score_text(adapt.new_lines)} " + f"route={_bool_text(adapt.counted)} why={adapt.why}" + ) + _write_outputs( + {"adapt_route": _bool_text(adapt.counted), "adapt_why": adapt.why, "adapt_prev": prev, "adapt_new": new} + ) + return 0 + + def cmd_decide(args: argparse.Namespace) -> int: regen, regen_error = load_regen_json(Path(args.regen_json)) characteristics: list[str] = [] @@ -2339,9 +3130,7 @@ def cmd_decide(args: argparse.Namespace) -> int: characteristics = parse_characteristics(Path(args.spec_file).read_text(encoding="utf-8")) score = parse_score(args.score) weaknesses_json = Path(args.weaknesses_json) if args.weaknesses_json else None - language = source_language(args.new_impl) - if language != source_language(args.prev_impl): - language = None + prev_source, new_source, language = _sources(args) inp = GateInput( spec_id=args.spec_id, score=score, @@ -2364,11 +3153,15 @@ def cmd_decide(args: argparse.Namespace) -> int: weakness_texts=load_weakness_texts(weaknesses_json), # A suffix the gate cannot read as code (or two different languages) # leaves the code path off, like an unreadable file. - prev_source=load_source(args.prev_impl) if language else None, - new_source=load_source(args.new_impl) if language else None, + prev_source=prev_source, + new_source=new_source, source_language=language or "python", + # All optional too: without them the adapt route and the adapt + # regression are off. + adapt=_adapt_input(args, review_skipped=args.review_skipped, route_held=args.adapt_route_held), ) result = decide(inp) + adapt = result.adapt or AdaptAssessment(why="no_record") rescored = str(result.prev_rescored) if result.prev_rescored is not None else "n/a" reason = _one_line(result.reason) # The stable prefix (spec … verdict) is what readers match on; the new @@ -2382,10 +3175,17 @@ def cmd_decide(args: argparse.Namespace) -> int: lines = "" if args.prev_impl or args.new_impl: lines = f" prev_lines={_score_text(result.prev_lines)} new_lines={_score_text(result.new_lines)}" + # Only with an adapt flag, for the same reason. + adapt_fields = "" + if inp.adapt.requested: + adapt_fields = ( + f" adapt_counted={_bool_text(adapt.counted)} adapt_why={adapt.why}" + f" adapt_overruled={_bool_text(adapt.overruled)}" + ) print( f"::notice::regen_gate spec={args.spec_id} lib={args.library} prev_stored={args.prev_stored} " f"prev_rescored={rescored} new={score if score is not None else 'n/a'} verdict={result.verdict} " - f"code={result.code}{provenance}{lines} reason={reason}" + f"code={result.code}{provenance}{lines}{adapt_fields} reason={reason}" ) if args.summary_out: Path(args.summary_out).write_text(render_summary(result, args.prev_stored, score), encoding="utf-8") @@ -2410,7 +3210,19 @@ def cmd_decide(args: argparse.Namespace) -> int: Path(args.record_out).write_text(json.dumps(record, separators=(",", ":")) + "\n", encoding="utf-8") except Exception as exc: print(f"::warning::regen gate record not written: {_one_line(str(exc))[:200]}") - _write_outputs({"verdict": result.verdict, "reason": reason, "prev_rescored": rescored, "code": result.code}) + _write_outputs( + { + "verdict": result.verdict, + "reason": reason, + "prev_rescored": rescored, + "code": result.code, + "adapt_counted": _bool_text(adapt.counted), + "adapt_why": adapt.why, + # The pre-review route held and the review's floor turned it off + # (impl-review labels the PR `regen:adapt-overruled`). + "adapt_overruled": _bool_text(adapt.overruled), + } + ) return 0 @@ -2574,10 +3386,37 @@ def build_parser() -> argparse.ArgumentParser: # The code path's sources; without both, it is off (older callers unchanged). dec.add_argument("--prev-impl", default="", help="the predecessor's source (/tmp/anyplot-prev-impl.EXT)") dec.add_argument("--new-impl", default="", help="the regenerated source") + # The adapt route's inputs; without them it is off (older callers unchanged). + _add_adapt_files(dec) + dec.add_argument( + "--review-skipped", action="store_true", help="no review ran (ADAPT_REVIEW_FREE): only the adapt route merges" + ) + dec.add_argument( + "--adapt-route-held", + action="store_true", + help="impl-review's adapt-route held before the review (the record's adapt.route_pre_review)", + ) dec.set_defaults(func=cmd_decide) + + route = sub.add_parser( + "adapt-route", help="Evaluate the adapt route's review-free conditions before the review (always exits 0)" + ) + route.add_argument("--spec-id", required=True) + route.add_argument("--library", required=True) + _add_adapt_files(route) + route.add_argument("--prev-impl", default="", help="the predecessor's source (/tmp/anyplot-prev-impl.py)") + route.add_argument("--new-impl", default="", help="the regenerated source") + route.add_argument("--canvas-failed", action="store_true") + route.set_defaults(func=cmd_adapt_route) return parser +def _add_adapt_files(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--adapt-new", default="", help="the new source's record (/tmp/anyplot-adapt-new.json)") + parser.add_argument("--adapt-prev", default="", help="the predecessor's record (/tmp/anyplot-adapt-prev.json)") + parser.add_argument("--adapt-pair", default="", help="impl-generate's render pair ($RUNNER_TEMP/adapt/pair.json)") + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) return int(args.func(args)) diff --git a/automation/scripts/regen_writeback.py b/automation/scripts/regen_writeback.py index 09d9a3c8f9b..0b085038dfd 100644 --- a/automation/scripts/regen_writeback.py +++ b/automation/scripts/regen_writeback.py @@ -10,6 +10,9 @@ verdict, review model and rules version — and puts the score into the ``Quality: N/100`` header (owner decision O9). ``updated``, ``rendered_at``, ``impl_tags``, the previews and every other key stay: the code did not change. +The one exception is ``review.carried``, which a review-free adapt merge sets +on the review it carries over (impl-review "Update metadata and implementation +header"): a re-score is a review that ran, so the write-back removes the mark. ``main`` takes changes only through a merged pull request, so the write-back travels as one: @@ -86,6 +89,9 @@ REVIEW_FIELDS = ("image_description", "criteria_checklist", "strengths", "weaknesses") PROVENANCE_FIELDS = ("model", "criteria_version") WRITABLE_REVIEW_KEYS = frozenset(REVIEW_FIELDS + ("verdict",) + PROVENANCE_FIELDS) +# Set on a review that a review-free adapt merge carried over unchanged +# (impl-review.yml). A write-back may remove it, never add or change it. +CARRIED_KEY = "carried" # A kept implementation stays live, which is what the stored verdict says (the # website's `review_verdict`). The reviewer writes no verdict: the bar it would # need is not in its prompts, and a kept review_prev.json that still carries @@ -173,8 +179,9 @@ def apply_review( Sets ``quality_score``, the four review fields of ``review_prev.json`` and ``review.verdict`` (``KEPT_VERDICT``, whatever the file says); sets ``review.model`` and ``review.criteria_version`` or, when the session - could not resolve one, removes it (as the metadata writer does). Every - other key keeps its value and its place. + could not resolve one, removes it (as the metadata writer does); removes + ``review.carried``, since the re-score is a review that ran. Every other + key keeps its value and its place. """ missing = [key for key in REVIEW_FIELDS if key not in prev_review] if missing: @@ -196,6 +203,7 @@ def apply_review( review[key] = value else: review.pop(key, None) + review.pop(CARRIED_KEY, None) out["review"] = review return out @@ -266,6 +274,8 @@ def metadata_diff_problems(old_text: str, new_text: str) -> list[str]: for key in sorted(set(old_review) | set(new_review), key=str): if key in WRITABLE_REVIEW_KEYS: continue + if key == CARRIED_KEY and key not in new_review: + continue # the mark of a carried review goes with the re-score if old_review.get(key) != new_review.get(key) or (key in old_review) != (key in new_review): problems.append( f"review key {key!r} changed; a write-back changes {', '.join(sorted(WRITABLE_REVIEW_KEYS))}" diff --git a/changelog.d/adaptable-code-rules.md b/changelog.d/adaptable-code-rules.md new file mode 100644 index 00000000000..15bb6862144 --- /dev/null +++ b/changelog.d/adaptable-code-rules.md @@ -0,0 +1,54 @@ +### Added + +- **The adaptable-code convention in all 15 library prompts.** A new + implementation keeps one replaceable `# Data` block that ends in a tidy + DataFrame `df` (or a `tibble`, a Julia `DataFrame`, or JavaScript records), + names each spec role once as a column constant with a `# role:` comment, lists + all eight Imprint hues and uses the first `n`, and derives labels, callout + targets, and limits from the data below the block. People who copy the code + and the "Use with my data" agent replace the block and the constants, and + nothing below them. Regenerations of R, Julia, and JavaScript pairs keep + their regeneration rules, because only Python has a checker so far. (#12261) +- **A Python self-check before every generation commits (Step 3c).** + `python -m core.smoke_run selfcheck` runs the static adaptability check and + the perturbation smoke run on the new file, up to three attempts. It never + blocks the commit, and the commit body records the result, so a first + generation that still fails merges and converts on its next regeneration. + (#12261) +- **A record-only smoke run and a predecessor render in `impl-generate.yml`.** + The step "Adaptability smoke run (record-only)" runs the smoke run on the + generated file and, on a regeneration, on the predecessor too. It renders both + sources in the same runner and stages the results and the raw renders under a + private `adapt/` prefix in GCS staging, which `impl-merge.yml` never promotes. + `impl-repair.yml` reruns the smoke run after every repair. Identity is measured + on the raw renders, before the PNGs are quantized. (#12261) +- **A record-only adaptability step in `impl-review.yml`.** The step + "Adaptability check (record-only)" runs the checker mirror on both sources + and reads the staged smoke results. It writes one record per source and an + `adapt_check` notice per side, and it never fails the run. The reviewer never + sees it, and the review prompt and the rubric are unchanged. (#12261) +- **The adapt route and the adapt regression in the regen gate.** A + regeneration that converts a Python implementation to the convention now + merges on render identity, without a visible improvement, when all of these + hold: the predecessor fails the check and the new source passes it, the two + renders differ in at most 0.5 % of their pixels in both themes at equal size, + the new source stays within max(1.25 × prev, prev + 15) lines, and the review + keeps every technical item at or above its re-score. The reason code is + `adapt_identity`, and the gate record gains an `adapt` block. A conversion + that breaks the check (the predecessor passes, the new source fails) is kept + as `adapt_regression`, on every spec, so converted code stays converted. + `regen_gate.py adapt-route` evaluates the route before the review. The + repository variable `ADAPT_REVIEW_FREE` lets identical conversions merge + without a review. It is off until the owner turns it on. The label + `regen:adapt` counts the merges on which the route held through the review, + and `regen:adapt-overruled` counts the PRs whose route held before the review + and that the review turned off. (#12261) + +### Changed + +- **`impl-review.yml` runs on the pinned Python.** A setup step reads + `.python-version` before the first regen tool runs, because the regen gate, + the provenance script, and the checker mirror use PEP 758 `except A, B:` + since the move to Python 3.14, and the runner image's own Python is not + pinned. `impl-merge.yml` now promotes only the `plot-*` files from staging, + so the private `adapt/` artifacts never reach production. (#12261) diff --git a/core/perturbation.py b/core/perturbation.py index 2b4c0096c27..2a6f95fb728 100644 --- a/core/perturbation.py +++ b/core/perturbation.py @@ -29,14 +29,26 @@ suffixes the values of the categorical columns. The colour-driving role's column keeps its values, numeric codes included (groups 0 and 1 stay two groups). * `scaled`: every numeric column times 1,000 plus 100,000. +* `missing_number` and `missing_nullable`: the sample in the dtypes the agent's loader + hands a table over in (`agents/anyplot/code/loader.py`, `PANDAS_DTYPES`: text becomes + `string`, integers `Int64`, floats `float64`, booleans `boolean`; datetimes stay as they + are), with one missing cell per column of a kind: `NaN` in every `float64` column, and + `` in every `Int64`, `boolean` and `string` column. The i-th such column gets its + missing cell in row `(len(df) // 2 + i) % len(df)`, so no row is entirely missing. No + other dtype is perturbed (datetimes in particular are not). Each is left out when the cast + table has no column of its kind. `missing_nullable` is `advisory` (open decision O2 of + the PR 2 contract): whether the convention allows `.astype(float)` on a drawn nullable + integer column, or the loader changes, is still open, so its failure is reported but does + not fail the run. A matrix (`df` a labeled square DataFrame) gets `renamed` (index and columns), `more` -and `fewer` (one label added or dropped on both axes) and `scaled`. Generated and -excluded specs have no table to perturb, and network specs are deferred: their block may -bind a `nodes` table beside the edge list, whose identifiers must follow the perturbed -endpoints, which comes with network eligibility (networks are not eligible for the agent in -phase 1). There is no nullable-dtype perturbation: the agent's loader hands over float64 -with NaN (decision of 2026-10-23). +and `fewer` (one label added or dropped on both axes) and `scaled`, and no missing-value +perturbation: the loader reads one flat `data.csv` into a table, never a labeled square +matrix, so a matrix never meets the loader's dtypes (and the agent serves tables only in +phase 1). Generated and excluded specs have no table to perturb, and network specs are +deferred: their block may bind a `nodes` table beside the edge list, whose identifiers +must follow the perturbed endpoints, which comes with network eligibility (networks are +not eligible for the agent in phase 1). `smoke_source(source, perturbation, data_path)` is the source to run: the block replaced by a `pandas.read_pickle` of the table and every column constant renamed through @@ -51,6 +63,7 @@ import numpy as np import pandas as pd +from pandas.api.extensions import ExtensionDtype from core.adaptability import replace_block, rewrite_columns from core.spec_roles import DataRole, Shape @@ -63,6 +76,9 @@ SHIFT = 100_000 NEW_LABEL = "Added" _SINGLE_GROUP_WORDS = ("binary", "two ", "pair", "split", "both", "left/right", "versus", " vs") +LOADER_NUMBER = "float64" +LOADER_NULLABLE = ("Int64", "boolean", "string") +"""The dtypes of `agents/anyplot/code/loader.py` `PANDAS_DTYPES`: number, then integer, boolean and text.""" @dataclass(frozen=True) @@ -76,6 +92,8 @@ class Perturbation: retired: frozenset[str] = field(default_factory=frozenset) renames: dict[str, str] = field(default_factory=dict) """Old column name -> new one, for every renamed column (spec roles and extras alike).""" + advisory: bool = False + """A failure on this table is reported but does not fail the smoke run (`missing_nullable`).""" def fixed_group_count(roles: Sequence[DataRole], color_role: str | None) -> bool: @@ -134,9 +152,95 @@ def perturbations( out.append(_fewer_rows(df, role_columns, color_column)) out.append(_more_rows(df, role_columns, categorical, color_column)) out.append(_scaled(df, role_columns)) + out.extend(_missing_values(df, role_columns)) return out +def loader_dtypes(df: pd.DataFrame) -> pd.DataFrame: + """The table in the dtypes the agent's loader reads `data.csv` into (`agents/anyplot/code/loader.py`). + + `PANDAS_DTYPES` maps text to `string`, integers to `Int64`, floats to `float64` and booleans to + `boolean`; dates are parsed and stay datetimes. A categorical column is text. An object column + takes the dtype its values would be read as: the loader profiles values, not the sample's dtype, + so an object column of ints reads as `Int64` and one of `datetime.date` objects stays as it is. + Any other dtype (timedelta, period, complex) stays as it is. + """ + out = df.copy() + for position in range(df.shape[1]): + series = df.iloc[:, position] + target = _loader_dtype(series) + if target is not None: + out.isetitem(position, series.astype(target).array) + return out + + +def _loader_dtype(series: pd.Series) -> ExtensionDtype | np.dtype | None: + text, integer, boolean, number = pd.StringDtype(), pd.Int64Dtype(), pd.BooleanDtype(), np.dtype(LOADER_NUMBER) + if pd.api.types.is_bool_dtype(series): + return boolean + if pd.api.types.is_integer_dtype(series): + return integer + if pd.api.types.is_float_dtype(series): + return number + if pd.api.types.is_datetime64_any_dtype(series): + return None + if isinstance(series.dtype, pd.CategoricalDtype) or pd.api.types.is_string_dtype(series): + return text + if pd.api.types.is_object_dtype(series): + inferred = pd.api.types.infer_dtype(series, skipna=True) + if inferred == "boolean": + return boolean + if inferred == "integer": + return integer + if inferred in ("floating", "mixed-integer-float", "decimal"): + return number + if inferred in ("date", "datetime", "datetime64", "time", "timedelta", "period"): + return None + return text + return None + + +def _missing_values(df: pd.DataFrame, columns: Mapping[str, str | list[str]]) -> list[Perturbation]: + """`missing_number` (binding) and `missing_nullable` (advisory), each left out without a column of its kind.""" + if df.empty: + return [] + cast = loader_dtypes(df) + numbers = [position for position in range(cast.shape[1]) if str(cast.dtypes.iloc[position]) == LOADER_NUMBER] + nullable = [position for position in range(cast.shape[1]) if str(cast.dtypes.iloc[position]) in LOADER_NULLABLE] + out: list[Perturbation] = [] + if numbers: + out.append( + Perturbation( + "missing_number", + "loader dtypes, NaN in every float64 column", + _with_missing(cast, numbers), + dict(columns), + ) + ) + if nullable: + out.append( + Perturbation( + "missing_nullable", + "loader dtypes, in every Int64, boolean and string column", + _with_missing(cast, nullable), + dict(columns), + advisory=True, + ) + ) + return out + + +def _with_missing(cast: pd.DataFrame, positions: Sequence[int]) -> pd.DataFrame: + """The i-th column of `positions` gets its dtype's missing value (`NaN` for float64, `` for the nullable + dtypes) in row `(len // 2 + i) % len`, so no row is entirely missing.""" + table = cast.copy() + rows = len(table) + for i, position in enumerate(positions): + hole = np.arange(rows) == (rows // 2 + i) % rows + table.isetitem(position, table.iloc[:, position].mask(hole).array) + return table + + def _is_categorical(series: pd.Series) -> bool: return bool( isinstance(series.dtype, pd.CategoricalDtype) diff --git a/core/smoke_run.py b/core/smoke_run.py new file mode 100644 index 00000000000..1495266aac9 --- /dev/null +++ b/core/smoke_run.py @@ -0,0 +1,1030 @@ +"""The perturbation smoke run: does a Python implementation still draw on other tables? + +The static checker (`core/adaptability.py`) tests proxies; this module runs the file +(docs/concepts/adaptable-code.md, "Perturbation smoke run" and "The adapt route"). It +composes the pure parts of PR 1: `core.adaptability` finds the `# Data` block and the +column constants, `core.perturbation` builds the perturbed tables and the source to run, +`core.canvas` measures a render and `core.render_identity` compares two. What it adds is +the execution: a sample probe, one process per table and theme, the stale-label hook, the +mapping of an error back to the original source and the JSON record. + +Three commands (exit codes are for people; the workflows record the JSON and never gate): + +* `run SOURCE`: the smoke run of one implementation. The sample probe runs the head and + the block once (`ANYPLOT_THEME=light`) and pickles `df` with the column each role's + constant names. Every perturbed table then replaces the block, the column constants are + renamed through their role comments, and the whole file runs once per theme in a fresh + directory, exactly as the generator's Step 3 runs it: ` -P {library}.py` with + `ANYPLOT_THEME`, `MPLBACKEND=Agg`, `PYTHONHASHSEED=0` and `PYTHONDONTWRITEBYTECODE=1`, + in a process group that a timeout kills whole (bokeh and kaleido start Chrome). An + execution passes when it exits 0 and writes `plot-{theme}.png` on the canvas; for + matplotlib and seaborn the `renamed` table must also leave no drawn text that shows a + retired sample category. The run passes when every binding table passes in every theme; + an advisory table (`missing_nullable`) is reported, never failed. Exit 0 on pass or + skipped, 1 on fail, 2 when the run cannot happen (`why: runner_error`). +* `pair --new NEW --prev PREV --out-dir DIR`: both sources rendered unchanged in one + environment, the new one first (its processes have exited before the predecessor's PNGs + exist), the raw PNGs copied to `DIR/new/` and `DIR/prev/`, and the render identity of + each theme. Exit 0 identical, 1 different, 2 error. +* `selfcheck SOURCE`: the generator's Step 3c, the static check and `run` in one command, + with one line per attempt appended to a JSON-lines log. Exit 0 when the static check + passes (or the spec is excluded) and the smoke run passes or is skipped for its shape + (`shape:`) or because the sample gives nothing to perturb + (`no_perturbations:`); neither skip is something the model can fix. + +The library's setup calls in the head (`core.adaptability.SETUP_CALLS`) run unchanged in +the probe and in every execution; nothing is stripped. Only the 8 Python libraries +(`core.adaptability.LIBRARIES`) run; the others are `skipped` with `why: not_python`. + +Mechanics decided here (PR 2 contract, section 10): executions run in parallel threads +(4, or 2 for the Chrome-driven bokeh and plotly), each in its own directory under one +temporary root; stdout and stderr go to files, so a straggler such as a Chrome that holds +the pipes cannot stall the run, and the process group is killed after every execution. +The stale-label hook wraps `matplotlib.text.Text.draw` through a runpy shim, so it sees +the text that was drawn, not every `Text` object a figure holds (an undrawn tick label or +a hidden axis); it skips the mandated title (any text with "anyplot.ai") and retired +values that read as prose, numbers or single characters, as the static checker does. +The executions do not inherit `PYTHONPATH`, so a file runs as Step 3 runs it even when +the runner itself comes from a workflow-ref copy of `core/`, and they do not inherit the +Actions runtime either (`GITHUB_*`, `ACTIONS_*`, `RUNNER_*`, `GH_*`, tokens included): the +code they run is model-written (`execution_env`). + +Usage:: + + python -m core.smoke_run run plots/bar-grouped/implementations/python/matplotlib.py --out smoke.json + python -m core.smoke_run pair --new NEW.py --prev PREV/matplotlib.py --out-dir /tmp/pair --out pair.json + python -m core.smoke_run selfcheck plots/bar-grouped/implementations/python/matplotlib.py +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pickle +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from datetime import UTC, datetime +from difflib import SequenceMatcher +from pathlib import Path + +import pandas as pd + +from core.adaptability import ( + CHECKER_VERSION, + LIBRARIES, + MIN_SAMPLE_CHARS, + PROSE_WORDS, + RULE_CLASSES, + Record, + check_file, + check_source, + library_of, + spec_context, +) +from core.canvas import TOLERANCE, NotAPngError, check_png, png_size +from core.constants import LIBRARY_LANGUAGES +from core.perturbation import Perturbation, perturbations, smoke_source +from core.render_identity import compare + + +RECORD_VERSION = 1 +THEMES = ("light", "dark") +PROBE_THEME = "light" +DEFAULT_TIMEOUT = 120 +DEFAULT_WORKERS = 4 +LIBRARY_WORKERS = {"bokeh": 2, "plotly": 2} +"""Libraries whose every execution starts a headless Chrome get fewer parallel executions.""" +STALE_LABEL_LIBRARIES = frozenset({"matplotlib", "seaborn"}) +SKIPPED_SHAPES = ("generated", "network", "excluded") +SELFCHECK_SKIP_PREFIXES = ("shape:", "no_perturbations:") +"""A skipped smoke run with one of these reasons is no failure of the self-check: nothing the model can fix. + +`shape:` skips a shape without tables, `no_perturbations:` a sample with nothing +to perturb. `automation/scripts/adapt_record.py` keeps a copy (a unit test keeps the two equal). +""" +TITLE_MARK = "anyplot.ai" +DEFAULT_SELFCHECK_LOG = Path("/tmp/anyplot-selfcheck.jsonl") +DEFAULT_SELFCHECK_STATIC = Path("/tmp/anyplot-selfcheck-static.json") +DEFAULT_SELFCHECK_SMOKE = Path("/tmp/anyplot-selfcheck-smoke.json") + +SAMPLE_FILE = "sample.pkl" +DATA_FILE = "data.pkl" +TEXTS_FILE = "texts.json" +HOOK_FILE = "_anyplot_text_hook.py" +STDOUT_FILE = "stdout.txt" +STDERR_FILE = "stderr.txt" +_MAX_ERROR_CHARS = 300 +_MAX_CODE_CHARS = 120 +_TRACEBACK = "Traceback (most recent call last):" +_IGNORED = "Exception ignored" +_FRAME = re.compile(r'^\s*File "(?P.+)", line (?P\d+)') +_NUMBER = re.compile(r"^[-+−]?\d[\d,._]*%?$") + +TEXT_HOOK = '''"""Run an implementation and record every matplotlib text it draws (core/smoke_run.py, stale labels).""" + +import atexit +import functools +import json +import runpy +import sys + + +def _main(): + target, out = sys.argv[1], sys.argv[2] + drawn = [] + + def _write(): + with open(out, "w", encoding="utf-8") as handle: + json.dump(drawn, handle) + + atexit.register(_write) + try: + from matplotlib.text import Text + except ImportError: + Text = None + if Text is not None: + original = Text.draw + + @functools.wraps(original) + def draw(self, renderer, *args, **kwargs): + text = self.get_text() + if self.get_visible() and text: + drawn.append(str(text)) + return original(self, renderer, *args, **kwargs) + + Text.draw = draw + sys.argv = [target] + runpy.run_path(target, run_name="__main__") + + +_main() +''' + + +def code_sha256(text: str) -> str: + """SHA-256 of a source without its leading docstring header (the workflow rewrites `Quality:` and `Updated:`). + + impl-review's record step reimplements this byte for byte; a parity test pins the two. + """ + body = text.lstrip() + if body.startswith('"""'): + end = body.find('"""', 3) + if end != -1: + body = body[end + 3 :] + return hashlib.sha256(body.strip().encode("utf-8")).hexdigest() + + +def language_of(library: str | None) -> str | None: + """The language of a catalogue library (`core.constants.LIBRARY_LANGUAGES`), None for an unknown one.""" + return LIBRARY_LANGUAGES.get(library) if library else None + + +EXECUTION_ENV_DROPPED_PREFIXES = ("GITHUB_", "ACTIONS_", "RUNNER_", "GH_") +"""Variable-name prefixes a model-written execution never sees: the runner's tokens, URLs and paths.""" + + +def execution_env(theme: str) -> dict[str, str]: + """The environment of one execution: the runner's, minus what a plot never needs, plus the theme settings. + + The implementation is model-written code, so it does not get the Actions runtime: every + variable whose name starts with `GITHUB_`, `ACTIONS_`, `RUNNER_` or `GH_` (tokens such as + `GITHUB_TOKEN`, `GH_TOKEN` and `ACTIONS_RUNTIME_TOKEN`, the cache and results URLs, the + runner's paths) is dropped, and so is `PYTHONPATH`, so a file runs as Step 3 runs it even + when the runner comes from a workflow-ref copy of `core/`. Everything else stays: `PATH`, + `HOME`, `DISPLAY`, the locale and the libraries' own settings. + """ + env = { + key: value + for key, value in os.environ.items() + if key != "PYTHONPATH" and not key.upper().startswith(EXECUTION_ENV_DROPPED_PREFIXES) + } + env.update(ANYPLOT_THEME=theme, MPLBACKEND="Agg", PYTHONHASHSEED="0", PYTHONDONTWRITEBYTECODE="1") + return env + + +# --- one execution ----------------------------------------------------------------------------- + + +@dataclass +class Execution: + """One process: its exit code (None on a timeout), its stderr and its wall time.""" + + returncode: int | None + stderr: str + seconds: float + + @property + def timed_out(self) -> bool: + return self.returncode is None + + +def _kill_group(pid: int) -> None: + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError, PermissionError: + pass + + +def execute(argv: Sequence[str], cwd: Path, env: Mapping[str, str], timeout: float) -> Execution: + """Run `argv` in `cwd` in a new process group, killed whole on a timeout and after the run. + + stdout and stderr go to files in `cwd`: a pipe would let a straggler that inherited it (a + Chrome the script did not stop) hold the run until the timeout. + """ + start = time.monotonic() + with (cwd / STDOUT_FILE).open("wb") as out, (cwd / STDERR_FILE).open("wb") as err: + process = subprocess.Popen( + list(argv), cwd=cwd, env=dict(env), stdin=subprocess.DEVNULL, stdout=out, stderr=err, start_new_session=True + ) + try: + returncode: int | None = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + returncode = None + _kill_group(process.pid) + process.wait() + finally: + _kill_group(process.pid) + stderr = (cwd / STDERR_FILE).read_text(encoding="utf-8", errors="replace") + return Execution(returncode, stderr, time.monotonic() - start) + + +def _same_file(name: str, script: Path) -> bool: + return os.path.realpath(os.path.join(script.parent, name)) == os.path.realpath(script) + + +def traceback_error(stderr: str, script: Path) -> tuple[str | None, int | None]: + """The exception line of the last traceback in `stderr` and the innermost line it names in `script`. + + A syntax error in the script prints its frame without a "Traceback" header, so the search runs + over the last traceback when there is one and over all of stderr otherwise. A traceback that + follows "Exception ignored in ..." (a destructor at shutdown, say selenium's) is not the + error. Without a frame line, the error is stderr's last non-empty line. + """ + lines = stderr.splitlines() + starts = [index for index, text in enumerate(lines) if text.startswith(_TRACEBACK)] + raised = [index for index in starts if index == 0 or not lines[index - 1].startswith(_IGNORED)] + starts = raised or starts + block = lines[starts[-1] + 1 :] if starts else lines + ignored = next((index for index, text in enumerate(block) if text.startswith(_IGNORED)), None) + if ignored is not None and ignored > 0: + block = block[:ignored] + line: int | None = None + last_frame = -1 + for index, text in enumerate(block): + match = _FRAME.match(text) + if match: + last_frame = index + if _same_file(match["file"], script): + line = int(match["line"]) + message = None + if last_frame >= 0: + message = next((text.strip() for text in block[last_frame + 1 :] if text and not text[0].isspace()), None) + if message is None: + tail = [text.strip() for text in lines if text.strip()] + message = tail[-1] if tail else None + if message is not None and len(message) > _MAX_ERROR_CHARS: + message = message[: _MAX_ERROR_CHARS - 1] + "…" + return message, line + + +def line_map(original: str, executed: str) -> dict[int, int]: + """Each 1-based line of `executed` mapped to a line of `original`. + + The executed source is the original with its block replaced and its column constants + rewritten (a constant spread over several lines becomes one), so the lines below the + block shift by the difference in block length. A line that has no counterpart maps to + the first original line of the region it replaced. + """ + executed_lines, original_lines = executed.splitlines(), original.splitlines() + mapping: dict[int, int] = {} + if not original_lines: + return mapping + matcher = SequenceMatcher(None, executed_lines, original_lines, autojunk=False) + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + for i in range(i1, i2): + if tag == "equal": + j = j1 + (i - i1) + elif j2 > j1: + j = j1 + min(i - i1, j2 - j1 - 1) + else: + j = min(j1, len(original_lines) - 1) + mapping[i + 1] = j + 1 + return mapping + + +def source_line(source: str, line: int | None) -> str | None: + """The stripped text of a 1-based line of `source`, cut to a readable length.""" + if line is None: + return None + lines = source.splitlines() + if not 1 <= line <= len(lines): + return None + text = lines[line - 1].strip() + return text if len(text) <= _MAX_CODE_CHARS else text[: _MAX_CODE_CHARS - 1] + "…" + + +def _checkable(value: str) -> bool: + """A retired category a drawn text can be tied to: not prose ("Total"), not a number, not one character.""" + text = value.strip() + return len(text) >= MIN_SAMPLE_CHARS and text.lower() not in PROSE_WORDS and not _NUMBER.match(text) + + +def stale_labels(texts: Sequence[str], retired: frozenset[str]) -> list[tuple[str, str]]: + """`(text, retired category)` for each drawn text that shows a retired category as a whole word.""" + patterns = [ + (value, re.compile(rf"(? list[str]: + return [str(r["name"]) for r in self.results if r["status"] == "fail" and not r["advisory"]] + + @property + def advisory_failed(self) -> list[str]: + return [str(r["name"]) for r in self.results if r["status"] == "fail" and r["advisory"]] + + def exit_code(self) -> int: + if self.why == "runner_error": + return 2 + return 1 if self.status == "fail" else 0 + + def to_dict(self) -> dict[str, object]: + record = self.record + return { + "v": RECORD_VERSION, + "tool": "core.smoke_run run", + "checker_version": CHECKER_VERSION, + "source": str(self.source_path), + "code_sha256": code_sha256(self.source) if self.source is not None else None, + "spec": self.spec_id, + "library": self.library, + "language": language_of(self.library), + "shape": record.shape if record else None, + "color_role": record.color_role if record else None, + "static_status": record.status if record else None, + "themes": list(self.themes), + "status": self.status, + "why": self.why, + "error": self.error, + "sample": self.sample, + "results": self.results, + "failed": self.failed, + "advisory_failed": self.advisory_failed, + "seconds": round(self.seconds, 1), + } + + def summary(self) -> list[str]: + """One summary line and one line per perturbation.""" + head = f"smoke_run {self.spec_id or '?'} {self.library or '?'}: {self.status}" + if self.why != "-": + head += f" ({self.why})" + if self.results: + head += ( + f": {len(self.results)} perturbations, {len(self.failed)} failed, " + f"{len(self.advisory_failed)} advisory failed" + ) + elif self.error: + head += f": {self.error}" + elif self.sample and self.sample.get("error"): + line = self.sample.get("line") + head += f": sample {'line ' + str(line) + ': ' if line else ''}{self.sample['error']}" + out = [head] + for result in self.results: + if result["status"] == "pass": + label = "pass" + else: + label = "fail (advisory)" if result["advisory"] else "FAIL" + text = f" {result['name']!s:<18} {label}" + if result["status"] != "pass": + where = f"line {result['line']}: " if result["line"] is not None else "" + text += f" {where}{result['error']}" + if result["code"]: + text += f" | {result['code']}" + if result["theme"] and result["theme"] != self.themes[0]: + text += f" [{result['theme']}]" + out.append(text) + return out + + +def run( + source_path: Path, + *, + spec_path: Path | None = None, + spec_id: str | None = None, + library: str | None = None, + themes: Sequence[str] = THEMES, + timeout: float = DEFAULT_TIMEOUT, + workers: int | None = None, + python: str | None = None, + skip_if_static_fails: bool = False, + keep: Path | None = None, +) -> SmokeRun: + """The perturbation smoke run of one source (module docstring); never raises.""" + started = time.monotonic() + library = ( + library or library_of(source_path) or (source_path.stem if source_path.stem in LIBRARY_LANGUAGES else None) + ) + smoke = SmokeRun(source_path, library, tuple(themes)) + try: + _run(smoke, spec_path, spec_id, timeout, workers, python, skip_if_static_fails, keep) + except Exception as exc: # the record is written whatever happens; the workflows read why + smoke.status, smoke.why, smoke.error = "skipped", "runner_error", f"{type(exc).__name__}: {exc}" + smoke.seconds = time.monotonic() - started + return smoke + + +def _run( + smoke: SmokeRun, + spec_path: Path | None, + spec_id: str | None, + timeout: float, + workers: int | None, + python: str | None, + skip_if_static_fails: bool, + keep: Path | None, +) -> None: + library, source_path = smoke.library, smoke.source_path + try: + smoke.source = source_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + smoke.error = f"cannot read the source: {exc}" + if library is not None and library not in LIBRARIES and library in LIBRARY_LANGUAGES: + smoke.why = "not_python" + return + if library is None: + smoke.error = f"cannot tell the library from {source_path.name}; pass --library" + return + if library not in LIBRARIES: + if library in LIBRARY_LANGUAGES: + smoke.why = "not_python" + else: + smoke.error = f"unknown library {library!r}" + return + if spec_path is None: + if len(source_path.parents) < 3: + smoke.error = "cannot find the spec next to the source; pass --spec" + return + spec_path = source_path.parents[2] / "specification.md" + try: + smoke.spec_id, roles, shape = spec_context(spec_path, spec_id) + except (OSError, UnicodeDecodeError) as exc: + smoke.error = f"cannot read the spec: {exc}" + return + interpreter = python or sys.executable + if not interpreter or shutil.which(interpreter) is None: + smoke.error = f"no Python interpreter at {interpreter!r}" + return + record = check_source(smoke.source, roles, shape, library=library) + smoke.record = record + if skip_if_static_fails and record.status == "fail": + smoke.why = "static_failed" + return + if shape in SKIPPED_SHAPES: + smoke.why = f"shape:{shape}" + return + if record.block is None: + smoke.why = "no_block" + return + + root = Path(tempfile.mkdtemp(prefix="anyplot-smoke-")) if keep is None else keep + root.mkdir(parents=True, exist_ok=True) + try: + sample = _probe(smoke.source, record, library, interpreter, root / "00-sample", timeout) + smoke.sample = sample.as_dict() + if sample.df is None: + smoke.status, smoke.why = "fail", "sample_failed" + return + items = perturbations(sample.df, sample.columns, roles, shape, record.color_role) + if not items: + # Not a shape skip: the shape has tables, this sample yields none to perturb. + smoke.why = f"no_perturbations:{shape}" + return + count = workers or LIBRARY_WORKERS.get(library, DEFAULT_WORKERS) + tasks = [_Task(index, item, theme) for index, item in enumerate(items, start=1) for theme in smoke.themes] + with ThreadPoolExecutor(max_workers=max(1, count)) as pool: + list( + pool.map( + lambda task: _execute_task(task, smoke.source or "", library, interpreter, root, timeout), tasks + ) + ) + smoke.results = [_result(item, [t for t in tasks if t.perturbation is item], smoke.source) for item in items] + smoke.status, smoke.why = ("fail", "perturbation_failed") if smoke.failed else ("pass", "-") + finally: + if keep is None: + shutil.rmtree(root, ignore_errors=True) + + +@dataclass +class _Sample: + df: pd.DataFrame | None + columns: dict[str, str | list[str]] + error: str | None = None + line: int | None = None + + def as_dict(self) -> dict[str, object]: + return { + "status": "pass" if self.df is not None else "fail", + "rows": len(self.df) if self.df is not None else None, + "columns": dict(self.columns), + "error": self.error, + "line": self.line, + } + + +def _probe_epilogue(constants: Mapping[str, str], path: Path) -> str: + """Pickle `df` and the column each role's constant names; exit with a message when no DataFrame `df` exists.""" + return ( + "\n\n# --- core.smoke_run sample probe ---\n" + "import pickle as _anyplot_pickle\n" + "import sys as _anyplot_sys\n\n" + "import pandas as _anyplot_pd\n\n" + '_anyplot_df = globals().get("df")\n' + "if _anyplot_df is None:\n" + ' _anyplot_sys.exit("smoke_run: the Data block binds no df")\n' + "if not isinstance(_anyplot_df, _anyplot_pd.DataFrame):\n" + ' _anyplot_sys.exit(f"smoke_run: df is a {type(_anyplot_df).__name__}, not a pandas DataFrame")\n' + f"_anyplot_names = {dict(constants)!r}\n" + "_anyplot_columns = {role: globals().get(name) for role, name in _anyplot_names.items()}\n" + f"with open({str(path)!r}, 'wb') as _anyplot_file:\n" + ' _anyplot_pickle.dump({"df": _anyplot_df, "columns": _anyplot_columns}, _anyplot_file)\n' + ) + + +def _probe(source: str, record: Record, library: str, python: str, directory: Path, timeout: float) -> _Sample: + """Run the head, the constants and the block (the lines above `# Plot`) and read `df` back.""" + assert record.block is not None + plot_line = record.block[1] + directory.mkdir(parents=True, exist_ok=True) + head = "".join(source.splitlines(keepends=True)[: plot_line - 1]) + script = directory / f"{library}.py" + script.write_text(head + _probe_epilogue(record.constants, directory / SAMPLE_FILE), encoding="utf-8") + execution = execute([python, "-P", script.name], directory, execution_env(PROBE_THEME), timeout) + if execution.timed_out: + return _Sample(None, {}, f"timed out after {timeout:g} s") + if execution.returncode != 0 or not (directory / SAMPLE_FILE).is_file(): + error, line = traceback_error(execution.stderr, script) + return _Sample( + None, {}, error or f"exit code {execution.returncode}", line if line and line < plot_line else None + ) + with (directory / SAMPLE_FILE).open("rb") as handle: + payload = pickle.load(handle) # written by the probe of the file under test; the runner is no sandbox + df = payload["df"] + columns: dict[str, str | list[str]] = {} + for role, value in payload["columns"].items(): + if isinstance(value, str) and value in df.columns: + columns[role] = value + elif ( + isinstance(value, list | tuple) + and value + and all(isinstance(item, str) and item in df.columns for item in value) + ): + columns[role] = list(value) + return _Sample(df, columns) + + +def _execute_task(task: _Task, source: str, library: str, python: str, root: Path, timeout: float) -> None: + item, theme = task.perturbation, task.theme + directory = root / f"{task.index:02d}-{item.name}-{theme}" + directory.mkdir(parents=True, exist_ok=True) + data = directory / DATA_FILE + item.df.to_pickle(data) + code = smoke_source(source, item, str(data)) + script = directory / f"{library}.py" + script.write_text(code, encoding="utf-8") + hook = item.name == "renamed" and library in STALE_LABEL_LIBRARIES and bool(item.retired) + if hook: + (directory / HOOK_FILE).write_text(TEXT_HOOK, encoding="utf-8") + argv = [python, "-P", HOOK_FILE, str(script.resolve()), str((directory / TEXTS_FILE).resolve())] + else: + argv = [python, "-P", script.name] + execution = execute(argv, directory, execution_env(theme), timeout) + task.seconds = execution.seconds + if execution.timed_out: + task.error = f"timed out after {timeout:g} s" + return + if execution.returncode != 0: + error, line = traceback_error(execution.stderr, script) + task.error = error or f"exit code {execution.returncode}" + task.line = line_map(source, code).get(line) if line is not None else None + return + png = directory / f"plot-{theme}.png" + if not png.is_file(): + task.error = f"no plot-{theme}.png written" + return + try: + verdict = check_png(png, library) + except (NotAPngError, OSError) as exc: + task.error = f"plot-{theme}.png is not a readable PNG: {exc}" + return + task.canvas = f"{verdict.width}x{verdict.height}" + if not verdict.ok: + (tw, th) = verdict.target + task.error = f"canvas {verdict.width}x{verdict.height}, target {tw}x{th} (+-{TOLERANCE} px)" + return + if hook: + try: + texts = json.loads((directory / TEXTS_FILE).read_text(encoding="utf-8")) + except OSError, ValueError: + texts = [] + stale = stale_labels([str(text) for text in texts], item.retired) + if stale: + task.stale = [text for text, _ in stale] + text, value = stale[0] + task.error = f"stale label: {text!r} shows the retired category {value!r}" + return + task.passed = True + + +def _result(item: Perturbation, tasks: Sequence[_Task], source: str | None) -> dict[str, object]: + failing = next((task for task in tasks if not task.passed), None) + canvas = {task.theme: task.canvas for task in tasks if task.canvas is not None} + return { + "name": item.name, + "description": item.description, + "advisory": item.advisory, + "status": "fail" if failing else "pass", + "rows": len(item.df), + "theme": failing.theme if failing else None, + "error": failing.error if failing else None, + "line": failing.line if failing else None, + "code": source_line(source or "", failing.line) if failing else None, + "canvas": canvas or None, + "stale_labels": failing.stale if failing else [], + "seconds": round(sum(task.seconds for task in tasks), 1), + } + + +# --- pair: two renders in one environment ------------------------------------------------------ + + +@dataclass +class _Side: + """One source of a pair: its hash, one render per theme and the first error.""" + + code_sha256: str | None = None + renders: dict[str, dict[str, object]] = field(default_factory=dict) + error: str | None = None + line: int | None = None + + @property + def ok(self) -> bool: + return bool(self.renders) and all(render["status"] == "ok" for render in self.renders.values()) + + def as_dict(self) -> dict[str, object]: + return {"code_sha256": self.code_sha256, "renders": self.renders, "error": self.error, "line": self.line} + + +def _render_side( + path: Path, library: str, themes: Sequence[str], python: str, root: Path, out: Path, timeout: float +) -> _Side: + """Render `path` unchanged once per theme and copy each raw PNG to `out/plot-{theme}.png`.""" + side = _Side() + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + side.error = f"cannot read {path}: {exc}" + return side + side.code_sha256 = code_sha256(source) + out.mkdir(parents=True, exist_ok=True) + + def render(theme: str) -> tuple[str, dict[str, object], str | None, int | None]: + directory = root / theme + directory.mkdir(parents=True, exist_ok=True) + script = directory / f"{library}.py" + script.write_text(source, encoding="utf-8") + execution = execute([python, "-P", script.name], directory, execution_env(theme), timeout) + seconds = round(execution.seconds, 1) + png = directory / f"plot-{theme}.png" + error: str | None = None + line: int | None = None + size: list[int] | None = None + if execution.timed_out: + error = f"timed out after {timeout:g} s" + elif execution.returncode != 0: + error, line = traceback_error(execution.stderr, script) + error = error or f"exit code {execution.returncode}" + elif not png.is_file(): + error = f"no plot-{theme}.png written" + else: + try: + size = list(png_size(png)) + except (NotAPngError, OSError) as exc: + error = f"plot-{theme}.png is not a readable PNG: {exc}" + if error is None: + shutil.copyfile(png, out / f"plot-{theme}.png") + status = "ok" if error is None else "fail" + return theme, {"status": status, "size": size, "seconds": seconds}, error, line + + with ThreadPoolExecutor(max_workers=max(1, len(themes))) as pool: + rendered = list(pool.map(render, themes)) + for theme, render_record, error, line in rendered: + side.renders[theme] = render_record + if error is not None and side.error is None: + side.error, side.line = f"{theme}: {error}", line + return side + + +def pair( + new_path: Path, + prev_path: Path, + out_dir: Path, + *, + library: str | None = None, + themes: Sequence[str] = THEMES, + timeout: float = DEFAULT_TIMEOUT, + python: str | None = None, +) -> dict[str, object]: + """Render the new source, then the predecessor, in one environment and compare the raw renders per theme.""" + library = library or library_of(new_path) + themes = tuple(themes) + record: dict[str, object] = { + "v": RECORD_VERSION, + "tool": "core.smoke_run pair", + "library": library, + "themes": list(themes), + "new": None, + "prev": None, + "identity": None, + "status": "error", + "why": "new_render_failed", + } + interpreter = python or sys.executable + if library not in LIBRARIES: + record["new"] = _Side(error=f"{library!r} is not one of the Python libraries {', '.join(LIBRARIES)}").as_dict() + return record + if not interpreter or shutil.which(interpreter) is None: + record["new"] = _Side(error=f"no Python interpreter at {interpreter!r}").as_dict() + return record + for side in ("new", "prev"): + shutil.rmtree(out_dir / side, ignore_errors=True) + root = Path(tempfile.mkdtemp(prefix="anyplot-pair-")) + try: + new = _render_side(new_path, library, themes, interpreter, root / "new", out_dir / "new", timeout) + prev = _render_side(prev_path, library, themes, interpreter, root / "prev", out_dir / "prev", timeout) + finally: + shutil.rmtree(root, ignore_errors=True) + record["new"], record["prev"] = new.as_dict(), prev.as_dict() + if not new.ok: + return record + if not prev.ok: + record["why"] = "prev_render_failed" + return record + identity = { + theme: compare(out_dir / "prev" / f"plot-{theme}.png", out_dir / "new" / f"plot-{theme}.png") + for theme in themes + } + record["identity"] = {theme: result.to_dict() for theme, result in identity.items()} + failing = [theme for theme, result in identity.items() if not result.passed] + if any(not result.same_size for result in identity.values()): + why = "size_changed" + elif not failing: + why = "-" + elif len(failing) == 1: + why = f"identity_{failing[0]}" + else: + why = "identity_both" + record["status"], record["why"] = ("identical", "-") if why == "-" else ("different", why) + return record + + +def pair_exit_code(record: Mapping[str, object]) -> int: + return {"identical": 0, "different": 1}.get(str(record.get("status")), 2) + + +def pair_summary(record: Mapping[str, object]) -> str: + identity = record.get("identity") + fractions = "" + if isinstance(identity, dict): + fractions = " " + " ".join( + f"{theme}={value['fraction']:.4f}" if value.get("fraction") is not None else f"{theme}=size" + for theme, value in identity.items() + ) + head = f"smoke_run pair {record.get('library')}: {record.get('status')} ({record.get('why')}){fractions}" + errors = [ + f" {side}: {side_record['error']}" + for side in ("new", "prev") + if isinstance(side_record := record.get(side), dict) and side_record.get("error") + ] + return "\n".join([head, *errors]) + + +# --- selfcheck: the generator's Step 3c -------------------------------------------------------- + + +def selfcheck( + source_path: Path, + *, + log: Path = DEFAULT_SELFCHECK_LOG, + static_out: Path = DEFAULT_SELFCHECK_STATIC, + smoke_out: Path = DEFAULT_SELFCHECK_SMOKE, + runner: Callable[[Path], SmokeRun] | None = None, +) -> tuple[int, list[str]]: + """The static check and the smoke run of one catalogue file, logged as one attempt; returns (exit, lines).""" + if library_of(source_path) is None: + return 2, [f"::error::selfcheck cannot tell the library from {source_path.name}"] + try: + source = source_path.read_text(encoding="utf-8") + record = check_file(source_path) # what `python -m core.adaptability check SOURCE` records + except (OSError, UnicodeDecodeError, IndexError) as exc: + return 2, [f"::error::selfcheck cannot read its input: {exc}"] + static_out.parent.mkdir(parents=True, exist_ok=True) + smoke_out.parent.mkdir(parents=True, exist_ok=True) + static_out.write_text(json.dumps(record.to_dict(), ensure_ascii=False) + "\n", encoding="utf-8") + smoke = (runner or run)(source_path) + smoke_out.write_text(json.dumps(smoke.to_dict(), ensure_ascii=False) + "\n", encoding="utf-8") + + classes = record.classes() + binding = [cls for cls in classes if not RULE_CLASSES[cls].advisory] + advisory = [cls for cls in classes if RULE_CLASSES[cls].advisory] + attempt = _attempts(log) + 1 + entry = { + "at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + "attempt": attempt, + "code_sha256": code_sha256(source), + "static": record.status, + "classes": binding, + "advisory": advisory, + "smoke": smoke.status, + # impl-review's adapt record reads it: only a skip for the shape counts as a passing attempt. + "smoke_why": smoke.why, + "failed": smoke.failed, + "advisory_failed": smoke.advisory_failed, + } + log.parent.mkdir(parents=True, exist_ok=True) + with log.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry, ensure_ascii=False) + "\n") + + static_ok = record.status in ("pass", "excluded") + smoke_ok = smoke.status == "pass" or (smoke.status == "skipped" and smoke.why.startswith(SELFCHECK_SKIP_PREFIXES)) + if smoke.why == "runner_error": + code = 2 + else: + code = 0 if static_ok and smoke_ok else 1 + head = f"static: {record.status}" + (f" ({', '.join(binding)})" if binding else "") + if advisory: + head += f"; advisory: {', '.join(advisory)}" + lines = [head, *record.defect_lines(), *smoke.summary()] + lines.append(f"selfcheck attempt {attempt}: {'pass' if code == 0 else 'fail'}") + return code, lines + + +def _attempts(log: Path) -> int: + try: + return sum(1 for line in log.read_text(encoding="utf-8").splitlines() if line.strip()) + except OSError, UnicodeDecodeError: + return 0 + + +# --- CLI --------------------------------------------------------------------------------------- + + +def _themes(text: str) -> tuple[str, ...]: + themes = tuple(dict.fromkeys(part.strip() for part in text.split(",") if part.strip())) + unknown = [theme for theme in themes if theme not in THEMES] + if not themes or unknown: + raise argparse.ArgumentTypeError(f"themes are a comma-separated subset of {','.join(THEMES)}") + return themes + + +def _write_json(path: Path | None, payload: Mapping[str, object]) -> None: + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m core.smoke_run", description="Run a Python implementation on perturbed tables." + ) + sub = parser.add_subparsers(dest="command", required=True) + + run_parser = sub.add_parser("run", help="the perturbation smoke run of one source") + run_parser.add_argument("source", type=Path) + run_parser.add_argument("--spec", type=Path, default=None, help="default: SOURCE.parents[2]/specification.md") + run_parser.add_argument("--spec-id", default=None, help="default: the spec file's directory name") + run_parser.add_argument("--library", default=None, help="default: SOURCE's stem") + run_parser.add_argument("--out", type=Path, default=None, help="write the JSON record here") + run_parser.add_argument("--themes", type=_themes, default=THEMES, help="default: light,dark") + run_parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT, help="seconds per execution") + run_parser.add_argument( + "--workers", type=int, default=None, help="parallel executions (default 4; 2 for bokeh/plotly)" + ) + run_parser.add_argument("--python", default=None, help="interpreter for the executions (default: this one)") + run_parser.add_argument("--skip-if-static-fails", action="store_true", help="skip when the static check fails") + run_parser.add_argument("--keep", type=Path, default=None, help="keep each execution's directory here") + + pair_parser = sub.add_parser("pair", help="render two sources in one environment and compare them") + pair_parser.add_argument("--new", type=Path, required=True, dest="new_source") + pair_parser.add_argument("--prev", type=Path, required=True, dest="prev_source") + pair_parser.add_argument("--out-dir", type=Path, required=True) + pair_parser.add_argument("--library", default=None) + pair_parser.add_argument("--out", type=Path, default=None) + pair_parser.add_argument("--themes", type=_themes, default=THEMES) + pair_parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT) + pair_parser.add_argument("--python", default=None) + + check_parser = sub.add_parser("selfcheck", help="the generator's Step 3c: static check plus smoke run") + check_parser.add_argument("source", type=Path) + check_parser.add_argument("--log", type=Path, default=DEFAULT_SELFCHECK_LOG) + check_parser.add_argument("--static-out", type=Path, default=DEFAULT_SELFCHECK_STATIC) + check_parser.add_argument("--smoke-out", type=Path, default=DEFAULT_SELFCHECK_SMOKE) + + args = parser.parse_args(argv) + if args.command == "run": + smoke = run( + args.source, + spec_path=args.spec, + spec_id=args.spec_id, + library=args.library, + themes=args.themes, + timeout=args.timeout, + workers=args.workers, + python=args.python, + skip_if_static_fails=args.skip_if_static_fails, + keep=args.keep, + ) + _write_json(args.out, smoke.to_dict()) + print("\n".join(smoke.summary())) + return smoke.exit_code() + if args.command == "pair": + try: + record = pair( + args.new_source, + args.prev_source, + args.out_dir, + library=args.library, + themes=args.themes, + timeout=args.timeout, + python=args.python, + ) + except Exception as exc: # the record is written whatever happens + record = {"v": RECORD_VERSION, "tool": "core.smoke_run pair", "status": "error", "why": "new_render_failed"} + record["error"] = f"{type(exc).__name__}: {exc}" + _write_json(args.out, record) + print(pair_summary(record)) + return pair_exit_code(record) + code, lines = selfcheck(args.source, log=args.log, static_out=args.static_out, smoke_out=args.smoke_out) + print("\n".join(lines)) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/concepts/adaptable-code.md b/docs/concepts/adaptable-code.md index de6b1fea8eb..5fe3e13fa35 100644 --- a/docs/concepts/adaptable-code.md +++ b/docs/concepts/adaptable-code.md @@ -1,6 +1,6 @@ # Adaptable catalogue code -> **Status (2026-10-23):** PR 1 (the checker, [#12244](https://github.com/MarkusNeusinger/anyplot/pull/12244)) is in progress; the rollout was revised on 2026-10-23 after a second opinion. Nothing is in the pipeline yet. +> **Status (2026-10-23):** PR 1 (the checker) merged as [#12244](https://github.com/MarkusNeusinger/anyplot/pull/12244); PR 2 (the rules) is in progress. The rollout was revised on 2026-10-23 after a second opinion. PR 2 departs from this document in four places: impl-generate measures render identity on the raw renders before the staged PNGs are quantized, and the gate reads the result as JSON instead of importing PIL; impl-review pins its Python to the repository's `.python-version` (3.14) for every step, the checker mirror included, because an older interpreter can't parse the `except A, B:` that ruff now writes and the runner image's own Python is not pinned; the missing-value perturbations are `missing_number` (binding) and `missing_nullable` (advisory until the nullable-integer question is settled); and the review-free path is the repository variable `ADAPT_REVIEW_FREE`, off by default and read against the route-held PRs: those labelled `regen:adapt` (the route held through the review) plus those labelled `regen:adapt-overruled` (the route held before the review, and the review turned it off). It is turned on only when the overruled share stays near zero over at least 50 route-held PRs. This document describes how catalogue implementations become code that you can point at other data: one replaceable data block, columns named once, and plot code that derives everything else from the table. It covers the convention, the checks that enforce it, the pipeline changes, the rollout through a targeted conversion pass and the daily regeneration, and how the planned "Use with my data" agent ([Agent network design](agent-network.md)) uses the result. @@ -75,7 +75,7 @@ The second opinion of 2026-10-23 found three things the 2026-10-09 design got wr A Python implementation has six zones, top to bottom: -1. **Head.** The docstring header, imports, theme tokens, and the full 8-entry `IMPRINT` in canonical order (`core/palette.py`, l.64). Library setup such as `LetsPlot.setup_html()` or `sns.set_theme(...)` goes here, never into the data block. +1. **Head.** The docstring header, imports, theme tokens, and the full 8-entry `IMPRINT` in canonical order (`core/palette.py`, l.64). The prompts name the constant `IMPRINT_PALETTE`, as the style guide and the library prompts do; the checker accepts both names. Library setup such as `LetsPlot.setup_html()` or `sns.set_theme(...)` goes here, never into the data block. 2. **`# Constants` (optional).** Values of the plot type or the domain that stay true for any data: thresholds (RSI 30 and 70, a `TARGET` the spec names), ordered scales (weekday names, Likert levels), and semantic color maps looked up with a palette fallback (`SEMANTIC.get(level, IMPRINT[i])`). Never sample categories, sample values, or data coordinates. 3. **Column constants.** One constant per spec `## Data` role the code uses: a domain name, the column name as value (written as the axis title should read, units included), and a trailing `# role: ` comment. A variadic family (`y1, y2, ...`) gets one constant holding a list. 4. **Exactly one `# Data` block.** Everything that invents the example: the seed, random draws, literal lists, category names, the formulas behind synthetic data, a `sns.load_dataset` call. It ends in one tidy DataFrame `df` (one row per mark or observation, one column per role, keyed by the column constants), binds nothing else that later code reads, and contains no plotting or styling call. Synthetic data, `np.random.seed(42)`, and "No external files" (`prompts/plot-generator.md`, l.283) stay; a CSV read appears only in the block's comment. @@ -331,7 +331,7 @@ The gate computes the route from the two records and the two renders, never from - the new source has at most `max(1.25 × prev, prev + 15)` lines; - the review is a sanity floor, when it runs: a score above 0, no canvas failure, and no technical item (VQ, SC, DQ, CQ) below its value in the re-score. There is no score tolerance: the reviewer's run-to-run spread of about 2 points and the re-score offset of about 10 points make any tolerance a coin flip in both directions. -A conversion outside the identity bound takes no shortcut: it is an ordinary regeneration and merges only with a carrier under today's rules. Two gate changes carry the route. With a review, `_judge` (l.1368) accepts the adapt route at its two no-carrier exits (l.1440 and l.1446) as it accepts the code path, and the `below_tolerance` check that follows them (l.1463, `score < prev_rescored - 1`) does not apply to the route: its floor is the technical items, so a lower DE or LM score on an identical render is not a reason to keep; the CQ-04 path stays as it is, tolerance included. Without a review, `_decide` (l.1033) needs an exit of its own, because it keeps on a missing score and on a missing `review_regen.json` before `_judge` runs: when the adapt record reports the route (predecessor fails, new source passes, identity inside the bound in both themes, line bound kept) and `review_skipped` is set, `_decide` returns merge with the code `adapt_identity` before the score checks, so an unreviewed conversion inside the bound follows a deterministic path and is never kept for the lack of a score. The review-dependent conditions (`scenario_changed`, `encodings_added`, the sanity floor) apply only when a review ran. The **adapt regression** applies on every spec: a predecessor that passes and a new source that fails means keep, so converted pairs stay converted. A missing or malformed record, or a missing predecessor render, turns the route off and records why; it never counts as a pass. +A conversion outside the identity bound takes no shortcut: it is an ordinary regeneration and merges only with a carrier under today's rules. Two gate changes carry the route. With a review, `_judge` (l.1368) accepts the adapt route at its two no-carrier exits (l.1440 and l.1446) as it accepts the code path, and the `below_tolerance` check that follows them (l.1463, `score < prev_rescored - 1`) does not apply to the route: its floor is the technical items, so a lower DE or LM score on an identical render is not a reason to keep; the CQ-04 path stays as it is, tolerance included. Without a review, `_decide` (l.1033) needs an exit of its own, because it keeps on a missing score and on a missing `review_regen.json` before `_judge` runs: when the adapt record reports the route (predecessor fails, new source passes, identity inside the bound in both themes, line bound kept) and `review_skipped` is set, `_decide` returns merge with the code `adapt_identity` before the score checks, so an unreviewed conversion inside the bound follows a deterministic path and is never kept for the lack of a score. The review-dependent conditions (`scenario_changed`, `encodings_added`, the sanity floor) apply only when a review ran. The **adapt regression** applies on every spec: a predecessor that passes and a new source that fails means keep, so converted pairs stay converted. A predecessor that passes the static check and whose smoke run is not known, against a new source that fails the static check, is a regression too: the static check alone shows it. A missing or malformed record, or a missing predecessor render, turns the route off and records why; it never counts as a pass. **The review on identical conversions.** For the first 50 conversions inside the bound the Opus review runs as usual, and the weekly review reads its scores against the re-scores: that is the measurement of whether identity plus checker plus smoke is enough. After 50, an identical conversion merges on the route without a review, which removes the expensive call from the pass. That needs a metadata handoff in PR 2: impl-generate resets `quality_score` to null and empties the stored review (`impl-generate.yml`, l.750), and impl-review writes the metadata and the implementation header only after a review with a score (l.1233), which impl-merge relies on. For a review-free merge the record step in impl-review carries the predecessor's stored score and review forward unchanged (the render is identical), marks them as carried in the adapt record, and writes the header from the measured versions as usual, so impl-merge's metadata update has its inputs and the weekly review and the database can tell a carried score from a reviewed one. A conversion outside the bound is always reviewed. diff --git a/docs/reference/repository.md b/docs/reference/repository.md index 835014cde26..606f42695ac 100644 --- a/docs/reference/repository.md +++ b/docs/reference/repository.md @@ -63,11 +63,12 @@ anyplot/ │ ├── prompts/ # AI agent prompts │ ├── plot-generator.md # Base rules for code generation +│ ├── adaptable-code.md # The Data block convention every implementation follows │ ├── quality-criteria.md # Quality evaluation criteria │ ├── quality-evaluator.md # AI quality evaluation prompt │ ├── spec-id-generator.md # Assigns spec IDs │ ├── default-style-guide.md # Default visual style rules -│ ├── library/ # Library-specific rules (11 files: 9 Python + ggplot2 + makie) +│ ├── library/ # Library-specific rules (15 files: 8 Python + ggplot2 + makie + 5 JavaScript) │ │ ├── matplotlib.md │ │ ├── seaborn.md │ │ └── ... @@ -88,6 +89,7 @@ anyplot/ │ ├── images.py # Image processing utilities │ ├── perturbation.py # Perturbed tables for the smoke run │ ├── render_identity.py # Pixel difference between two renders +│ ├── smoke_run.py # Perturbation smoke run, render pair, generator self-check │ ├── spec_roles.py # Data roles, data shape and color role of a spec │ ├── utils.py # General utilities │ ├── database/ # Database layer @@ -485,12 +487,13 @@ plt.savefig('plot.png', dpi=300) **Purpose**: AI agent prompts for code generation and quality evaluation **Subdirectories**: -- `library/` - Library-specific rules (11 files: 9 Python + ggplot2 + makie) +- `library/` - Library-specific rules (15 files: 8 Python + ggplot2 + makie + 5 JavaScript) - `templates/` - Templates for new specs (`specification.md`, `specification.yaml`) - `workflow-prompts/` - Workflow-specific prompt templates **Files**: - `plot-generator.md` - Base rules for all implementations +- `adaptable-code.md` - The Data block convention: one replaceable block that ends in `df`, columns named once, plot code derived from `df` - `quality-criteria.md` - Definition of quality - `quality-evaluator.md` - AI quality evaluation - `spec-id-generator.md` - Assigns spec IDs @@ -511,8 +514,9 @@ plt.savefig('plot.png', dpi=300) - `spec_roles.py` - Data roles from a spec's `## Data` bullets, the data shape (table, matrix, network, generated, excluded) and the color-driving role; shared by the agent and the adaptability checker - `adaptability.py` - Adaptability checker for Python implementations ([Adaptable catalogue code](../concepts/adaptable-code.md)): the record, the CQ-01 defect lines, and the block and constant substitution; `python -m core.adaptability report plots/` prints the catalogue baseline - `adaptability_mirror.py` - Builds `automation/scripts/adaptability_check.py`, the checker as one standard-library file; `uv run python -m core.adaptability_mirror` after a change to `adaptability.py` or `spec_roles.py` -- `perturbation.py` - The perturbation smoke run's tables (renamed, more and fewer categories, one group, fewer and more rows, scaled values) and the source to run on each +- `perturbation.py` - The perturbation smoke run's tables (renamed, more and fewer categories, one group, fewer and more rows, scaled values, a missing value in the loader's dtypes) and the source to run on each - `render_identity.py` - Share of differing pixels between two renders against a bound (0.5 % per theme); `python -m core.render_identity old.png new.png` +- `smoke_run.py` - Runs an implementation: `run` executes it on each perturbed table in both themes, `pair` renders a new source and its predecessor in one environment and compares them, and `selfcheck` is the generator's Step 3c (static check plus smoke run, logged per attempt); `python -m core.smoke_run run plots/{spec}/implementations/python/{library}.py` --- @@ -526,7 +530,8 @@ plt.savefig('plot.png', dpi=300) - `scripts/label_manager.py` - Label operations and transitions - `scripts/workflow_cli.py` - CLI interface for workflow steps - `scripts/regen_gate.py` - Regen gate: keeps or replaces a regeneration (used by impl-review.yml) -- `scripts/adaptability_check.py` - The adaptability checker as one generated standard-library file for the workflow runner (wired into the pipeline by the adaptable-code rules PR) +- `scripts/adaptability_check.py` - The adaptability checker as one generated standard-library file for the workflow runner (impl-review.yml, "Adaptability check (record-only)") +- `scripts/adapt_record.py` - Combines the checker mirror's record and the staged smoke run into one adapt record per source (used by impl-review.yml) - `scripts/regen_writeback.py` - Stores a kept regeneration's re-score as the live implementation's review (used by impl-review.yml and impl-merge.yml) - `gcp/budget-guard/` - Cloud Run function that disables the Vertex AI API when its Cloud Billing budget is reached; a deploy unit, not a Python package (see [GCP budget guard](gcp-budget-guard.md)) diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index 0de3e4b03af..bb554e6adaa 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -123,15 +123,17 @@ Issue + [generate:{library}] label OR workflow_dispatch v impl-generate.yml |-- Creates branch: implementation/{spec-id}/{library} - |-- AI generates code + |-- AI generates code (Python: adaptability self-check, Step 3c) |-- Fails if the commit changes a dependency file |-- Creates metadata/python/{library}.yaml (initial) + |-- Python: adaptability smoke run, plus the predecessor's render on a regeneration (record-only) |-- Tests execution - |-- Uploads preview to GCS staging + |-- Uploads preview to GCS staging (adaptability records under a private adapt/ prefix) |-- Creates PR --> main | v impl-review.yml + |-- Python: adaptability check of the PR's file and its predecessor (record-only) |-- AI evaluates code + image |-- Posts review comment with score |-- Updates metadata/python/{library}.yaml (quality_score, review feedback) @@ -146,14 +148,31 @@ impl-review.yml |-- Below Threshold --> [ai-rejected] --> impl-repair.yml (max 4 attempts) | |-- Reads AI feedback | |-- Fixes implementation + | |-- Python: reruns the adaptability smoke run (record-only) | |-- Re-triggers impl-review.yml | |-- Regeneration (implementation already on main) --> regen gate, one review, no repair - |-- Replace --> [regen:improved] + [ai-approved] --> impl-merge.yml - |-- Keep --> [regen:kept] --> PR closed, live code and GCS unchanged + |-- Replace --> [regen:improved] (+ [regen:adapt] or [regen:adapt-overruled]) + [ai-approved] --> impl-merge.yml + |-- Keep --> (+ [regen:adapt-overruled]) [regen:kept] --> PR closed, live code and GCS unchanged |-- Stored review <-- re-score: [review-writeback] PR --> impl-merge.yml (writeback job) ``` +#### Adaptability steps (Python, record-only) + +Python implementations follow the Data block convention of [Adaptable catalogue code](../concepts/adaptable-code.md). These steps measure it. None of them fails a run, and the reviewer never sees their output: + +| Workflow | Step | What it does | +|---|---|---| +| `impl-generate.yml` | Adaptability smoke run (record-only) | Runs the new source on perturbed tables (`core/smoke_run.py run`). On a regeneration it also renders the predecessor next to the new source in the same runner (`pair`) and runs the smoke run on it. It records the results under `$RUNNER_TEMP/adapt` without gating. | +| `impl-generate.yml` | Upload to GCS Staging | Clears `staging/{spec}/python/{library}/adapt/` and stages this run's smoke results, pair record, raw pair renders, and the generator's self-check log there, privately and never promoted. | +| `impl-repair.yml` | Adaptability smoke run (record-only) | Reruns the smoke run on the repaired file. | +| `impl-repair.yml` | Upload repaired plot to GCS staging | Replaces `adapt/new-smoke.json`, or removes the stale one when the repair run wrote none. | +| `impl-review.yml` | Set up Python | Pins the job's `python3` to `.python-version`, which the regen tools and the checker mirror need to parse PEP 758 `except A, B:`. | +| `impl-review.yml` | Adaptability check (record-only) | Runs the workflow-ref checker mirror on the PR's Python file and, on a regeneration, on its predecessor. It combines each with the staged smoke run into `/tmp/anyplot-adapt-new.json` and `/tmp/anyplot-adapt-prev.json`, asks the gate whether the adapt route holds (`regen_gate.py adapt-route`), and skips the review only when the route holds and the owner's `ADAPT_REVIEW_FREE` switch is on. | +| `impl-review.yml` | Add carried quality score label | On a review-free adapt merge, labels the PR with the stored score that carries over. | + +`impl-merge.yml` promotes only the `plot-*` files from staging, so `adapt/` never reaches production; its cleanup removes `adapt/` with the rest of staging. + ### Dependency pins and library versions Each language's plotting stack is pinned in the repository: @@ -223,6 +242,8 @@ PR. The same test checks that every pin resolves. | `regen:forced` | Regeneration dispatched with `regen_gate=false`: bypasses the regen gate and takes the fresh-generation path | Workflow | | `regen:improved` | The regen gate replaced the live implementation (added before `ai-approved`) | Workflow | | `regen:kept` | The regen gate kept the live implementation; the PR is closed unmerged | Workflow | +| `regen:adapt` | The regen gate replaced the live implementation on the adapt route (added after `regen:improved`); one of the two counts the owner reads before turning on `ADAPT_REVIEW_FREE` | Workflow | +| `regen:adapt-overruled` | The adapt route held before the review, and the review's floor turned it off (added before the verdict label, on a merge or a keep); the other count the owner reads before turning on `ADAPT_REVIEW_FREE` | Workflow | | `review-writeback` | A metadata PR that stores a kept regeneration's re-score as the live implementation's review (branch `review-writeback/{spec}/{library}/{kept PR}`); `impl-merge.yml` merges it | Workflow | ### Approval labels @@ -277,7 +298,7 @@ A unit test (`tests/unit/prompts/test_prompts.py`) keeps the thresholds out of t `impl-review.yml` re-runs a review at most once by itself, whatever the cause. A comment carrying the `` marker records that retry: - **No score.** The review wrote no `quality_score.txt` and no review comment ("Validate review output"): one retry through `repository_dispatch`. -- **Scored, then a later step failed before any verdict label landed.** For example, a GitHub 5xx on the `quality:N` label ("Re-dispatch review after a post-score failure"): one retry through `gh workflow run --ref` on the run's own ref. If the PR already has a verdict label (`ai-approved`, `ai-rejected`, `regen:improved`, or `regen:kept`), the step does nothing, and the watchdog's merge or repair cases take over. A verdict label that can't be added after three tries fails "Add preliminary verdict label (early)" on every path, including `regen:kept`, so this retry also covers a lost verdict label. +- **Scored, then a later step failed before any verdict label landed.** For example, a GitHub 5xx on the `quality:N` label ("Re-dispatch review after a post-score failure"): one retry through `gh workflow run --ref` on the run's own ref. If the PR already has a verdict label (`ai-approved`, `ai-rejected`, `regen:improved`, or `regen:kept`), the step does nothing, and the watchdog's merge or repair cases take over. A verdict label that can't be added after three tries fails "Add preliminary verdict label (early)" on every path, including `regen:kept`, so this retry also covers a lost verdict label. A review-free adapt run (the review skipped on `ADAPT_REVIEW_FREE`) has the same retry when a step fails before its verdict label, such as the carried `quality:N` label; its comment names the stored score, because no review scored. Both paths write the marker comment before they dispatch the retry, and they retry the comment and the dispatch three times each. A marker that can't be read counts as a spent retry, so a GitHub outage can't buy a second one. When the retry is spent, or the marker comment or the dispatch itself fails, the PR gets `ai-review-failed`. The watchdog re-dispatches the review once for a fresh generation and only flags a regeneration. A PR left with only `quality:N` and no verdict is picked up by the watchdog's never-reviewed case. `impl-review.yml` adds these labels with `GITHUB_TOKEN`, which doesn't start workflow runs, so `impl-review-retry.yml` reacts only to a label that a person adds. @@ -302,7 +323,24 @@ A regeneration is an implementation PR for a (spec, library) pair that already h - no regressions. On a `*-basic` spec, a replaced data scenario or added encodings count as regressions unless a change request asked for them. **Code path.** When no improvement fixes a defect, a regeneration still replaces the live implementation when it fixes a code defect the previous review named. The review lists it under `code_improvements`, and the gate counts it when all of these hold: the previous weakness is a `CQ-04 (code): … → \`call\`` (or `→ remove`) line that the re-score classed a CQ-04 defect; CQ-04 scores higher in the new review than in the re-score; the new source calls the named call more often than the predecessor (or the target was a removal), where a comment, a string literal, or a definition of the same name doesn't count as a call; the new source is shorter; and the data scenario and encodings are unchanged, on any spec. The score tolerance and the regression rule apply as above, and visible suggestions ride along. The gate reads both sources (`--prev-impl`, `--new-impl`); without them the code path is off. A code-path replacement is a merge, so like any merge it writes no re-score back. -4. Replace: `regen:improved`, then `ai-approved`, a PR comment with the same summary (stored, re-scored, and new score, improvements, regressions, reason), then the normal merge. Keep: `regen:kept`, the PR is closed with that comment, the issue gets `impl:{library}:done` back, and the live code and GCS production stay as they are. The re-score of the live implementation becomes its stored review through a separate metadata PR (see [Stored review on a keep](#stored-review-on-a-keep)). + + **Adapt route.** A Python regeneration that converts the implementation to the adaptable-code convention ([Adaptable catalogue code](../concepts/adaptable-code.md)) changes the code but not the picture, so it has no visible improvement to carry it. It replaces the live implementation on render identity instead, when all of these hold: + - the predecessor fails the adaptability check (the static check or the smoke run) and the new source passes both; + - the two sources' renders, both made in the same `impl-generate.yml` runner ("Adaptability smoke run (record-only)") from the very sources the records checked, differ in at most 0.5 % of their pixels in each theme, at equal size; a size change fails, and so does a pair rendered from other source versions (`pair_stale`); + - the new source has at most max(1.25 × prev, prev + 15) lines, and the canvas gate passed; + - when the review ran: a score above 0, no regression, an unchanged data scenario and no added encoding (on any spec), and no VQ, SC, DQ, or CQ item below its value in the re-score. + + The route takes the place of three keeps: `no_visible_improvement`, `no_defect_improvement`, and `below_tolerance`. The score tolerance never applies to it: the reviewer's run-to-run spread and the re-score's offset from the stored score make any tolerance a coin flip, so the technical items are the floor instead. The CQ-04 code path keeps its tolerance. A replacement on the route carries the reason code `adapt_identity`. The gate reads three files: the two records that "Adaptability check (record-only)" writes, one per source, and the render pair that impl-generate staged. A missing or malformed record, a missing render pair, or an implementation in another language turns the route off, and the record says why. It never counts as a pass. A record whose status does not follow from its static and smoke results counts as malformed (`record_invalid`), and so does a render pair that does not name the PR's library. + + `impl-review.yml` evaluates the conditions that need no review before the review runs (`regen_gate.py adapt-route`, the `adapt_route` notice). When the repository variable `ADAPT_REVIEW_FREE` is `true` and those conditions hold, the review is skipped and the gate merges on them alone (`decide --review-skipped`), carrying the stored score and review forward. A skipped review whose route does not hold is kept as `no_score`. The variable is off by default, and only the owner turns it on. The evidence is the route-held PRs: `regen:adapt` marks a merge on which the route held before and after the review, and `regen:adapt-overruled` marks a PR whose route held before the review and whose review then turned it off (`decide --adapt-route-held`, the gate output `adapt_overruled`). Route-held PRs are the `regen:adapt` PRs plus the `regen:adapt-overruled` PRs. The owner turns the switch on only when the overruled share stays near zero over at least 50 route-held PRs: + + ```bash + gh pr list --state merged --label regen:adapt --limit 1000 --json number --jq length + gh pr list --state all --label regen:adapt-overruled --limit 1000 --json number --jq length + ``` + + **Adapt regression.** On every spec, when the predecessor passes the adaptability check and the new source fails it, the live implementation stays (`adapt_regression`), whatever else would replace it. The same holds without a smoke run: when the predecessor passes the static check, its smoke run is missing, stale, or skipped, and the new source fails the static check. Converted code stays converted. +4. Replace: `regen:improved`, then `regen:adapt` when the adapt route counted or `regen:adapt-overruled` when the review overruled it, then `ai-approved`, a PR comment with the same summary (stored, re-scored, and new score, improvements, regressions, reason), then the normal merge. Keep: `regen:adapt-overruled` when the review overruled the adapt route, then `regen:kept`, the PR is closed with that comment, the issue gets `impl:{library}:done` back, and the live code and GCS production stay as they are. The re-score of the live implementation becomes its stored review through a separate metadata PR (see [Stored review on a keep](#stored-review-on-a-keep)). A crashed regen review is auto-retried once by `impl-review.yml`; after that the PR carries `ai-review-failed` and the watchdog only flags it — it never dispatches a further review for a regeneration. The same one-retry budget covers a review of any PR that scored but failed before a verdict label landed, for example on a GitHub 5xx (see [Review recovery](#review-recovery)). @@ -312,9 +350,9 @@ Every review writes its weaknesses in the two formats: defect lines first, then Every decision leaves three traces: -- **The gate record in the PR comment.** Both the keep and the replace comment end with an invisible `` marker: one line of JSON with the scores, improvement and regression counts (the improvements split into carriers, suggestions, and obsolete citations; the suggestions name additions, polish, and improvements without a kind apart, and a histogram counts the kind of every listed improvement, and `code` counts the code improvements the gate counted), the reason code (`merge`, `below_tolerance`, `no_visible_improvement`, `no_defect_improvement`, `regression`, `regen_json_invalid`, `canvas_failed`, and so on), and the provenance of both reviews (resolved model and rules version of this review and of the stored one). A keep record also says what happened to the re-score (`writeback`, see the next section). It holds no model-written text. PR comments are permanent, so read this first. -- **The notice line in the run log.** `::notice::regen_gate spec=… lib=… prev_stored=… prev_rescored=… new=… verdict=… code=… model=… criteria=… prev_lines=… new_lines=… reason=…`, where `prev_lines` and `new_lines` are the two sources' line counts. Run logs expire. -- **The pair artifact.** `regen-pair--` on the `impl-review.yml` run, kept 60 days: both renders, both sources (the predecessor's with its score hidden), the previous review, this review's files, and the gate record. The predecessor's production renders are overwritten on the next merge, so this is the only copy of what the gate compared. +- **The gate record in the PR comment.** Both the keep and the replace comment end with an invisible `` marker: one line of JSON with the scores, improvement and regression counts (the improvements split into carriers, suggestions, and obsolete citations; the suggestions name additions, polish, and improvements without a kind apart, and a histogram counts the kind of every listed improvement, and `code` counts the code improvements the gate counted), the reason code (`merge`, `below_tolerance`, `no_visible_improvement`, `no_defect_improvement`, `regression`, `adapt_identity`, `adapt_regression`, `regen_json_invalid`, `canvas_failed`, and so on), and the provenance of both reviews (resolved model and rules version of this review and of the stored one). The `adapt` block holds the adapt route: `version` (the checker version), `prev` and `new` (each source's adaptability `status`, with its `static` and `smoke` statuses), `identity` (the share of differing pixels per theme), `counted`, `why` (`route` when it counted, otherwise the first condition that failed, such as `not_python`, `prev_passes`, `identity_dark`, or `technical_below`), `review` (`ran` or `skipped`), and `route_pre_review` (whether `adapt-route` held before the review: `true`, `false`, or `null` without adapt input). A keep record also says what happened to the re-score (`writeback`, see the next section). It holds no model-written text. PR comments are permanent, so read this first. +- **The notice line in the run log.** `::notice::regen_gate spec=… lib=… prev_stored=… prev_rescored=… new=… verdict=… code=… model=… criteria=… prev_lines=… new_lines=… adapt_counted=… adapt_why=… adapt_overruled=… reason=…`, where `prev_lines` and `new_lines` are the two sources' line counts, and `adapt_counted`, `adapt_why`, and `adapt_overruled` appear when the step passes the adapt files. Run logs expire. +- **The pair artifact.** `regen-pair--` on the `impl-review.yml` run, kept 60 days: both renders, both sources (the predecessor's with its score hidden), the previous review, this review's files, and the gate record. For a Python pair, `adapt/` adds both adapt records and what impl-generate staged for them: the smoke runs, `pair.json`, the raw renders of both sources, and the generator's self-check log. The predecessor's production renders are overwritten on the next merge, so this is the only copy of what the gate compared. `uv run python -m automation.scripts.review_retest gate-report` aggregates the records across pull requests: merge rate, reason codes, how far re-scored predecessors land from their stored scores, the counted visible improvements and carriers, cited permissions and obsolete weaknesses, what happened to the re-scores of kept implementations, the merges the code path carried, and soft alarms. See [Review retest](review-retest.md#monitor-the-regen-gate). diff --git a/prompts/README.md b/prompts/README.md index 3295dfb83d6..52cbc7a6855 100644 --- a/prompts/README.md +++ b/prompts/README.md @@ -12,6 +12,7 @@ Git history shows all changes (`git log -p prompts/plot-generator.md`). | File | Agent | Task | |------|-------|------| | `plot-generator.md` | Plot Generator | Base rules for all plot implementations | +| `adaptable-code.md` | Plot Generator, Repair | The Data block convention: one replaceable `# Data` block ending in `df`, columns named once, plot code derived from `df` (`docs/concepts/adaptable-code.md`) | | `default-style-guide.md` | Plot Generator | Default visual style (colors, typography, layout) | | `library/*.md` | Plot Generator | Library-specific rules (15 files: 8 Python + ggplot2 + makie + 5 JavaScript: chartjs, d3, echarts, highcharts, muix) | | `quality-criteria.md` | All | Definition of what "good code" means | diff --git a/prompts/adaptable-code.md b/prompts/adaptable-code.md new file mode 100644 index 00000000000..c6feabb060d --- /dev/null +++ b/prompts/adaptable-code.md @@ -0,0 +1,147 @@ +# Adaptable code + +Every implementation is code that someone points at other data. A reader copies it from the plot page and swaps in a CSV; the "Use with my data" agent replaces the sample with the user's table. Write it so that one replaceable block holds the sample, each column is named once, and the plot code derives everything else from the table. + +This file is the convention, in one place. `prompts/plot-generator.md` applies it, and each `prompts/library/{library}.md` has a "Data block" section with the library's idiom. For Python, a static checker and a perturbation smoke run test it: the generator runs both as its self-check (`prompts/workflow-prompts/impl-generate-claude.md`, Step 3c). R, Julia and JavaScript follow the same convention without a checker yet. + +## The six zones + +A Python implementation has six zones, top to bottom: + +1. **Head.** The docstring header, the imports, the theme tokens, and the palette: the full 8-entry `IMPRINT_PALETTE` list in canonical order, the name the style guide and every library prompt use (the checker also accepts the short name `IMPRINT` for the same list), or `BRAND` alone for a single-series plot. Library setup goes here, never into the data block: `sns.set_theme(...)`, `LetsPlot.setup_html()`, `theme_set(...)`, an Altair theme. No figure is created and nothing is plotted here. +2. **`# Constants`, optional.** Values of the plot type or the domain that stay true for any data: thresholds (RSI 30 and 70, a target the spec names), ordered scales (weekday names, Likert levels), and semantic color maps looked up with a palette fallback (`SEMANTIC.get(level, IMPRINT_PALETTE[i])`). Never sample categories, sample values or data coordinates. +3. **Column constants.** One constant per spec `## Data` role the code uses: a domain name (`MONTH`, `SALES`, never `CATEGORY_COL`), the column name as its value, written the way the axis title should read with the unit included, and a trailing role comment: `SALES = "Sales (thousand USD)" # role: value`. A variadic family (`y1, y2, ...`) gets one constant that holds a list. A column that no spec role names gets `# role: extra`. +4. **Exactly one `# Data` block.** Everything that invents the example: the seed, random draws, literal lists, category names, the formulas behind synthetic data, a `sns.load_dataset` call. It ends in one tidy DataFrame `df`: one row per mark or observation, one column per role, keyed by the column constants. It binds nothing else that later code reads, and it holds no plotting or styling call and no theme token. The data stays synthetic and in memory; a CSV read appears only in the block's opening comment. +5. **Plot code, below a `# Plot` marker.** Everything derived: pivots, statistics, fits, sorting, positions, widths, limits. It reads only `df`, the column constants, `# Constants`, the theme tokens and the palette. +6. **Output.** `plt.savefig(f"plot-{THEME}.png", dpi=400, facecolor=PAGE_BG)` or the library's equivalent, never with `bbox_inches="tight"`. + +The markers are comment lines at column 0: `# Data` and `# Plot`, the word alone or followed by a colon and a hint, such as `# Data: replace this block with your own table, for example df = pd.read_csv("sales.csv")`. The role in `# role: ` is the backticked name of the spec's `## Data` bullet (`category`, `value`); the family `y1, y2, ...` is the role `y`. Other comments, such as `# Style`, are free. + +## Rules for the plot code + +- **Colors are looked up per category:** `IMPRINT_PALETTE[i]`, `dict(zip(groups, IMPRINT_PALETTE))`, or the library's color cycle fed the full list. Never `zip(groups, IMPRINT_PALETTE, strict=True)` and never a literal slice sized to the sample (`IMPRINT_PALETTE[:3]`). seaborn takes `IMPRINT_PALETTE[:n]` with `n` counted from `df`, because it warns when the palette outnumbers the hue levels. With `IMPRINT_PALETTE[i]` (an `IndexError`) or a lookup such as `colors[group]` in `dict(zip(groups, IMPRINT_PALETTE))` (a `KeyError`), a ninth group fails loudly, which matches the style guide's "9+: out of palette, use small multiples". A library's color cycle repeats colors instead, so prefer a lookup by position or by group where the code assigns colors itself. +- **Labels, ticks and limits come from `df` and the constants:** `ax.set_xlabel(QUARTER)`, `ax.set_xticks(x, table.index)`, `ax.set_ylim(0, top * 1.15)`. Below the block, never type a column name, a sample category, or a data coordinate (a limit, a tick, a reference-line position, a text position in data units). Text placed in axes fractions or offset points is fine, and so are fixed domains such as 0 to 1, 0 to 100 and 0 to 360. +- **Statistics are computed f-strings:** `f"r = {df[HOURS].corr(df[SCORE]):.2f}"`, never `"r = 0.87"`. +- **Callout targets come from a rule over `df`:** `idxmax`, the latest row, the smallest gap between the two leading groups, or the first crossing of a `# Constants` threshold. Never a literal such as `categories.index("Q3")`, and never a comparison such as `df[MONTH] == 7`, also not through an alias like `peak_month = 7`. The text claims only what the rule guarantees, and a conditional claim (a near-tie, a record) is drawn only when `df` meets it. +- **Counts come from `df`:** `range(len(table))`, `np.arange(df[MONTH].nunique())`, never `range(12)` for twelve sample months. +- **A missing value stays missing.** A user's table can hold `NaN` in a number column, and the agent's loader hands categories over as pandas `string` columns and integers as nullable `Int64`. Never fill a drawn value with zero (`fill_value=0`, `fillna(0)`); reduce with NaN-aware calls (`np.nanmax`, `Series.max()`) and let the library skip the mark. +- **Category order is the data's order:** `pivot_table(..., sort=False)`, `groupby(..., sort=False)`. A plain `pivot` sorts the labels and puts months in alphabetical order. +- **Restraint is unchanged.** Whether a callout exists at all is still decided by "Annotation restraint" in `prompts/plot-generator.md` and by the `-basic` variant rule. The convention only decides where the choice of target lives. + +## Data shapes + +The spec's `## Data` role bullets decide what `df` is: + +- **Table, the default.** Every role is a column of one table, one row per mark, observation, edge or interval. OHLC rows, Sankey and chord edge lists (source, target, value), hierarchies (id, parent, value), Gantt tasks (task, start, end), and grids given as long x, y, z observations are tables, pivoted below the block. Model diagnostics take the raw inputs the spec names (`y_true`, `y_score`) and compute the curve below the block. +- **Matrix.** A role names a matrix (correlation, confusion, adjacency): `df` is a labeled square DataFrame with the row labels as its index, and only the column-valued roles, such as a value label for the color bar, get constants. When the plot computes the matrix itself, its roles are the observations and the shape is a table. +- **Network.** Node and edge roles: `df` is the edge list, a second `nodes` table exists only when the spec names node attributes, and the layout is computed below the block. +- **Generated.** Functions, simulations, encoded text and puzzle layouts (a surface from a formula, a QR code, a maze): the block holds named parameters, with no `df` and no column constants. +- **Maps** fetch their geometry over the network and are exempt. + +The self-check prints the shape it reads from the spec. + +## Example: bar-grouped in matplotlib + +Condensed: no shadows, value labels or chrome styling. It renders 3200 × 1800 in both themes. + +```python +import os + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +# Theme tokens +THEME = os.getenv("ANYPLOT_THEME", "light") +PAGE_BG = "#FAF8F1" if THEME == "light" else "#1A1A17" +INK = "#1A1A17" if THEME == "light" else "#F0EFE8" +INK_SOFT = "#4A4A44" if THEME == "light" else "#B8B7B0" + +# Imprint palette: all eight positions, one per group in order +IMPRINT_PALETTE = ["#009E73", "#C475FD", "#4467A3", "#BD8233", "#AE3030", "#2ABCCD", "#954477", "#99B314"] + +# Constants +NEAR_TIE = 0.05 # a gap below this share of the tallest bar counts as a near-tie + +# Columns: set these to your table's column names +QUARTER = "Quarter" # role: category +PRODUCT_LINE = "Product line" # role: group +SALES = "Sales (thousand USD)" # role: value + +# Data: replace this block with your own table, for example df = pd.read_csv("sales.csv") +df = pd.DataFrame( + { + QUARTER: ["Q1", "Q2", "Q3", "Q4"] * 3, + PRODUCT_LINE: ["Electronics"] * 4 + ["Clothing"] * 4 + ["Home & Garden"] * 4, + SALES: [245, 312, 287, 425, 178, 195, 285, 310, 125, 210, 195, 165], + } +) + +# Plot: reads only df and the constants above +table = df.pivot_table(index=QUARTER, columns=PRODUCT_LINE, values=SALES, sort=False) +x = np.arange(len(table.index)) +width = 0.75 / len(table.columns) +top = np.nanmax(table.to_numpy()) + +fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400, facecolor=PAGE_BG) +ax.set_facecolor(PAGE_BG) +for i, line in enumerate(table.columns): + offset = (i - (len(table.columns) - 1) / 2) * width + ax.bar(x + offset, table[line], width, label=line, color=IMPRINT_PALETTE[i], edgecolor=INK_SOFT, linewidth=1.0) + +# Callout: target chosen by a rule over df, drawn only when the claim holds +if len(table.columns) > 1: + ranked = np.sort(np.nan_to_num(table.to_numpy(), nan=-np.inf), axis=1) + gaps = ranked[:, -1] - ranked[:, -2] + tie = int(np.argmin(gaps)) + if gaps[tie] <= NEAR_TIE * top: + ax.annotate( + f"Near-tie in {table.index[tie]}", + xy=(x[tie], ranked[tie, -1]), + xytext=(0, 14), + textcoords="offset points", + ha="center", + fontsize=8, + color=INK_SOFT, + style="italic", + ) + +# Style +ax.set_title("bar-grouped · python · matplotlib · anyplot.ai", fontsize=13, fontweight="bold", color=INK) +ax.set_xticks(x, table.index) +ax.set_xlabel(QUARTER, color=INK) +ax.set_ylabel(SALES, color=INK) +ax.set_ylim(0, top * 1.15) +ax.legend(title=PRODUCT_LINE, fontsize=8) +plt.savefig(f"plot-{THEME}.png", dpi=400, facecolor=PAGE_BG) +``` + +`sort=False` keeps the data's category order. The pivot has no `fill_value`: a group that is missing in a quarter stays `NaN` and matplotlib skips the bar instead of drawing a zero. The axis maximum and the ranking skip those cells (`np.nanmax`, and `-inf` for a missing group), so a missing cell neither raises on the axis limit nor fakes a near-tie. `0.75 / len(table.columns)` gives the width of 0.25 that a hand-tuned file would use for three groups, and narrower bars for more. + +To use your own CSV, change the three strings and the block: + +```diff +-QUARTER = "Quarter" # role: category +-PRODUCT_LINE = "Product line" # role: group +-SALES = "Sales (thousand USD)" # role: value ++QUARTER = "Region" # role: category ++PRODUCT_LINE = "Channel" # role: group ++SALES = "Revenue (k EUR)" # role: value + + # Data: replace this block with your own table, for example df = pd.read_csv("sales.csv") +-df = pd.DataFrame({...}) ++df = pd.read_csv("revenue.csv") +``` + +## Other languages + +R, Julia and JavaScript use the same zones with their own comment syntax: + +| Language | Markers | Role comment | The block ends in | +|---|---|---|---| +| R (ggplot2) | `# Data`, `# Plot` | `QUARTER <- "Quarter" # role: category` | `df`, a `tibble` keyed by the constants | +| Julia (Makie) | `# Data`, `# Plot` | `const QUARTER = "Quarter" # role: category` | `df`, a `DataFrame` keyed by the constants | +| JavaScript | `// Data`, `// Plot` | `const QUARTER = "Quarter"; // role: category` | `const data`, an array of records, one object per row keyed by the constants | + +The rules for the plot code apply unchanged: colors per category from the palette, labels and limits from the data, callout targets from a rule. Each library prompt's "Data block" section shows the idiom. No checker runs on these languages yet, so the convention applies to new files; on a regeneration it is no reason to rewrite working data code. diff --git a/prompts/default-style-guide.md b/prompts/default-style-guide.md index 5a37a6246a7..c73b4641dd8 100644 --- a/prompts/default-style-guide.md +++ b/prompts/default-style-guide.md @@ -116,6 +116,8 @@ Categorical-color distinguishability falls with n. The hybrid-v3 sort holds ΔE **Default to fewer series.** Before adding the Nth color, ask: "could this be small multiples / faceting?" Usually the answer is yes and the chart improves. +**In code, list all eight hues and use the first n.** The palette constant always holds the eight categorical hues in canonical order, and each group takes its color by position (`IMPRINT_PALETTE[i]`, or the library's color cycle fed the full list), with `n` counted from the data, never a literal slice sized to the sample (`[:3]`). The same code then colors one more group correctly on other data. Where the code picks colors by position or by group (`IMPRINT_PALETTE[i]`, `colors[group]`), a ninth group fails loudly instead of recycling a color; a library's color cycle repeats colors instead, so prefer the explicit lookup where the code assigns colors itself. + ### Optional outline pattern (AI judgment) 5 of the 8 categorical hues + amber fall below WCAG 2.1 SC 1.4.11's 3:1 contrast on the cream bg (lavender, ochre, cyan, lime, and amber). On dark bg, only matte-red is marginally below. This is the known muted-palette / WCAG tension Okabe-Ito, Paul Tol "muted", and ColorBrewer Set2 all share. @@ -213,7 +215,7 @@ Output file names: `plot-light.png` / `plot-dark.png` (static libraries) plus `p - Generous margins around the plot area - Padding between legend and plot, between title and axes - Let the data breathe — don't cram elements together -- `tight_layout()` / `bbox_inches='tight'` to avoid wasted canvas, but don't compress +- `tight_layout()` or the library's margin settings to avoid wasted canvas, but don't compress. Never `bbox_inches='tight'`: it trims the saved PNG below the fixed canvas size ### Typography Hierarchy diff --git a/prompts/library/altair.md b/prompts/library/altair.md index 61f07ce87c2..1bf36bc7d5c 100644 --- a/prompts/library/altair.md +++ b/prompts/library/altair.md @@ -51,8 +51,8 @@ The HTML save is left untouched — only PNGs are gated. ```python chart = alt.Chart(df).mark_point(size=60).encode( # size ~2-3x default, density-aware - x='col_x:Q', - y='col_y:Q' + x=alt.X(STUDY_HOURS.translate(ESCAPE), type="quantitative", title=STUDY_HOURS), # see "Data block" + y=alt.Y(EXAM_SCORE.translate(ESCAPE), type="quantitative", title=EXAM_SCORE), ).properties( width=620, # see Canvas table — landscape inner-view height=320, @@ -74,20 +74,40 @@ See `prompts/default-style-guide.md` "Proportional Sizing" for review criteria. ## Encoding Types +Pass the type as `type=` next to the escaped field (see "Data block"); the shorthand codes in a field string (`"Sales:Q"`) break on a column name that contains a colon. + ```python # Q = Quantitative (numeric) -x='value:Q' +alt.X(SALES.translate(ESCAPE), type="quantitative") # N = Nominal (categorical, no order) -color='category:N' +alt.Color(REGION.translate(ESCAPE), type="nominal") # O = Ordinal (categorical, with order) -x='month:O' +alt.X(MONTH.translate(ESCAPE), type="ordinal") # T = Temporal (date/time) -x='date:T' +alt.X(DATE.translate(ESCAPE), type="temporal") +``` + +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`. Altair reads the tidy `df` directly with `alt.Chart(df)` and the column constants as fields. Vega-Lite reads `.` and `[]` in a field name as nested access and `:` as a type suffix, so a column such as `"Sales (k USD.)"` breaks the chart, silently or with an error. Escape the constants once below the block, and give every encoding its `type=` and its `title=`, because an escaped field shows its backslashes in the automatic title: + +```python +# Plot: reads only df and the constants above +ESCAPE = str.maketrans({".": "\\.", "[": "\\[", "]": "\\]", ":": "\\:"}) +chart = alt.Chart(df).mark_bar().encode( + x=alt.X(QUARTER.translate(ESCAPE), type="nominal", title=QUARTER, sort=None), + xOffset=alt.XOffset(PRODUCT_LINE.translate(ESCAPE), type="nominal", sort=None), + y=alt.Y(SALES.translate(ESCAPE), type="quantitative", title=SALES), + color=alt.Color(PRODUCT_LINE.translate(ESCAPE), type="nominal", title=PRODUCT_LINE, sort=None, + scale=alt.Scale(range=IMPRINT_PALETTE)), +) ``` +`sort=None` keeps the data's order instead of sorting the labels. Scale domains come from the data (`alt.Scale(domain=[0, df[SALES].max() * 1.15])`) or stay automatic, never typed numbers. + ## Marks ```python @@ -116,8 +136,9 @@ chart.save(f'plot-{THEME}.png', scale_factor=4.0) # Enable zoom/pan chart = chart.interactive() -# Tooltips -.encode(tooltip=['col_x', 'col_y']) +# Tooltips (escaped fields, readable titles) +.encode(tooltip=[alt.Tooltip(QUARTER.translate(ESCAPE), title=QUARTER), + alt.Tooltip(SALES.translate(ESCAPE), title=SALES)]) ``` ## Colors @@ -130,19 +151,20 @@ IMPRINT_PALETTE = ['#009E73', '#C475FD', '#4467A3', '#BD8233', ANYPLOT_AMBER = '#DDCC77' # warning / caution (outside the categorical pool) # Single-series -alt.Chart(df).mark_circle(color=IMPRINT_PALETTE[0]).encode(x='x', y='y') +alt.Chart(df).mark_circle(color=IMPRINT_PALETTE[0]).encode(x=..., y=...) -# Multi-series +# Multi-series: the full list; Vega-Lite hands out the first n in order alt.Chart(df).mark_circle().encode( - x='x', y='y', - color=alt.Color('category:N', scale=alt.Scale(range=IMPRINT_PALETTE)), + x=..., y=..., + color=alt.Color(SCHOOL.translate(ESCAPE), type="nominal", title=SCHOOL, + scale=alt.Scale(range=IMPRINT_PALETTE)), ) # Continuous — only the two Imprint palette-derived cmaps are allowed: # Sequential: two-stop range -alt.Color('value:Q', scale=alt.Scale(range=['#009E73', '#4467A3'])) +alt.Color(DENSITY.translate(ESCAPE), type="quantitative", scale=alt.Scale(range=['#009E73', '#4467A3'])) # Diverging: three-stop range with domainMid at 0 -alt.Color('delta:Q', scale=alt.Scale( +alt.Color(CHANGE.translate(ESCAPE), type="quantitative", scale=alt.Scale( range=['#AE3030', '#FAF8F1', '#4467A3'], domainMid=0, )) diff --git a/prompts/library/bokeh.md b/prompts/library/bokeh.md index 6b56d0a9810..3b9198ae167 100644 --- a/prompts/library/bokeh.md +++ b/prompts/library/bokeh.md @@ -8,6 +8,7 @@ from bokeh.plotting import figure from bokeh.models import ColumnDataSource from bokeh.io import output_file, save +from bokeh.transform import dodge # grouped bars, see "Data block" ``` ## Canvas — hard rule, no deviation @@ -48,25 +49,41 @@ p = figure( ## Plot Methods ```python +source = ColumnDataSource(df) + # Numeric data -p.scatter(x='x', y='y', source=source) -p.line(x='x', y='y', source=source) +p.scatter(x=STUDY_HOURS, y=EXAM_SCORE, source=source) +p.line(x=DATE, y=PRICE, source=source) -# IMPORTANT: Categorical axes -p = figure(x_range=categories, ...) # define x_range! -source = ColumnDataSource(data={'x': cat_data, 'y': num_data}) -p.scatter(x='x', y='y', source=source) +# IMPORTANT: Categorical axes — define x_range from the data, in its own order +p = figure(x_range=list(df[QUARTER].unique()), ...) +p.vbar(x=QUARTER, top=SALES, width=0.8, source=source) ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`. bokeh reads the tidy `df` through `ColumnDataSource(df)`, and every glyph names its columns with the constants: a string argument is a column name, so names with spaces, units and dots work as they are. bokeh never titles an axis from a column name, so pass `x_axis_label=QUARTER, y_axis_label=SALES` and the legend title from the constants. Groups come from the data, each with its palette position: + +```python +# Plot: reads only df and the constants above +p = figure(x_range=list(df[QUARTER].unique()), x_axis_label=QUARTER, y_axis_label=SALES, ...) +lines = list(df[PRODUCT_LINE].unique()) +width = 0.75 / len(lines) +for i, line in enumerate(lines): + rows = ColumnDataSource(df[df[PRODUCT_LINE] == line]) + offset = (i - (len(lines) - 1) / 2) * width + p.vbar(x=dodge(QUARTER, offset, range=p.x_range), top=SALES, width=width, source=rows, + color=IMPRINT_PALETTE[i], legend_label=str(line)) +p.legend.title = PRODUCT_LINE +``` + +In a tooltip, wrap a column name with spaces in braces: `tooltips=[(SALES, f"@{{{SALES}}}")]`. Ranges come from the data (`Range1d(0, df[SALES].max() * 1.15)`) or stay automatic, never typed numbers. + ## ColumnDataSource ```python -# Always use ColumnDataSource for flexibility -source = ColumnDataSource(data={ - 'x': df['col_x'], - 'y': df['col_y'], - 'color': df['col_color'] -}) +# Always use ColumnDataSource, built from the tidy df +source = ColumnDataSource(df) ``` ## Save (HTML + PNG via headless Chrome) @@ -149,11 +166,12 @@ IMPRINT_PALETTE = ['#009E73', '#C475FD', '#4467A3', '#BD8233', ANYPLOT_AMBER = '#DDCC77' # warning / caution (outside the categorical pool) # Single-series -p.scatter(x, y, color=IMPRINT_PALETTE[0]) +p.scatter(x=STUDY_HOURS, y=EXAM_SCORE, source=source, color=IMPRINT_PALETTE[0]) -# Multi-series: iterate in canonical order -for i, group in enumerate(groups): - p.scatter(..., color=IMPRINT_PALETTE[i], legend_label=group) +# Multi-series: iterate the data's groups in canonical palette order +for i, (school, rows) in enumerate(df.groupby(SCHOOL, sort=False)): + p.scatter(x=STUDY_HOURS, y=EXAM_SCORE, source=ColumnDataSource(rows), + color=IMPRINT_PALETTE[i], legend_label=str(school)) # Continuous — only the two Imprint palette-derived cmaps are allowed. # Build a 256-stop ramp by interpolating between the endpoints: diff --git a/prompts/library/chartjs.md b/prompts/library/chartjs.md index 8d79a6f237f..9eaf3a48aa5 100644 --- a/prompts/library/chartjs.md +++ b/prompts/library/chartjs.md @@ -83,6 +83,54 @@ browser; if you need pseudo-randomness, use a tiny fixed-seed LCG or hard-code the data array. **Never `fetch()` or load from a CDN/network** — the runtime is offline and the HTML must be self-contained. +## Data block + +The file follows the six zones of `prompts/adaptable-code.md` in JavaScript +syntax: role constants above the block, one `// Data` block that ends in +`const data`, an array of records (one object per row, keyed by the +constants), and plot code below `// Plot` that reads fields with `d[QUARTER]`: + +```js +// Columns: set these to your data's column names +const QUARTER = "Quarter"; // role: category +const PRODUCT_LINE = "Product line"; // role: group +const SALES = "Sales (thousand USD)"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Electronics", [SALES]: 245 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Electronics", [SALES]: 312 }, + { [QUARTER]: "Q3", [PRODUCT_LINE]: "Electronics", [SALES]: 287 }, + { [QUARTER]: "Q4", [PRODUCT_LINE]: "Electronics", [SALES]: 425 }, + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Clothing", [SALES]: 178 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Clothing", [SALES]: 195 }, + { [QUARTER]: "Q3", [PRODUCT_LINE]: "Clothing", [SALES]: 285 }, + { [QUARTER]: "Q4", [PRODUCT_LINE]: "Clothing", [SALES]: 310 }, +]; + +// Plot: reads only data and the constants above +const quarters = [...new Set(data.map((d) => d[QUARTER]))]; +const lines = [...new Set(data.map((d) => d[PRODUCT_LINE]))]; +const datasets = lines.map((line, i) => ({ + label: line, + data: quarters.map((q) => data.find((d) => d[QUARTER] === q && d[PRODUCT_LINE] === line)?.[SALES] ?? null), + backgroundColor: t.palette[i], +})); +// … new Chart(canvas, { type: "bar", data: { labels: quarters, datasets }, options: { scales: { +// x: { title: { display: true, text: QUARTER } }, y: { title: { display: true, text: SALES } } } } }) +``` + +- Categories, groups and their order come from the records (`new Set` keeps the + order of appearance), never from a typed list. +- Each group takes its palette position (`t.palette[i]`). +- Axis titles and series labels come from the constants. +- A missing value stays `null`, which Chart.js draws as a gap. +- Limits come from the data or stay automatic, never typed numbers. + +The "Use with my data" agent will later replace the block with +`const data = window.ANYPLOT_DATA;`. Do not read that global yourself: the +harness does not set it. + ## Colors — Imprint palette (from `window.ANYPLOT_TOKENS`) The first categorical series is **always** `#009E73` (`tokens.palette[0]`). @@ -93,8 +141,10 @@ light and dark — only chrome (text, grid, background) flips. const t = window.ANYPLOT_TOKENS; // single series backgroundColor: t.palette[0] // #009E73 brand green -// multi series -backgroundColor: categories.map((_, i) => t.palette[i % t.palette.length]) +// multi series: one palette position per series, never recycled (no `i % t.palette.length`) +const datasets = lines.map((line, i) => ({ label: line, backgroundColor: t.palette[i] /* , data: … */ })); +// one dataset colored per category (pie, doughnut): the same rule per label +backgroundColor: quarters.map((_, i) => t.palette[i]) ``` Never use `viridis`/`rainbow`/`jet` or arbitrary hex. For continuous color use @@ -117,8 +167,8 @@ options: { legend: { labels: { color: t.ink, font: { size: 16 } } }, }, scales: { - x: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid }, title: { display: true, text: "X", color: t.ink } }, - y: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid }, title: { display: true, text: "Y", color: t.ink } }, + x: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid }, title: { display: true, text: DAY, color: t.ink } }, + y: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid }, title: { display: true, text: VISITS, color: t.ink } }, }, } ``` @@ -131,25 +181,34 @@ and rely on `plugins` + per-arc `backgroundColor`. A complete Chart.js implementation reads the theme tokens, generates data, appends a canvas to `#container`, and builds the chart. The same single `.js` file handles both themes (the harness sets `window.ANYPLOT_THEME` per render). +It follows the zones of "Data block" above. ```js //# anyplot-orientation: landscape const t = window.ANYPLOT_TOKENS; -// --- Data (in-memory, deterministic) --------------------------------------- -const labels = ["Mon", "Tue", "Wed", "Thu", "Fri"]; -const values = [12, 19, 7, 15, 11]; +// Columns: set these to your data's column names +const DAY = "Day"; // role: category +const VISITS = "Visits"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [DAY]: "Mon", [VISITS]: 12 }, + { [DAY]: "Tue", [VISITS]: 19 }, + { [DAY]: "Wed", [VISITS]: 7 }, + { [DAY]: "Thu", [VISITS]: 15 }, + { [DAY]: "Fri", [VISITS]: 11 }, +]; -// --- Mount ----------------------------------------------------------------- +// Plot: reads only data and the constants above const canvas = document.createElement("canvas"); document.getElementById("container").appendChild(canvas); -// --- Chart ----------------------------------------------------------------- new Chart(canvas, { type: "bar", data: { - labels, - datasets: [{ label: "Visits", data: values, backgroundColor: t.palette[0], borderWidth: 0 }], + labels: data.map((d) => d[DAY]), + datasets: [{ label: VISITS, data: data.map((d) => d[VISITS]), backgroundColor: t.palette[0], borderWidth: 0 }], }, options: { responsive: true, @@ -160,8 +219,10 @@ new Chart(canvas, { legend: { labels: { color: t.ink, font: { size: 16 } } }, }, scales: { - x: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid } }, - y: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid }, beginAtZero: true }, + x: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid }, + title: { display: true, text: DAY, color: t.ink } }, + y: { ticks: { color: t.inkSoft, font: { size: 14 } }, grid: { color: t.grid }, beginAtZero: true, + title: { display: true, text: VISITS, color: t.ink } }, }, }, }); diff --git a/prompts/library/d3.md b/prompts/library/d3.md index 4a24964164f..4bdbab9a87f 100644 --- a/prompts/library/d3.md +++ b/prompts/library/d3.md @@ -89,6 +89,52 @@ browser; use a tiny fixed-seed LCG or hard-code the data. **Never `fetch()`, `d3.csv`, `d3.json`, or load from a CDN/network** — the runtime is offline and the HTML must be self-contained. Build arrays in code. +## Data block + +The file follows the six zones of `prompts/adaptable-code.md` in JavaScript +syntax: role constants above the block, one `// Data` block that ends in +`const data`, an array of records (one object per row, keyed by the +constants), and plot code below `// Plot` that reads fields with `d[QUARTER]`. +Records are D3's native form, so scales and marks read them directly: + +```js +// Columns: set these to your data's column names +const QUARTER = "Quarter"; // role: category +const PRODUCT_LINE = "Product line"; // role: group +const SALES = "Sales (thousand USD)"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Electronics", [SALES]: 245 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Electronics", [SALES]: 312 }, + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Clothing", [SALES]: 178 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Clothing", [SALES]: 195 }, +]; + +// Plot: reads only data and the constants above +const x = d3.scaleBand().domain(new Set(data.map((d) => d[QUARTER]))).range([0, iw]).padding(0.2); +const xGroup = d3.scaleBand().domain(new Set(data.map((d) => d[PRODUCT_LINE]))).range([0, x.bandwidth()]).padding(0.05); +const y = d3.scaleLinear().domain([0, d3.max(data, (d) => d[SALES])]).nice().range([ih, 0]); +const color = d3.scaleOrdinal().domain(xGroup.domain()).range(t.palette); +g.selectAll("rect").data(data.filter((d) => d[SALES] != null)).join("rect") + .attr("x", (d) => x(d[QUARTER]) + xGroup(d[PRODUCT_LINE])) + .attr("y", (d) => y(d[SALES])) + .attr("width", xGroup.bandwidth()).attr("height", (d) => ih - y(d[SALES])) + .attr("fill", (d) => color(d[PRODUCT_LINE])); +``` + +- Categories, groups and their order come from the records (a `Set` keeps the + order of appearance), never from a typed list, and each group takes its + palette position. +- Domains come from the data (`d3.max`, `d3.extent`, which skip missing + values), never typed numbers. A missing value (`null`) is filtered out of the + marks, never drawn as zero. +- Axis titles and legend labels come from the constants. + +The "Use with my data" agent will later replace the block with +`const data = window.ANYPLOT_DATA;`. Do not read that global yourself: the +harness does not set it. + ## Colors — Imprint palette (from `window.ANYPLOT_TOKENS`) The first categorical series is **always** `#009E73` (`tokens.palette[0]`). Data @@ -125,6 +171,8 @@ svg.append("text").attr("fill", t.ink).style("font-size", "22px") /* … */; ## Script Skeleton +A complete D3 implementation in the zones of "Data block" above: + ```js //# anyplot-orientation: landscape const t = window.ANYPLOT_TOKENS; @@ -133,21 +181,25 @@ const margin = { top: 80, right: 60, bottom: 70, left: 80 }; const iw = width - margin.left - margin.right; const ih = height - margin.top - margin.bottom; -// --- Data (in-memory, deterministic) --------------------------------------- +// Columns: set these to your data's column names +const DAY = "Day"; // role: category +const VISITS = "Visits"; // role: value + +// Data: replace this block with your own records, one object per row const data = [ - { label: "Mon", value: 12 }, { label: "Tue", value: 19 }, { label: "Wed", value: 7 }, - { label: "Thu", value: 15 }, { label: "Fri", value: 11 }, + { [DAY]: "Mon", [VISITS]: 12 }, { [DAY]: "Tue", [VISITS]: 19 }, { [DAY]: "Wed", [VISITS]: 7 }, + { [DAY]: "Thu", [VISITS]: 15 }, { [DAY]: "Fri", [VISITS]: 11 }, ]; -// --- SVG mount -------------------------------------------------------------- +// Plot: reads only data and the constants above const svg = d3.select("#container").append("svg").attr("width", width).attr("height", height); const g = svg.append("g").attr("transform", `translate(${margin.left},${margin.top})`); -// --- Scales ----------------------------------------------------------------- -const x = d3.scaleBand().domain(data.map((d) => d.label)).range([0, iw]).padding(0.2); -const y = d3.scaleLinear().domain([0, d3.max(data, (d) => d.value)]).nice().range([ih, 0]); +// Scales +const x = d3.scaleBand().domain(data.map((d) => d[DAY])).range([0, iw]).padding(0.2); +const y = d3.scaleLinear().domain([0, d3.max(data, (d) => d[VISITS])]).nice().range([ih, 0]); -// --- Axes ------------------------------------------------------------------- +// Axes const xAxis = g.append("g").attr("transform", `translate(0,${ih})`).call(d3.axisBottom(x)); const yAxis = g.append("g").call(d3.axisLeft(y)); for (const ax of [xAxis, yAxis]) { @@ -155,14 +207,18 @@ for (const ax of [xAxis, yAxis]) { ax.selectAll("line").attr("stroke", t.grid); ax.select(".domain").attr("stroke", t.inkSoft); } +g.append("text").attr("x", iw / 2).attr("y", ih + 56).attr("text-anchor", "middle") + .attr("fill", t.ink).style("font-size", "16px").text(DAY); +g.append("text").attr("transform", "rotate(-90)").attr("x", -ih / 2).attr("y", -60) + .attr("text-anchor", "middle").attr("fill", t.ink).style("font-size", "16px").text(VISITS); -// --- Bars ------------------------------------------------------------------- +// Bars g.selectAll("rect").data(data).join("rect") - .attr("x", (d) => x(d.label)).attr("y", (d) => y(d.value)) - .attr("width", x.bandwidth()).attr("height", (d) => ih - y(d.value)) + .attr("x", (d) => x(d[DAY])).attr("y", (d) => y(d[VISITS])) + .attr("width", x.bandwidth()).attr("height", (d) => ih - y(d[VISITS])) .attr("fill", t.palette[0]); -// --- Title ------------------------------------------------------------------ +// Title svg.append("text").attr("x", width / 2).attr("y", 44).attr("text-anchor", "middle") .attr("fill", t.ink).style("font-size", "22px").style("font-weight", "600") .text("bar-basic · javascript · d3 · anyplot.ai"); diff --git a/prompts/library/echarts.md b/prompts/library/echarts.md index 1fa85db1197..3aba037bc4d 100644 --- a/prompts/library/echarts.md +++ b/prompts/library/echarts.md @@ -85,6 +85,58 @@ browser; use a tiny fixed-seed LCG or hard-code the data. **Never `fetch()` or load from a CDN/network** — the runtime is offline and the HTML must be self-contained. +## Data block + +The file follows the six zones of `prompts/adaptable-code.md` in JavaScript +syntax: role constants above the block, one `// Data` block that ends in +`const data`, an array of records (one object per row, keyed by the +constants), and plot code below `// Plot` that reads fields with `d[QUARTER]`: + +```js +// Columns: set these to your data's column names +const QUARTER = "Quarter"; // role: category +const PRODUCT_LINE = "Product line"; // role: group +const SALES = "Sales (thousand USD)"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Electronics", [SALES]: 245 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Electronics", [SALES]: 312 }, + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Clothing", [SALES]: 178 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Clothing", [SALES]: 195 }, +]; + +// Plot: reads only data and the constants above +const quarters = [...new Set(data.map((d) => d[QUARTER]))]; +const lines = [...new Set(data.map((d) => d[PRODUCT_LINE]))]; +const chart = echarts.init(document.getElementById("container")); +chart.setOption({ + color: t.palette, + xAxis: { type: "category", data: quarters, name: QUARTER }, + yAxis: { type: "value", name: SALES }, + legend: { data: lines }, + series: lines.map((line) => ({ + type: "bar", + name: line, + data: quarters.map((q) => data.find((d) => d[QUARTER] === q && d[PRODUCT_LINE] === line)?.[SALES] ?? null), + })), + // … animation, chrome and grid as in "Theme-adaptive chrome" +}); +``` + +- Categories, groups and their order come from the records (`new Set` keeps + the order of appearance), never from a typed list. +- The palette goes in whole as `color: t.palette`; ECharts hands out the first + n in order. +- Axis names and series names come from the constants. +- A missing value stays `null`, which ECharts draws as a gap. +- Limits come from the data or stay automatic, never typed numbers. + +A single series can also read the records directly: `dataset: { source: data }` +with `encode: { x: DAY, y: VISITS }` on the series. The "Use with my data" +agent will later replace the block with `const data = window.ANYPLOT_DATA;`. +Do not read that global yourself: the harness does not set it. + ## Colors — Imprint palette (from `window.ANYPLOT_TOKENS`) The first categorical series is **always** `#009E73` (`tokens.palette[0]`). Set @@ -130,18 +182,28 @@ Pie/radar/sunburst have no `xAxis`/`yAxis` — drop those and style via `label` ## Script Skeleton +A complete ECharts implementation in the zones of "Data block" above: + ```js //# anyplot-orientation: landscape const t = window.ANYPLOT_TOKENS; -// --- Data (in-memory, deterministic) --------------------------------------- -const labels = ["Mon", "Tue", "Wed", "Thu", "Fri"]; -const values = [12, 19, 7, 15, 11]; +// Columns: set these to your data's column names +const DAY = "Day"; // role: category +const VISITS = "Visits"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [DAY]: "Mon", [VISITS]: 12 }, + { [DAY]: "Tue", [VISITS]: 19 }, + { [DAY]: "Wed", [VISITS]: 7 }, + { [DAY]: "Thu", [VISITS]: 15 }, + { [DAY]: "Fri", [VISITS]: 11 }, +]; -// --- Init ------------------------------------------------------------------- +// Plot: reads only data and the constants above const chart = echarts.init(document.getElementById("container")); -// --- Option ----------------------------------------------------------------- chart.setOption({ animation: false, color: t.palette, @@ -149,15 +211,17 @@ chart.setOption({ title: { text: "bar-basic · javascript · echarts · anyplot.ai", left: "center", textStyle: { color: t.ink, fontSize: 22 } }, grid: { left: 80, right: 60, top: 90, bottom: 70 }, - xAxis: { type: "category", data: labels, + xAxis: { type: "category", data: data.map((d) => d[DAY]), name: DAY, + nameLocation: "middle", nameGap: 36, nameTextStyle: { color: t.ink, fontSize: 16 }, axisLabel: { color: t.inkSoft, fontSize: 14 }, axisLine: { lineStyle: { color: t.inkSoft } }, splitLine: { show: false } }, - yAxis: { type: "value", + yAxis: { type: "value", name: VISITS, + nameTextStyle: { color: t.ink, fontSize: 16 }, axisLabel: { color: t.inkSoft, fontSize: 14 }, axisLine: { lineStyle: { color: t.inkSoft } }, splitLine: { lineStyle: { color: t.grid } } }, - series: [{ type: "bar", data: values, itemStyle: { color: t.palette[0] } }], + series: [{ type: "bar", name: VISITS, data: data.map((d) => d[VISITS]), itemStyle: { color: t.palette[0] } }], }); ``` diff --git a/prompts/library/ggplot2.md b/prompts/library/ggplot2.md index f1589a65ec2..195cdba0ef9 100644 --- a/prompts/library/ggplot2.md +++ b/prompts/library/ggplot2.md @@ -58,15 +58,46 @@ Optional dataset packages available in the CI runtime: `palmerpenguins`, ## Reproducibility ```r -set.seed(42) +set.seed(42) # first line of the # Data block ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md` in R syntax: the column constants above the block, one `# Data` block that builds a `tibble` keyed by them and ends in `df`, and plot code below `# Plot` that reads columns through `.data[[...]]`: + +```r +# Columns: set these to your table's column names +QUARTER <- "Quarter" # role: category +PRODUCT_LINE <- "Product line" # role: group +SALES <- "Sales (thousand USD)" # role: value + +# Data: replace this block with your own table, for example df <- read.csv("sales.csv", check.names = FALSE) +df <- tibble::tibble( + !!QUARTER := rep(c("Q1", "Q2", "Q3", "Q4"), times = 3), + !!PRODUCT_LINE := rep(c("Electronics", "Clothing", "Home & Garden"), each = 4), + !!SALES := c(245, 312, 287, 425, 178, 195, 285, 310, 125, 210, 195, 165) +) + +# Plot: reads only df and the constants above +df <- df |> mutate(across(all_of(c(QUARTER, PRODUCT_LINE)), \(v) factor(v, levels = unique(v)))) +p <- ggplot(df, aes(x = .data[[QUARTER]], y = .data[[SALES]], fill = .data[[PRODUCT_LINE]])) + + geom_col(position = position_dodge(width = 0.8), width = 0.75) + + scale_fill_manual(values = IMPRINT_PALETTE) + + labs(x = QUARTER, y = SALES, fill = PRODUCT_LINE, title = "bar-grouped · r · ggplot2 · anyplot.ai") +``` + +- `!!QUARTER :=` names a tibble column by the constant's value, so the column name is typed once. +- `.data[[QUARTER]]` reads a column whose name has spaces or a unit, which a bare name in `aes()` cannot. +- Set every axis and legend title with `labs()` from the constants: `aes()` with `.data[[...]]` does not reliably title the axes with the column name. +- A character column sorts alphabetically, on the axis and in the color mapping. The `factor(v, levels = unique(v))` line keeps the data's order, so the first group in the data gets `#009E73`. +- `scale_fill_manual()` gets all eight hues and uses the first n. Limits come from the data (`ylim(0, max(df[[SALES]], na.rm = TRUE) * 1.15)`) or stay automatic, never typed numbers. + ## Create Plot ```r -p <- ggplot(df, aes(x = col_x, y = col_y)) + +p <- ggplot(df, aes(x = .data[[STUDY_HOURS]], y = .data[[EXAM_SCORE]])) + geom_point() + - labs(x = x_label, y = y_label, title = plot_title) + + labs(x = STUDY_HOURS, y = EXAM_SCORE, title = plot_title) + theme_minimal(base_size = 8) ``` @@ -194,7 +225,7 @@ anyplot_theme <- theme_minimal(base_size = 8) + # `alpha` for grid lines, so use a faint INK color and rely on the natural # contrast against PAGE_BG instead. -p <- ggplot(df, aes(x, y)) + +p <- ggplot(df, aes(x = .data[[STUDY_HOURS]], y = .data[[EXAM_SCORE]])) + geom_point(color = IMPRINT_PALETTE[1], size = 2.5) + anyplot_theme ``` @@ -221,16 +252,15 @@ facet_grid(rows ~ cols) A complete ggplot2 implementation reads `ANYPLOT_THEME`, generates data, builds the plot, and saves both themed PNGs by being invoked twice with different env -values. The same single `.R` file handles both themes — no two-file split. +values. The same single `.R` file handles both themes — no two-file split. It +follows the zones of "Data block" above. ```r library(ggplot2) library(dplyr) library(ragg) -set.seed(42) - -# --- Theme tokens ----------------------------------------------------------- +# Theme tokens THEME <- Sys.getenv("ANYPLOT_THEME", "light") PAGE_BG <- if (THEME == "light") "#FAF8F1" else "#1A1A17" INK <- if (THEME == "light") "#1A1A17" else "#F0EFE8" @@ -238,16 +268,22 @@ INK_SOFT <- if (THEME == "light") "#4A4A44" else "#B8B7B0" IMPRINT_PALETTE <- c("#009E73", "#C475FD", "#4467A3", "#BD8233", "#AE3030", "#2ABCCD", "#954477", "#99B314") -# --- Data ------------------------------------------------------------------- +# Columns: set these to your table's column names +STUDY_HOURS <- "Study hours per day" # role: x +EXAM_SCORE <- "Exam score (%)" # role: y + +# Data: replace this block with your own table, for example df <- read.csv("scores.csv", check.names = FALSE) +set.seed(42) +hours <- rnorm(100, mean = 6, sd = 2) df <- tibble::tibble( - x = rnorm(100), - y = rnorm(100) + !!STUDY_HOURS := hours, + !!EXAM_SCORE := hours * 8 + rnorm(100, sd = 5) + 30 ) -# --- Plot ------------------------------------------------------------------- -p <- ggplot(df, aes(x, y)) + +# Plot: reads only df and the constants above +p <- ggplot(df, aes(x = .data[[STUDY_HOURS]], y = .data[[EXAM_SCORE]])) + geom_point(color = IMPRINT_PALETTE[1], size = 2.5, alpha = 0.7) + - labs(title = "scatter-basic · r · ggplot2 · anyplot.ai", x = "X", y = "Y") + + labs(title = "scatter-basic · r · ggplot2 · anyplot.ai", x = STUDY_HOURS, y = EXAM_SCORE) + theme_minimal(base_size = 8) + theme( plot.background = element_rect(fill = PAGE_BG, color = PAGE_BG), @@ -259,7 +295,7 @@ p <- ggplot(df, aes(x, y)) + plot.title = element_text(color = INK, size = 12) ) -# --- Save ------------------------------------------------------------------- +# Save ggsave( filename = sprintf("plot-%s.png", THEME), plot = p, @@ -289,8 +325,9 @@ ggsave( - `ggsave()` resets the active graphics device. Use `device = ragg::agg_png` explicitly — the platform default Cairo path is not installed in the CI image. -- Use `tibble::tibble(...)` (or plain `data.frame(...)`) — `dplyr::tibble` was - removed in dplyr ≥ 1.0; `tibble::tibble` is the correct path. +- Use `tibble::tibble(...)` — `dplyr::tibble` was removed in dplyr ≥ 1.0; + `tibble::tibble` is the correct path. It also takes the `!!QUARTER :=` + column names of the Data block, which plain `data.frame(...)` does not. - Avoid `print(p)` at script end — `ggsave()` already renders the plot, and an extra `print()` opens an unused interactive device. diff --git a/prompts/library/highcharts.md b/prompts/library/highcharts.md index 6ee355af3c3..669757988de 100644 --- a/prompts/library/highcharts.md +++ b/prompts/library/highcharts.md @@ -101,6 +101,55 @@ browser; use a tiny fixed-seed LCG or hard-code the data. **Never `fetch()` or load from a CDN/network** — the runtime is offline and the HTML must be self-contained. +## Data block + +The file follows the six zones of `prompts/adaptable-code.md` in JavaScript +syntax: role constants above the block, one `// Data` block that ends in +`const data`, an array of records (one object per row, keyed by the +constants), and plot code below `// Plot` that reads fields with `d[QUARTER]`: + +```js +// Columns: set these to your data's column names +const QUARTER = "Quarter"; // role: category +const PRODUCT_LINE = "Product line"; // role: group +const SALES = "Sales (thousand USD)"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Electronics", [SALES]: 245 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Electronics", [SALES]: 312 }, + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Clothing", [SALES]: 178 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Clothing", [SALES]: 195 }, +]; + +// Plot: reads only data and the constants above +const quarters = [...new Set(data.map((d) => d[QUARTER]))]; +const lines = [...new Set(data.map((d) => d[PRODUCT_LINE]))]; +Highcharts.chart("container", { + colors: t.palette, + xAxis: { categories: quarters, title: { text: QUARTER } }, + yAxis: { title: { text: SALES } }, + series: lines.map((line) => ({ + type: "column", + name: line, + data: quarters.map((q) => data.find((d) => d[QUARTER] === q && d[PRODUCT_LINE] === line)?.[SALES] ?? null), + })), + // … palette, chart, credits, animation and chrome as in "Theme-adaptive chrome" +}); +``` + +- Categories, groups and their order come from the records (`new Set` keeps + the order of appearance), never from a typed list. +- The palette goes in whole as `colors: t.palette`; Highcharts hands out the + first n in order. +- Axis titles and series names come from the constants. +- A missing value stays `null`, which Highcharts draws as a gap. +- Limits come from the data or stay automatic, never typed numbers. + +The "Use with my data" agent will later replace the block with +`const data = window.ANYPLOT_DATA;`. Do not read that global yourself: the +harness does not set it. + ## Colors — Imprint palette (from `window.ANYPLOT_TOKENS`) The first categorical series is **always** `#009E73` (`tokens.palette[0]`). Set @@ -165,15 +214,26 @@ light or dark. ## Script Skeleton +A complete Highcharts implementation in the zones of "Data block" above: + ```js //# anyplot-orientation: landscape const t = window.ANYPLOT_TOKENS; -// --- Data (in-memory, deterministic) --------------------------------------- -const categories = ["Mon", "Tue", "Wed", "Thu", "Fri"]; -const values = [12, 19, 7, 15, 11]; +// Columns: set these to your data's column names +const DAY = "Day"; // role: category +const VISITS = "Visits"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [DAY]: "Mon", [VISITS]: 12 }, + { [DAY]: "Tue", [VISITS]: 19 }, + { [DAY]: "Wed", [VISITS]: 7 }, + { [DAY]: "Thu", [VISITS]: 15 }, + { [DAY]: "Fri", [VISITS]: 11 }, +]; -// --- Chart ----------------------------------------------------------------- +// Plot: reads only data and the constants above Highcharts.chart("container", { palette: { colorScheme: t.theme }, chart: { type: "column", backgroundColor: "transparent", animation: false, @@ -182,16 +242,17 @@ Highcharts.chart("container", { colors: t.palette, title: { text: "bar-basic · javascript · highcharts · anyplot.ai", style: { color: t.ink, fontSize: "22px", fontWeight: "600" } }, - xAxis: { categories, + xAxis: { categories: data.map((d) => d[DAY]), + title: { text: DAY, style: { color: t.inkSoft, fontSize: "16px" } }, lineColor: t.inkSoft, tickColor: t.inkSoft, labels: { style: { color: t.inkSoft, fontSize: "14px" } } }, - yAxis: { title: { text: "Value", style: { color: t.inkSoft, fontSize: "16px" } }, + yAxis: { title: { text: VISITS, style: { color: t.inkSoft, fontSize: "16px" } }, gridLineColor: t.grid, labels: { style: { color: t.inkSoft, fontSize: "14px" } } }, legend: { enabled: false }, plotOptions: { series: { animation: false }, column: { borderWidth: 0 } }, - series: [{ name: "Visits", data: values, colorByPoint: true }], + series: [{ name: VISITS, data: data.map((d) => d[VISITS]), colorByPoint: true }], }); ``` diff --git a/prompts/library/letsplot.md b/prompts/library/letsplot.md index 0a3d570d2ee..c82e6bcb318 100644 --- a/prompts/library/letsplot.md +++ b/prompts/library/letsplot.md @@ -15,12 +15,27 @@ LetsPlot.setup_html() # Required for notebook/export ```python plot = ( - ggplot(df, aes(x='col_x', y='col_y')) + ggplot(df, aes(x=STUDY_HOURS, y=EXAM_SCORE)) + geom_point() - + labs(x=x_label, y=y_label, title=title) + + labs(title=title) ) ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`. lets-plot reads the tidy `df` directly with the column constants: `ggplot(df, aes(x=QUARTER, y=SALES, fill=PRODUCT_LINE))`. Column names with spaces, units and dots work as they are. lets-plot titles the axes and the legend from the column names, so `labs()` is needed only for the title or to change a label, and it keeps the data's order of a string column on the axis and in the color mapping. `LetsPlot.setup_html()` belongs in the head, above the column constants, never in the block. + +```python +# Plot: reads only df and the constants above +plot = ( + ggplot(df, aes(x=QUARTER, y=SALES, fill=PRODUCT_LINE)) + + geom_bar(stat="identity", position=position_dodge(width=0.8), width=0.75) + + scale_fill_manual(values=IMPRINT_PALETTE) +) +``` + +Continuous limits come from the data (`scale_y_continuous(limits=[0, df[SALES].max() * 1.15])`) or stay automatic, never typed numbers. + ## Canvas — hard rule, no deviation The saved PNG must be **exactly** one of these two sizes (post-render gate in `impl-review.yml` rejects anything off by more than 16 px and re-triggers repair): @@ -73,11 +88,11 @@ ggsave(plot, f'plot-{THEME}.html') ## Aesthetics ```python -# Mappings -aes(x='col_x', y='col_y', color='category', size='value') +# Mappings: the column constants, never typed column names +aes(x=STUDY_HOURS, y=EXAM_SCORE, color=SCHOOL, size=CLASS_SIZE) # Fixed aesthetics (outside aes) -geom_point(color='blue', size=5, alpha=0.7) +geom_point(color=IMPRINT_PALETTE[0], size=5, alpha=0.7) ``` ## Geoms @@ -132,8 +147,8 @@ geom_density() # Density ## Facets ```python -+ facet_wrap('category', ncol=3) -+ facet_grid(x='row_var', y='col_var') ++ facet_wrap(REGION, ncol=3) ++ facet_grid(x=REGION, y=CHANNEL) ``` ## Colors @@ -179,7 +194,7 @@ anyplot_theme = theme( legend_title=element_text(color=INK), ) -plot = (ggplot(df, aes('x', 'y')) + geom_point(color=IMPRINT_PALETTE[0]) + anyplot_theme) +plot = (ggplot(df, aes(x=STUDY_HOURS, y=EXAM_SCORE)) + geom_point(color=IMPRINT_PALETTE[0]) + anyplot_theme) ``` ## Output Files diff --git a/prompts/library/makie.md b/prompts/library/makie.md index 142536bd147..6a80b7f705e 100644 --- a/prompts/library/makie.md +++ b/prompts/library/makie.md @@ -70,9 +70,40 @@ ImageCore at 0.9 and cannot coexist with Makie 0.24. ## Reproducibility ```julia -Random.seed!(42) +Random.seed!(42) # first line of the # Data block +``` + +## Data block + +The file follows the six zones of `prompts/adaptable-code.md` in Julia syntax: the column constants above the block, one `# Data` block that builds a `DataFrame` keyed by them and ends in `df`, and plot code below `# Plot` that reads columns with `df[!, QUARTER]`: + +```julia +# Columns: set these to your table's column names +const QUARTER = "Quarter" # role: category +const PRODUCT_LINE = "Product line" # role: group +const SALES = "Sales (thousand USD)" # role: value + +# Data: replace this block with your own table, for example df = CSV.read("sales.csv", DataFrame) +df = DataFrame( + QUARTER => repeat(["Q1", "Q2", "Q3", "Q4"], 3), + PRODUCT_LINE => repeat(["Electronics", "Clothing", "Home & Garden"]; inner = 4), + SALES => [245, 312, 287, 425, 178, 195, 285, 310, 125, 210, 195, 165], +) + +# Plot: reads only df and the constants above +quarters = unique(df[!, QUARTER]) +lines = unique(df[!, PRODUCT_LINE]) +x = [findfirst(==(q), quarters) for q in df[!, QUARTER]] +group = [findfirst(==(l), lines) for l in df[!, PRODUCT_LINE]] +ax = Axis(fig[1, 1]; xlabel = QUARTER, ylabel = SALES, xticks = (1:length(quarters), quarters)) +barplot!(ax, x, df[!, SALES]; dodge = group, color = IMPRINT_PALETTE[group]) +Legend(fig[1, 2], [PolyElement(color = IMPRINT_PALETTE[i]) for i in eachindex(lines)], lines, PRODUCT_LINE) ``` +- `QUARTER => ...` names a column by the constant's value, so the column name is typed once; `df[!, QUARTER]` reads it back, spaces and units included. +- Categories, groups and their order come from `unique(...)` over the data, never from a typed list, and each group takes its palette position. Set the axis labels from the constants. +- Limits come from the data (`ylims!(ax, 0, maximum(skipmissing(df[!, SALES])) * 1.15)`) or stay automatic, never typed numbers. `skipmissing` keeps a missing value from a user's CSV out of the reduction instead of turning it into an error. + ## Canvas — hard rule, no deviation The saved PNG must be **exactly** one of these two sizes (post-render gate @@ -229,16 +260,15 @@ ax1 = Axis(fig[1, 1]); ax2 = Axis(fig[1, 2]) A complete Makie implementation reads `ANYPLOT_THEME`, generates data, builds the figure, and saves both themed PNGs by being invoked twice with different env values. The same single `.jl` file handles both themes — no -two-file split. +two-file split. It follows the zones of "Data block" above. ```julia using CairoMakie using Colors +using DataFrames using Random -Random.seed!(42) - -# --- Theme tokens ----------------------------------------------------------- +# Theme tokens const THEME = get(ENV, "ANYPLOT_THEME", "light") const PAGE_BG = THEME == "light" ? colorant"#FAF8F1" : colorant"#1A1A17" const INK = THEME == "light" ? colorant"#1A1A17" : colorant"#F0EFE8" @@ -248,11 +278,16 @@ const IMPRINT_PALETTE = [ colorant"#AE3030", colorant"#2ABCCD", colorant"#954477", colorant"#99B314", ] -# --- Data ------------------------------------------------------------------- -x = randn(100) -y = randn(100) +# Columns: set these to your table's column names +const STUDY_HOURS = "Study hours per day" # role: x +const EXAM_SCORE = "Exam score (%)" # role: y + +# Data: replace this block with your own table, for example df = CSV.read("scores.csv", DataFrame) +Random.seed!(42) +hours = 6 .+ 2 .* randn(100) +df = DataFrame(STUDY_HOURS => hours, EXAM_SCORE => hours .* 8 .+ 5 .* randn(100) .+ 30) -# --- Plot ------------------------------------------------------------------- +# Plot: reads only df and the constants above fig = Figure( size = (1600, 900), fontsize = 14, @@ -264,8 +299,8 @@ ax = Axis( title = "scatter-basic · julia · makie · anyplot.ai", titlesize = 20, titlecolor = INK, - xlabel = "X", - ylabel = "Y", + xlabel = STUDY_HOURS, + ylabel = EXAM_SCORE, xlabelcolor = INK, ylabelcolor = INK, xticklabelcolor = INK_SOFT, @@ -279,9 +314,9 @@ ax = Axis( ygridcolor = RGBAf(INK.r, INK.g, INK.b, 0.15), ) -scatter!(ax, x, y; color = IMPRINT_PALETTE[1], markersize = 12, strokewidth = 0) +scatter!(ax, df[!, STUDY_HOURS], df[!, EXAM_SCORE]; color = IMPRINT_PALETTE[1], markersize = 12, strokewidth = 0) -# --- Save ------------------------------------------------------------------- +# Save save("plot-$(THEME).png", fig; px_per_unit = 2) ``` @@ -320,9 +355,12 @@ assigned. If the file has no leading `#` block, the rewrite prepends one. the runner; keep the script silent except for the `save` call. - `colorant"#RRGGBB"` (from `Colors`) is the canonical hex-to-color helper. Prefer it over manual `RGB(r/255, g/255, b/255)` constructors. -- For multi-series categorical color, pass an `IMPRINT_PALETTE[1:n]` slice to - `colormap = ...` rather than letting Makie's default cycle pick the - first color (which is **not** anyplot brand green). +- For multi-series categorical color, index the palette by group position + (`IMPRINT_PALETTE[group]`, as in "Data block") or pass + `IMPRINT_PALETTE[1:n]` with `n = length(unique(df[!, GROUP]))` to + `colormap = ...`, rather than letting Makie's default cycle pick the first + color (which is **not** anyplot brand green). Count `n` from the data, never + type it. - The implementation file must end with a trailing newline — Julia parses fine without it, but the impl-review header rewrite assumes one. diff --git a/prompts/library/matplotlib.md b/prompts/library/matplotlib.md index 87a33444990..d922b380f34 100644 --- a/prompts/library/matplotlib.md +++ b/prompts/library/matplotlib.md @@ -50,6 +50,22 @@ ax.bar(x, y) plt.scatter(x, y) ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`, whose full example is a matplotlib bar chart. matplotlib reads the tidy `df` through the column constants and never titles an axis from a column name, so set the axis labels and the legend title from the constants. Groups come from the data in its own order, each with its palette position: + +```python +# Plot: reads only df and the constants above +fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400, facecolor=PAGE_BG) +for i, (region, rows) in enumerate(df.groupby(REGION, sort=False)): + ax.plot(rows[MONTH], rows[SALES], color=IMPRINT_PALETTE[i], linewidth=2.5, label=region) +ax.set_xlabel(MONTH, fontsize=10, color=INK) +ax.set_ylabel(SALES, fontsize=10, color=INK) +ax.legend(title=REGION, fontsize=8) +``` + +Ticks and limits come from the data too: `ax.set_xticks(x, table.index)` after a `pivot_table(..., sort=False)`, `ax.set_ylim(0, df[SALES].max() * 1.15)`, never a typed list or number. + ## Save ```python @@ -121,8 +137,9 @@ ANYPLOT_AMBER = '#DDCC77' # warning / caution # Single-series: always position 1 (brand green) color = IMPRINT_PALETTE[0] # '#009E73' -# Multi-series: take the first N colors in order, don't cherry-pick -ax.set_prop_cycle(color=IMPRINT_PALETTE[:N]) +# Multi-series: list all eight; the cycle hands out the first n in order, so don't cherry-pick +# and don't slice the list to the sample's group count +ax.set_prop_cycle(color=IMPRINT_PALETTE) # Continuous data — only the two Imprint palette-derived cmaps are allowed # (no viridis/cividis/BrBG/Reds/Blues/Greens/jet/hsv/rainbow): diff --git a/prompts/library/muix.md b/prompts/library/muix.md index 819aededa0e..03c2486144f 100644 --- a/prompts/library/muix.md +++ b/prompts/library/muix.md @@ -59,6 +59,14 @@ import { BarChart } from "@mui/x-charts/BarChart"; const t = window.ANYPLOT_TOKENS; // see "Globals" below +// Columns: set these to your data's column names +const DAY = "Day"; // role: category +const VISITS = "Visits"; // role: value + +// Data: replace this block with your own records (see "Data block" below) +const data = [{ [DAY]: "Mon", [VISITS]: 12 }, { [DAY]: "Tue", [VISITS]: 19 }, { [DAY]: "Wed", [VISITS]: 7 }]; + +// Plot: reads only data and the constants above export default function Chart() { return ( d[DAY]) }]} + series={[{ data: data.map((d) => d[VISITS]), label: VISITS }]} /> ); } @@ -162,6 +170,63 @@ browser; use a tiny fixed-seed LCG or hard-code the data. **Never `fetch()` or load from a CDN/network** — the runtime is offline and the HTML must be self-contained. +## Data block + +The file follows the six zones of `prompts/adaptable-code.md` in JavaScript +syntax: role constants above the block, one `// Data` block that ends in +`const data`, an array of records (one object per row, keyed by the +constants), and plot code below `// Plot` that reads fields with `d[QUARTER]`. +All three sit at module level, above the default-exported component: + +```tsx +// Columns: set these to your data's column names +const QUARTER = "Quarter"; // role: category +const PRODUCT_LINE = "Product line"; // role: group +const SALES = "Sales (thousand USD)"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Electronics", [SALES]: 245 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Electronics", [SALES]: 312 }, + { [QUARTER]: "Q1", [PRODUCT_LINE]: "Clothing", [SALES]: 178 }, + { [QUARTER]: "Q2", [PRODUCT_LINE]: "Clothing", [SALES]: 195 }, +]; + +// Plot: reads only data and the constants above +const quarters = [...new Set(data.map((d) => d[QUARTER]))]; +const lines = [...new Set(data.map((d) => d[PRODUCT_LINE]))]; + +export default function Chart() { + return ( + ({ + label: line, + data: quarters.map((q) => data.find((d) => d[QUARTER] === q && d[PRODUCT_LINE] === line)?.[SALES] ?? null), + }))} + /> + ); +} +``` + +- Categories, groups and their order come from the records (`new Set` keeps + the order of appearance), never from a typed list. +- The palette goes in whole as `colors={t.palette}`; MUI X hands out the first + n in order. +- Axis labels and series labels come from the constants; size the axis band + for a label as described under "Sizing". +- A missing value stays `null`, which MUI X draws as a gap. +- Limits come from the data or stay automatic, never typed numbers. + +The "Use with my data" agent will later replace the block with +`const data = window.ANYPLOT_DATA;`. Do not read that global yourself: the +harness does not set it. + ## When MUI X can't express the spec MUI X community covers bar, line, area, pie, scatter, sparkline, gauge, radar, @@ -207,11 +272,20 @@ import { BarChart } from "@mui/x-charts/BarChart"; const t = window.ANYPLOT_TOKENS; -// --- Data (in-memory, deterministic) --------------------------------------- -const categories = ["Mon", "Tue", "Wed", "Thu", "Fri"]; -const values = [12, 19, 7, 15, 11]; +// Columns: set these to your data's column names +const DAY = "Day"; // role: category +const VISITS = "Visits"; // role: value + +// Data: replace this block with your own records, one object per row +const data = [ + { [DAY]: "Mon", [VISITS]: 12 }, + { [DAY]: "Tue", [VISITS]: 19 }, + { [DAY]: "Wed", [VISITS]: 7 }, + { [DAY]: "Thu", [VISITS]: 15 }, + { [DAY]: "Fri", [VISITS]: 11 }, +]; -// --- Chart (default-exported component — the harness mounts it) ------------- +// Plot: the default-exported component reads only data and the constants above export default function Chart() { return ( d[DAY]) }]} + series={[{ data: data.map((d) => d[VISITS]), label: VISITS }]} /> ); } diff --git a/prompts/library/plotly.md b/prompts/library/plotly.md index 005edd1e291..dd574aaf581 100644 --- a/prompts/library/plotly.md +++ b/prompts/library/plotly.md @@ -15,12 +15,33 @@ import plotly.express as px ```python # Graph Objects fig = go.Figure() -fig.add_trace(go.Scatter(x=x, y=y)) +fig.add_trace(go.Scatter(x=df[DATE], y=df[PRICE])) # Express (for quick plots) -fig = px.scatter(df, x='col_x', y='col_y') +fig = px.scatter(df, x=STUDY_HOURS, y=EXAM_SCORE) ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`. plotly express reads the tidy `df` directly with the column constants and titles the axes and the legend from the column names: + +```python +# Plot: reads only df and the constants above +fig = px.bar(df, x=QUARTER, y=SALES, color=PRODUCT_LINE, barmode="group", + color_discrete_sequence=IMPRINT_PALETTE) +``` + +With graph objects, loop over the groups of `df` in their own order and set the titles from the constants: + +```python +fig = go.Figure() +for i, (line, rows) in enumerate(df.groupby(PRODUCT_LINE, sort=False)): + fig.add_trace(go.Bar(x=rows[QUARTER], y=rows[SALES], name=str(line), marker_color=IMPRINT_PALETTE[i])) +fig.update_layout(xaxis_title=QUARTER, yaxis_title=SALES, legend_title_text=PRODUCT_LINE) +``` + +Axis ranges come from the data (`yaxis_range=[0, df[SALES].max() * 1.15]`) or stay automatic, never typed numbers. + ## Canvas — hard rule, no deviation The saved PNG must be **exactly** one of these two sizes (post-render gate in `impl-review.yml` rejects anything off by more than 16 px and re-triggers repair): @@ -87,11 +108,11 @@ IMPRINT_PALETTE = ['#009E73', '#C475FD', '#4467A3', '#BD8233', ANYPLOT_AMBER = '#DDCC77' # warning / caution (outside the categorical pool) # Single-series -fig = go.Figure(go.Scatter(x=x, y=y, mode='markers', +fig = go.Figure(go.Scatter(x=df[STUDY_HOURS], y=df[EXAM_SCORE], mode='markers', marker=dict(color=IMPRINT_PALETTE[0]))) -# Multi-series via color_discrete_sequence (plotly express) -fig = px.scatter(df, x='x', y='y', color='category', +# Multi-series via color_discrete_sequence (plotly express): the full list, never a slice +fig = px.scatter(df, x=STUDY_HOURS, y=EXAM_SCORE, color=SCHOOL, color_discrete_sequence=IMPRINT_PALETTE) # Continuous — only the two Imprint palette-derived cmaps are allowed: diff --git a/prompts/library/plotnine.md b/prompts/library/plotnine.md index eb9759eae4f..63bb6747085 100644 --- a/prompts/library/plotnine.md +++ b/prompts/library/plotnine.md @@ -43,13 +43,31 @@ from plotnine import ( ```python plot = ( - ggplot(df, aes(x='col_x', y='col_y')) + ggplot(df, aes(x=STUDY_HOURS, y=EXAM_SCORE)) + geom_point() - + labs(x=x_label, y=y_label, title=title) + + labs(title=title) + theme_minimal() ) ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`. plotnine reads the tidy `df` directly: `ggplot(df, aes(x=QUARTER, y=SALES, fill=PRODUCT_LINE))`. A string in `aes()` that names a column is used as that column, so names with spaces, units and dots work as they are; compute a derived column below the block instead of writing an expression into `aes()`. plotnine titles the axes and the legend from the column names, so `labs()` is needed only for the title or to change a label. + +plotnine sorts a string column alphabetically, on the axis and in the color mapping. Keep the data's order, so the first group in the data gets `#009E73`, with limits taken from `df`: + +```python +# Plot: reads only df and the constants above +plot = ( + ggplot(df, aes(x=QUARTER, y=SALES, fill=PRODUCT_LINE)) + + geom_col(position=position_dodge(width=0.8), width=0.75) + + scale_x_discrete(limits=df[QUARTER].unique().tolist()) + + scale_fill_manual(values=IMPRINT_PALETTE, limits=df[PRODUCT_LINE].unique().tolist()) +) +``` + +Continuous limits come from the data (`scale_y_continuous(limits=(0, df[SALES].max() * 1.15))`) or stay automatic, never typed numbers. + ## Canvas — hard rule, no deviation The saved PNG must be **exactly** one of these two sizes (post-render gate in `impl-review.yml` rejects anything off by more than 16 px and re-triggers repair): @@ -111,9 +129,9 @@ ANYPLOT_AMBER = '#DDCC77' # warning / caution (outside the categorical pool) # Single-series + geom_point(color=IMPRINT_PALETTE[0]) -# Multi-series -+ scale_color_manual(values=IMPRINT_PALETTE) -+ scale_fill_manual(values=IMPRINT_PALETTE) +# Multi-series: the full list, in the data's group order (see "Data block") ++ scale_color_manual(values=IMPRINT_PALETTE, limits=df[GROUP].unique().tolist()) ++ scale_fill_manual(values=IMPRINT_PALETTE, limits=df[GROUP].unique().tolist()) # Continuous — only the two Imprint palette-derived cmaps are allowed: from plotnine import scale_color_gradient, scale_color_gradient2, scale_fill_gradient, scale_fill_gradient2 @@ -155,7 +173,7 @@ anyplot_theme = theme( legend_title=element_text(color=INK), ) -plot = (ggplot(df, aes('x', 'y')) + geom_point(color=IMPRINT_PALETTE[0]) + anyplot_theme) +plot = (ggplot(df, aes(x=STUDY_HOURS, y=EXAM_SCORE)) + geom_point(color=IMPRINT_PALETTE[0]) + anyplot_theme) ggsave(plot, filename=f'plot-{THEME}.png', dpi=400, width=8, height=4.5) ``` diff --git a/prompts/library/pygal.md b/prompts/library/pygal.md index 28fa5758a7e..9afec7eaef7 100644 --- a/prompts/library/pygal.md +++ b/prompts/library/pygal.md @@ -43,13 +43,17 @@ pygal.Histogram() # Histogram ## Add Data ```python -chart.add('Series 1', [1, 2, 3, 4]) -chart.add('Series 2', [4, 3, 2, 1]) - -# X-axis labels -chart.x_labels = ['A', 'B', 'C', 'D'] +# Plot: pygal takes plain lists, built from df below the Data block +table = df.pivot_table(index=QUARTER, columns=PRODUCT_LINE, values=SALES, sort=False) +chart.x_labels = table.index.tolist() +for line in table.columns: + chart.add(str(line), table[line].tolist()) ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`: the block still ends in a tidy `df`. pygal takes plain Python lists, never a DataFrame, so build them from `df` below the block with `.tolist()`: `df[SALES].tolist()` for one series, a `pivot_table(..., sort=False)` per group as above, and `list(zip(df[STUDY_HOURS], df[EXAM_SCORE]))` for the `(x, y)` points of an XY chart. pygal never titles an axis from a column name, so pass `x_title=QUARTER, y_title=SALES` from the constants. The palette goes in whole through `Style(colors=IMPRINT_PALETTE)`; pygal hands out the first n in order. + ## Save ```python diff --git a/prompts/library/seaborn.md b/prompts/library/seaborn.md index 360d7e0ad6b..b410a80c999 100644 --- a/prompts/library/seaborn.md +++ b/prompts/library/seaborn.md @@ -52,13 +52,25 @@ g.figure.set_dpi(400) ```python # Axes-level (preferred) -sns.scatterplot(data=df, x='col_x', y='col_y', ax=ax) +sns.scatterplot(data=df, x=STUDY_HOURS, y=EXAM_SCORE, ax=ax) # Figure-level (for complex layouts) -g = sns.relplot(data=df, x='col_x', y='col_y') +g = sns.relplot(data=df, x=STUDY_HOURS, y=EXAM_SCORE) fig = g.figure ``` +## Data block + +The file follows the six zones of `prompts/adaptable-code.md`. seaborn reads the tidy `df` directly: pass `data=df` and the column constants, never typed column names. It titles the axes and the legend from the column names, so the constants' values are what the reader sees; set a label explicitly only to change it. Give the palette the first n hues with n counted from the data, because seaborn warns when the palette outnumbers the hue levels: + +```python +# Plot: reads only df and the constants above +n_lines = df[PRODUCT_LINE].nunique() +sns.barplot(data=df, x=QUARTER, y=SALES, hue=PRODUCT_LINE, palette=IMPRINT_PALETTE[:n_lines], ax=ax) +``` + +A long table needs no reshaping. A `sns.load_dataset(...)` call belongs inside the block, with its columns renamed to the constants there. + ## Save ```python @@ -98,11 +110,13 @@ See `prompts/default-style-guide.md` "Proportional Sizing" for the review-step c ## API Compatibility (0.14+) ```python +n_groups = df[GROUP].nunique() + # WARNING: palette without hue -sns.boxplot(data=df, x='group', y='value', palette=IMPRINT_PALETTE[:N]) # Warning +sns.boxplot(data=df, x=GROUP, y=VALUE, palette=IMPRINT_PALETTE[:n_groups]) # Warning # CORRECT: hue with palette (the Imprint palette, see Colors below) -sns.boxplot(data=df, x='group', y='value', hue='group', palette=IMPRINT_PALETTE[:N], legend=False) +sns.boxplot(data=df, x=GROUP, y=VALUE, hue=GROUP, palette=IMPRINT_PALETTE[:n_groups], legend=False) ``` ## Colors @@ -117,10 +131,10 @@ ANYPLOT_AMBER = '#DDCC77' # warning / caution (outside the categorical pool) # Single-series color = IMPRINT_PALETTE[0] # '#009E73' -sns.scatterplot(data=df, x='x', y='y', color=color) +sns.scatterplot(data=df, x=STUDY_HOURS, y=EXAM_SCORE, color=color) -# Multi-series (hue) -sns.scatterplot(data=df, x='x', y='y', hue='category', palette=IMPRINT_PALETTE[:N]) +# Multi-series (hue): the first n hues, n counted from the data, never a literal slice +sns.scatterplot(data=df, x=STUDY_HOURS, y=EXAM_SCORE, hue=SCHOOL, palette=IMPRINT_PALETTE[: df[SCHOOL].nunique()]) # Set once globally for a whole figure sns.set_palette(IMPRINT_PALETTE) diff --git a/prompts/plot-generator.md b/prompts/plot-generator.md index d014980d015..7599c9d14d6 100644 --- a/prompts/plot-generator.md +++ b/prompts/plot-generator.md @@ -98,7 +98,7 @@ charts rendered by different engines defeat the point. - `plots/{spec-id}/implementations/{language}/{this-library}{ext}` (if regenerating, same library only — `.py` for python, `.R` for ggplot2, `.jl` for makie, `.js` for the JavaScript libs, `.tsx` for muix) - `plots/{spec-id}/metadata/{language}/{this-library}.yaml` (its own previous review only) - `prompts/library/{this-library}.md` -- `prompts/plot-generator.md`, `prompts/quality-criteria.md`, `prompts/default-style-guide.md` +- `prompts/plot-generator.md`, `prompts/adaptable-code.md`, `prompts/quality-criteria.md`, `prompts/default-style-guide.md` **Forbidden:** - Reading another library's source file or `.yaml` under `plots/{spec-id}/implementations/` or `plots/{spec-id}/metadata/` — even "for reference" or "to stay consistent" @@ -117,7 +117,7 @@ implementation's own decision. ## Output -A simple script with the structure below. The example is Python; ggplot2 follows the same imports → data → plot → save shape — see `prompts/library/ggplot2.md` for the R-flavoured version and `prompts/library/makie.md` for the Julia-flavoured version. +A simple script in the six zones of `prompts/adaptable-code.md`: the head, an optional `# Constants` zone, the column constants, one `# Data` block that ends in `df`, the plot code below `# Plot`, and the save. Read that file on every generation: it is the convention that makes the code adaptable to other data, with the rules for the plot code and a full example. The example below is Python; R, Julia and JavaScript use the same zones in their own syntax, shown in the "Data block" section of `prompts/library/{library}.md`. ```python """ anyplot.ai @@ -127,30 +127,37 @@ Quality: pending | Created: 2025-12-21 """ import os + import matplotlib.pyplot as plt import numpy as np +import pandas as pd + # Theme tokens (see prompts/default-style-guide.md "Background" + "Theme-adaptive Chrome") THEME = os.getenv("ANYPLOT_THEME", "light") PAGE_BG = "#FAF8F1" if THEME == "light" else "#1A1A17" INK = "#1A1A17" if THEME == "light" else "#F0EFE8" INK_SOFT = "#4A4A44" if THEME == "light" else "#B8B7B0" -BRAND = "#009E73" # Imprint palette position 1 — ALWAYS first series +BRAND = "#009E73" # Imprint palette position 1 — a single-series plot uses it alone -# Data +# Columns: set these to your table's column names +STUDY_HOURS = "Study hours per day" # role: x +EXAM_SCORE = "Exam score (%)" # role: y + +# Data: replace this block with your own table, for example df = pd.read_csv("scores.csv") np.random.seed(42) -study_hours = np.random.normal(6, 2, 80) -exam_scores = study_hours * 8 + np.random.normal(0, 5, 80) + 30 +hours = np.random.normal(6, 2, 80) +df = pd.DataFrame({STUDY_HOURS: hours, EXAM_SCORE: hours * 8 + np.random.normal(0, 5, 80) + 30}) -# Plot — see default-style-guide.md "Visual Sizing Defaults" for the canvas + sizing values +# Plot: reads only df and the constants above — see default-style-guide.md "Visual Sizing Defaults" fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400, facecolor=PAGE_BG) ax.set_facecolor(PAGE_BG) -ax.scatter(study_hours, exam_scores, alpha=0.7, s=100, +ax.scatter(df[STUDY_HOURS], df[EXAM_SCORE], alpha=0.7, s=100, color=BRAND, edgecolors=PAGE_BG, linewidth=0.5) # Style — title kept compact because the mandated anyplot title is ~67 chars long -ax.set_xlabel('Study Hours per Day', fontsize=12, color=INK) -ax.set_ylabel('Exam Score (%)', fontsize=12, color=INK) +ax.set_xlabel(STUDY_HOURS, fontsize=12, color=INK) +ax.set_ylabel(EXAM_SCORE, fontsize=12, color=INK) ax.set_title('scatter-basic · python · matplotlib · anyplot.ai', fontsize=14, fontweight='medium', color=INK) ax.tick_params(axis='both', labelsize=10, colors=INK_SOFT) @@ -161,7 +168,7 @@ for s in ('left', 'bottom'): ax.yaxis.grid(True, alpha=0.15, linewidth=0.8, color=INK) plt.tight_layout() -plt.savefig(f'plot-{THEME}.png', dpi=400, bbox_inches='tight', facecolor=PAGE_BG) +plt.savefig(f'plot-{THEME}.png', dpi=400, facecolor=PAGE_BG) # never bbox_inches='tight': it trims the canvas ``` **The generated script must be run twice by the pipeline** — once with `ANYPLOT_THEME=light`, once with `ANYPLOT_THEME=dark` — producing `plot-light.png` and `plot-dark.png`. Interactive libraries additionally produce `plot-light.html` and `plot-dark.html`. @@ -219,25 +226,39 @@ The middot (·) separator is required. No color or style requirements - the AI d ### Structure -1. **Docstring** - 4 lines: anyplot.ai, spec-id:title, library+versions, quality+date +The six zones of `prompts/adaptable-code.md`, top to bottom: + +1. **Head** - the docstring, the imports (only what's needed), the theme tokens, the palette (all eight `IMPRINT_PALETTE` hues, or `BRAND` alone for a single series) and the library's setup calls + - Docstring: 4 lines: anyplot.ai, spec-id:title, library+versions, quality+date - New implementation: `Quality: pending | Created: {YYYY-MM-DD}` - After first review: `Quality: {score}/100 | Created: {YYYY-MM-DD}` - After update + review: `Quality: {score}/100 | Updated: {YYYY-MM-DD}` -2. **Imports** - Only what's needed -3. **Data** - Prepare/generate data (use spec example if provided, or create realistic sample) -4. **Plot** - Create figure and plot data -5. **Style** - Labels, title, grid, etc. -6. **Save** - Save as `plot-{THEME}.png` (plus `plot-{THEME}.html` for interactive libs). Never a bare `plot.png`. +2. **`# Constants`** (optional) - values that stay true for any data: thresholds, ordered scales, semantic color maps +3. **Column constants** - one per spec `## Data` role: `SALES = "Sales (thousand USD)" # role: value` +4. **`# Data` block** - prepare or generate the data (use the spec's example if provided, or create a realistic sample); it ends in one tidy DataFrame `df` +5. **`# Plot`** - create the figure, plot `df`, and style it (labels, title, grid); everything derived from the data is computed here +6. **Save** - Save as `plot-{THEME}.png` (plus `plot-{THEME}.html` for interactive libs). Never a bare `plot.png`, and never `bbox_inches='tight'`. + +R, Julia and JavaScript keep the same order in their own syntax; see the "Data block" section of `prompts/library/{library}.md`. ### Data Generation Strategy -Choose the appropriate data generation method based on the plot type: +Choose the appropriate data generation method based on the plot type. Whatever the method, it lives in the one `# Data` block (`prompts/adaptable-code.md`) and obeys these rules: + +- **Everything that invents the example sits in the block, and nothing else does:** the seed, random draws, literal lists, category names, the formulas behind synthetic data, a dataset loader call. +- **The block ends in one tidy DataFrame `df`, keyed by the column constants:** one row per mark or observation, one column per spec role. Three product lines are one `PRODUCT_LINE` column, not three lists. A matrix role makes `df` a labeled square DataFrame; a generated spec (a surface from a formula, a maze) holds named parameters instead. +- **The plot code reads only `df`:** a helper name bound in the block (`hours`, `base`) is never read below it. What the plot needs (pivots, sorted orders, positions, limits) it derives from `df`. +- **No plotting, styling or theme token inside the block.** **1. Synthetic Data with NumPy (default for most plots):** ```python +# Data: replace this block with your own table, for example df = pd.read_csv("salaries.csv") np.random.seed(42) # Always set seed when using random data for reproducibility! -x = np.random.normal(loc=50000, scale=15000, size=500) # Salaries -y = np.random.uniform(0, 100, size=120) # Test scores +experience = np.random.uniform(0, 30, size=500) +df = pd.DataFrame({ + EXPERIENCE: experience, + SALARY: 38000 + 1500 * experience + np.random.normal(0, 8000, size=500), +}) ``` - **Use for**: Basic plots, general examples, custom distributions - **Benefits**: Fast, flexible, reproducible, no external dependencies @@ -245,9 +266,15 @@ y = np.random.uniform(0, 100, size=120) # Test scores **2. Scikit-learn Datasets (for ML-related plots):** ```python -from sklearn.datasets import load_iris, make_classification -iris = load_iris() -X, y = iris.data, iris.target +from sklearn.datasets import load_iris + +# Data: replace this block with your own table +iris = load_iris(as_frame=True) +df = pd.DataFrame({ + PETAL_LENGTH: iris.data["petal length (cm)"], + PETAL_WIDTH: iris.data["petal width (cm)"], + SPECIES: iris.target_names[iris.target], +}) ``` - **Use for**: Classification plots, clustering, regression, ML metrics - **Available datasets**: `load_iris()`, `load_wine()`, `load_breast_cancer()`, `load_digits()` @@ -256,23 +283,28 @@ X, y = iris.data, iris.target **3. Seaborn Datasets (for realistic domain examples):** ```python import seaborn as sns -df = sns.load_dataset('tips') # Restaurant tipping data + +# Data: replace this block with your own table +df = sns.load_dataset("tips").rename(columns={"total_bill": TOTAL_BILL, "tip": TIP, "day": DAY}) ``` - **Use for**: When spec asks for realistic domain data or named datasets - **Available**: `'tips'`, `'titanic'`, `'iris'`, `'flights'`, `'planets'`, `'penguins'` - **Benefits**: Real-world patterns, clean data, good for demonstrations +- Rename the dataset's columns to the column constants inside the block, so the axis titles read as words with units (`"Total bill ($)"`), never as `total_bill`. **4. Domain-specific synthetic (for specialized plots):** ```python # Time series -dates = pd.date_range('2024-01-01', periods=100, freq='D') -values = np.cumsum(np.random.randn(100)) + 100 +dates = pd.date_range("2024-01-01", periods=100, freq="D") +df = pd.DataFrame({DATE: dates, PRICE: np.cumsum(np.random.randn(100)) + 100}) -# Correlation matrix +# Correlation matrix (a matrix role): df is a labeled square DataFrame np.random.seed(42) -corr_matrix = np.random.rand(5, 5) -corr_matrix = (corr_matrix + corr_matrix.T) / 2 # Symmetric -np.fill_diagonal(corr_matrix, 1.0) # Diagonal = 1 +metrics = ["Revenue", "Costs", "Margin", "Headcount", "Churn"] +values = np.random.rand(5, 5) +values = (values + values.T) / 2 # Symmetric +np.fill_diagonal(values, 1.0) # Diagonal = 1 +df = pd.DataFrame(values, index=metrics, columns=metrics) ``` **Guidelines:** @@ -280,7 +312,8 @@ np.fill_diagonal(corr_matrix, 1.0) # Diagonal = 1 - **Use sklearn/seaborn datasets** when you need realistic patterns or the spec mentions them - **Always set** `np.random.seed(42)` when using random data - **Make data realistic**: Use meaningful variable names, realistic ranges, proper units -- **No external files**: Never load CSV/JSON - generate everything in-memory +- **Name each column once:** the column constant's value is the column name, written as the axis title should read, units included (`"Sales (thousand USD)"`); the plot code labels axes and legends from the constants +- **No external files**: Never load CSV/JSON - generate everything in-memory. A CSV read appears only in the block's opening comment, as the hint for a reader who swaps in their own table ### Data Content Guidelines @@ -421,7 +454,7 @@ Code MUST NOT contain comments like: ### Variable Naming -Use descriptive, domain-appropriate names: +Use descriptive, domain-appropriate names. Column constants are upper-case domain names (`STUDY_HOURS`, `MONTH`, `SALES`), never role names such as `CATEGORY_COL`: a reader who plots by year instead of by month renames the constant. ```python # Good @@ -440,10 +473,14 @@ data1 = [1, 2, 3, 4] ### Section Comments -Short, clear section markers with blank line before each: +Short, clear section markers with blank line before each. `# Data` and `# Plot` are the block markers of `prompts/adaptable-code.md`: exactly one of each, at column 0, the word alone or followed by a colon and a hint. ```python -# Data +# Columns: set these to your table's column names +MONTH = "Month" # role: x +... + +# Data: replace this block with your own table np.random.seed(42) ... @@ -452,11 +489,11 @@ fig, ax = plt.subplots(figsize=(8, 4.5), dpi=400) ... # Style -ax.set_xlabel(...) +ax.set_xlabel(MONTH, ...) ... # Save -plt.savefig(f'plot-{THEME}.png', dpi=400, bbox_inches='tight') +plt.savefig(f'plot-{THEME}.png', dpi=400, facecolor=PAGE_BG) ``` ### Import Organization @@ -485,7 +522,7 @@ Blank line between groups. Only import what you use. - Break long calls across multiple lines: ```python -ax.scatter(study_hours, exam_scores, +ax.scatter(df[STUDY_HOURS], df[EXAM_SCORE], alpha=0.7, s=100, color=BRAND, edgecolors=PAGE_BG) ``` @@ -529,6 +566,7 @@ anyplot renders at **3200 × 1800 px** (16:9) or **2400 × 2400 px** (1:1) — l - **Annotation restraint (DEFAULT):** Do NOT add text annotations, callout boxes, arrows, or labeled data points unless the specification explicitly asks for them (e.g., spec-id contains "annotated"). Good storytelling comes from visual design — color contrast, size variation, strategic data choice — not text overlays. - **When annotations ARE appropriate:** Only when spec-id contains "annotated" or the spec explicitly describes annotations as a required feature. Even then, use sparingly. - **Respect the spec variant:** If the spec-id contains `basic`, storytelling comes from well-chosen data and clean design — NOT from adding annotations, trendlines, or extra visual elements. A basic scatter plot should remain a basic scatter plot. Extra encodings on a `-basic` spec (a color channel from a derived variable, per-group fits, reference lines, highlights, callouts, annotation layers) are scored as the wrong variant (SC-01) and earn no storytelling credit. +- **Callout targets come from a rule over the data:** when a callout or highlight is drawn at all (the restraint rules above decide that), choose its target with a rule over `df` — `idxmax`, the latest row, the smallest gap between the two leading groups, the first crossing of a `# Constants` threshold — never by typing a sample value (`categories.index("Q3")`, `peak_month = 7`). Its text is a computed f-string that claims only what the rule guarantees, and a conditional claim (a near-tie, a record) is drawn only when the data meets it. With other data the callout moves to the right place or disappears, instead of pointing at a value that is no longer there. See `prompts/adaptable-code.md`. ## Output Files @@ -538,7 +576,8 @@ anyplot renders at **3200 × 1800 px** (16:9) or **2400 × 2400 px** (1:1) — l THEME = os.getenv("ANYPLOT_THEME", "light") # already defined at top # matplotlib/seaborn/plotnine (static, PNG only) — figsize=(8, 4.5) dpi=400 → 3200×1800 -plt.savefig(f'plot-{THEME}.png', dpi=400, bbox_inches='tight', facecolor=PAGE_BG) +# (no bbox_inches='tight': it trims the canvas below the fixed size) +plt.savefig(f'plot-{THEME}.png', dpi=400, facecolor=PAGE_BG) # plotly (PNG + HTML) — width=800 height=450 scale=4 → 3200×1800 fig.write_image(f'plot-{THEME}.png', width=800, height=450, scale=4) @@ -570,5 +609,6 @@ After generating the code: - Are top/right spines removed? - Is the design polished beyond defaults? 3. **For interactive libraries**, also check `plot-light.html` and `plot-dark.html` render correctly. +4. **For Python**, run the adaptability self-check (`prompts/workflow-prompts/impl-generate-claude.md`, Step 3c): the static check of the six zones and a smoke run of the file on perturbed tables (renamed columns, more and fewer categories, more and fewer rows, scaled values, a missing value). If there are issues, fix them and re-run both themes until both plots look correct. diff --git a/prompts/workflow-prompts/impl-generate-claude.md b/prompts/workflow-prompts/impl-generate-claude.md index 79402633aaf..125007a6379 100644 --- a/prompts/workflow-prompts/impl-generate-claude.md +++ b/prompts/workflow-prompts/impl-generate-claude.md @@ -61,9 +61,10 @@ A post-render gate in `impl-review.yml` measures the saved PNG dimensions and re Read these files to understand the requirements: 1. `prompts/plot-generator.md` - Base generation rules -2. `prompts/default-style-guide.md` - **CRITICAL**: Imprint palette, continuous-data rules, theme-adaptive chrome tokens. Every new implementation must comply. Always refer to the palette as **Imprint** in code comments and metadata — never "anyplot palette". -3. `prompts/library/{LIBRARY}.md` - Library-specific rules + theme-adaptive chrome mapping for this library -4. `plots/{SPEC_ID}/specification.md` - What to visualize. Its closing `## What a good version looks like` section (when present) names what a good render of this plot type shows, and the review scores against it. Build toward its `A good version shows:` bullets (the review deducts when one is missing). Its `Expected, not a defect:` bullets are permissions, not targets: never shape the data to produce them (no clustering points to show overlap). Without the section, derive the same from Description, Data and Notes. +2. `prompts/adaptable-code.md` - The Data block convention: six zones, one `# Data` block that ends in `df`, columns named once as constants, plot code that derives everything from `df`. Every new implementation follows it, in every language. +3. `prompts/default-style-guide.md` - **CRITICAL**: Imprint palette, continuous-data rules, theme-adaptive chrome tokens. Every new implementation must comply. Always refer to the palette as **Imprint** in code comments and metadata — never "anyplot palette". +4. `prompts/library/{LIBRARY}.md` - Library-specific rules + theme-adaptive chrome mapping for this library, including its "Data block" idiom +5. `plots/{SPEC_ID}/specification.md` - What to visualize. Its closing `## What a good version looks like` section (when present) names what a good render of this plot type shows, and the review scores against it. Build toward its `A good version shows:` bullets (the review deducts when one is missing). Its `Expected, not a defect:` bullets are permissions, not targets: never shape the data to produce them (no clustering points to show overlap). Without the section, derive the same from Description, Data and Notes. ### If regenerating (`IS_REGENERATION=true`) — MANDATORY @@ -82,13 +83,15 @@ When regenerating an existing implementation, you MUST read these BEFORE writing - Whatever its tag, a weakness that asks for less of something an `Expected, not a defect:` bullet names is obsolete — e.g. "bubbles overlap in the dense cluster" on a bubble chart that already uses translucency and outlines: decline it as `W1 — obsolete (C2)`. - The checklist is context, not a list of orders: a ❌ item tells you where to look, and you act on it only when it is a defect in the same sense. - **Answer overlap weaknesses through data generation, marker size or alpha — never by moving marks off their data values.** No force/collision simulations, nudge or declutter passes, or offsets on data marks; moving labels is fine. The review deducts displaced marks (SC-03). Exempt, as in SC-03: jitter in categorical strip/swarm plots, layout-positioned types (networks, treemaps, word clouds, packed circles), and any jitter, dodge or offset the spec's Data or Notes ask for. -- **Keep the data scenario and the variant.** Same domain, same story, same encodings unless a weakness names the scenario itself as the problem or a change request (below) asks for a different one. On a `-basic` spec, add no new encodings or elements (derived color channels, trend or reference lines, highlights, callouts, annotation layers) — the review scores them as the wrong variant. -- **Don't add code for changes that don't show.** Every edit should be visible in the render or fix a named code-quality item; a longer file with an unchanged picture is not an improvement. A `CQ-04` weakness names its replacement: apply exactly that replacement, keep the data and the render the same, and change nothing else for it. Without a CQ-04 weakness, leave computations as they are on a regeneration. +- **Keep the data scenario and the variant.** Same domain, same story, same encodings unless a weakness names the scenario itself as the problem or a change request (below) asks for a different one. On a `-basic` spec, add no new encodings or elements (derived color channels, trend or reference lines, highlights, callouts, annotation layers) — the review scores them as the wrong variant. For Python (`LANGUAGE=python`): re-expressing the same values as one `df` with column constants (the Data block of `prompts/adaptable-code.md`) keeps the scenario; it is not a new one. +- **Don't add code for changes that don't show.** Every edit should be visible in the render or fix a named code-quality item; a longer file with an unchanged picture is not an improvement. A `CQ-04` weakness names its replacement: apply exactly that replacement, keep the data and the render the same, and change nothing else for it. Without a CQ-04 weakness, leave computations as they are on a regeneration. For Python (`LANGUAGE=python`): a `CQ-01 (code)` line or a failed perturbation from the Step 3c self-check is a named code-quality item, like a CQ-04 weakness. Converting the file to the Data block convention is the one change you make although the picture stays the same, and it may move computations below the block (widths, limits and positions derived from `df`); see "Converting a Python file" below. - **Canvas size: the Step 0 contract is non-negotiable on regen.** The previous file's `figsize` / `dpi` / `width` / `height` / `scale_factor` values are **historical**, never current — overwrite them to the canonical pair from `prompts/library/{LIBRARY}.md` as your *first* edit, before touching anything else. The post-render gate checks this and re-triggers repair on drift; do not let that fire. - **Base style wins on everything else.** If anything in `prompts/default-style-guide.md` or `prompts/library/{LIBRARY}.md` differs from the previous implementation, update the previous code to match. This includes **font sizes** (title, axis labels, tick labels, legend), **marker and line sizes**, **palette** (Imprint palette positions), **theme tokens** (background, INK, INK_SOFT, ELEVATED_BG, GRID), and **chrome** (spines, gridlines, legend frame). The previous review may not have flagged the old values because they were valid at the time — that does NOT make them current. Always re-read the library prompt's "Sizing" section and the style guide's "Visual Sizing Defaults" table on every regen and align. Also normalise any stale "anyplot palette" wording in the previous code's comments to "Imprint palette". -- Do NOT discard working structure / data generation / layout choices that the previous review did not flag. -- No defect listed and nothing in Step 0 or the base style to align → a small edit, and a kept regeneration, is the expected outcome. Do not invent changes to have something to show. +- Do NOT discard working structure / data generation / layout choices that the previous review did not flag. For Python (`LANGUAGE=python`): the data generation keeps its values, not its form — the same numbers, names and order move into the one `# Data` block and end in `df`. +- No defect listed and nothing in Step 0 or the base style to align → a small edit, and a kept regeneration, is the expected outcome. Do not invent changes to have something to show. For Python (`LANGUAGE=python`): a previous file that fails the adaptability check has a listed defect, so convert it even when nothing else needs to change. - Your deliverable is a refined version of the previous file, not a fresh rewrite from the spec. +- **Converting a Python file (`LANGUAGE=python`).** Before you edit, run the static check on the previous file to see what it lacks: `python -m core.adaptability check plots/{SPEC_ID}/implementations/python/{LIBRARY}.py`, from the repository root with `.venv` activated. It prints the record and the `CQ-01 (code)` lines and, unlike Step 3c, writes no log entry. Convert toward an identical picture: keep every value, label, order, color assignment and size, and replace each literal tied to the sample with an expression over `df` that reproduces it for the sample as closely as you can — `0.75 / len(table.columns)` for a bar width of 0.25 with three groups, a factor on the data's maximum for a typed axis limit (`df[SALES].max() * 1.13` where the old file typed 480 over a maximum of 425). A pure conversion merges when the new render differs from the previous one in at most 0.5 % of its pixels per theme at the same canvas size; a conversion that moves things is an ordinary regeneration and merges only with a visible improvement. When you also fix a defect, that fix is the only visible change. The optional `pair` command in Step 3c measures the difference. +- **R, Julia and JavaScript (`LANGUAGE` other than `python`):** the Data block convention applies to new files. On a regeneration it is no reason to rewrite working data code; the mindset above applies unchanged. ### Library Independence — DO NOT read sibling implementations @@ -154,7 +157,7 @@ plots/{SPEC_ID}/implementations/{LANGUAGE}/{LIBRARY}{EXT} where `{EXT}` is `.py` for `python`, `.R` for `r`, `.jl` for `julia`, and `.js` for `javascript`. The script MUST: -- Follow the KISS structure: imports → data → plot → save +- Follow the six zones of `prompts/adaptable-code.md`: head (imports, theme tokens, palette, setup) → optional `# Constants` → column constants with `# role:` comments → one `# Data` block that ends in `df` (`const data` in JavaScript) → plot code below `# Plot` → save - Read `ANYPLOT_THEME` (`"light"` or `"dark"`, default `"light"`) and render accordingly. The same single script file handles both themes. - Python: `os.getenv("ANYPLOT_THEME", "light")` - R: `Sys.getenv("ANYPLOT_THEME", "light")` @@ -235,6 +238,34 @@ sys.exit(0 if ok else 1) If this exits non-zero, you have **not** satisfied the Step 0 contract. Re-read `prompts/library/{LIBRARY}.md` "Canvas — hard rule" and apply the exact code there; common causes are listed in that file (e.g. matplotlib/seaborn `bbox_inches="tight"` shaves ~40 px; bokeh's default toolbar leaves ~140 px; altair's vl-convert pads outside `width`/`height`). Re-render and re-check until OK. Do not skip this and rely on the post-render gate to catch it — repair cycles cost compute. +### Step 3c: Adaptability self-check (Python only) + +**Python (`LANGUAGE=python`) only; R, Julia and JavaScript skip this step.** After Step 3b passes, check that the file follows the Data block convention (`prompts/adaptable-code.md`) and still renders on other data. One command runs both checks: the static check of the six zones, and a smoke run that executes the file on perturbed copies of its own `df` (renamed columns, one with spaces, a unit and a dot; one more and one fewer category; a single group where the spec allows one; fewer and more rows; scaled values; a missing value). Run it from the repository root, and give the Bash call a 10-minute timeout, because bokeh and plotly start Chrome for every render: + +```bash +cd "$GITHUB_WORKSPACE" +source .venv/bin/activate +python -m core.smoke_run selfcheck plots/{SPEC_ID}/implementations/python/{LIBRARY}.py --log /tmp/anyplot-selfcheck.jsonl +``` + +It prints one `CQ-01 (code): line N … → …. Likely cause: ….` line per failing rule class, then one line per perturbation, and exits 0 when both checks are clean. A map spec, and a generated spec with nothing to perturb, count as clean. + +- **Fix every line and every failed perturbation that is not tagged `(advisory)`,** then re-render both themes (Step 3), re-run Step 3b, and run this step again. A `CQ-01 (code)` line names the line of your file, what it found, the target and the likely cause; a failed perturbation names the error and the line where it was raised. Typical fixes: a literal tied to the sample becomes an expression over `df`, a palette sized to the sample becomes the full eight-entry list, a typed label becomes a column constant, a zero fill becomes a NaN-aware reduction. +- **Lines and perturbations tagged `(advisory)` never fail the check.** Fix one when the fix is small and leaves the render unchanged; otherwise leave it. +- **At most three attempts in total,** the first run included. Adaptability never blocks the implementation: if the check still fails after the third attempt, keep the best version that renders correctly in both themes and passes Step 3b, and go on to Step 4. Never trade a defect in the render for a clean check. +- **Each run appends one line to `/tmp/anyplot-selfcheck.jsonl`,** which the workflow stages; never delete or edit it. If the command itself cannot run (exit 2, or a runner error you cannot fix in your file), say so in the commit body and go on to Step 4. + +Regeneration only, optional: to see whether your file still renders the same picture as the previous version, render both in one environment and compare them: + +```bash +mkdir -p /tmp/anyplot-prev-src +git show origin/main:plots/{SPEC_ID}/implementations/python/{LIBRARY}.py > /tmp/anyplot-prev-src/{LIBRARY}.py +python -m core.smoke_run pair --new plots/{SPEC_ID}/implementations/python/{LIBRARY}.py \ + --prev /tmp/anyplot-prev-src/{LIBRARY}.py --out-dir /tmp/anyplot-pair +``` + +It exits 0 when both themes stay within 0.5 % of differing pixels at the same canvas size (`identical`), 1 when they differ (it names the theme and the fraction), and 2 when a render fails. It is a measurement for you, never a gate here: a regeneration that fixes a visible defect differs by design. + ## Step 4: Visual self-check (BOTH renders) Look at `plot-light.png` AND `plot-dark.png`: @@ -290,6 +321,13 @@ git push -u origin implementation/{SPEC_ID}/{LIBRARY} Commit only the implementation file, and use only the packages the runner already has: never run `Pkg.add`, `uv add`, `pip install` or `npm install`. The workflow fails the run when your commit changes `Manifest.toml`, `Project.toml`, `uv.lock`, `pyproject.toml`, `package.json`, `package-lock.json` or `.python-version`, because a re-resolved lock file moves the library version for every implementation of the language. +For Python (`LANGUAGE=python`), the commit body always carries the result of the last Step 3c run, first generations included: `pass`, or `fail` with the rule classes of the remaining non-advisory lines and the names of the remaining failed perturbations, and the attempt number: + +```bash +git commit -m "feat({LIBRARY}): implement {SPEC_ID}" -m "Adaptability self-check: pass, attempt 1" +# or, after three attempts: -m "Adaptability self-check: fail (palette, more_categories), attempt 3" +``` + If `IS_REGENERATION=true`, use an expanded commit body that names the weaknesses you addressed and the ones you declined, by W id with the tag the previous review gave them, each declined one with its reason. Example: ``` @@ -305,6 +343,8 @@ Declined: - W2 — suggestion, not taken: "a focal highlight on the densest cluster" - W5 — asks for something the spec does not require: "add a trend line" - W6 — names a call the runtime does not provide: "StatsBase.autocor" + +Adaptability self-check: pass, attempt 2 ``` Pass the multi-line message via `-F -` or a heredoc so git preserves the body. @@ -318,6 +358,7 @@ Before finishing, confirm: 4. ✅ Changes were committed and pushed 5. ✅ If regenerating: `/tmp/anyplot-prev-review.md` and the previous source file were read; every defect was fixed, every `Suggestion:` line was declined as "suggestion, not taken", every obsolete weakness was declined with its C id, and each older note was acted on only when it names something visibly wrong under the current criteria (declined ones named with a reason in the commit body) 6. ✅ Every data mark sits at its data value — overlap was handled through data generation, marker size or alpha, not by moving marks (SC-03 exemptions only: categorical strip/swarm jitter, layout-positioned types, jitter/dodge/offset the spec asks for) -7. ✅ If regenerating: same data scenario and variant as before (unless a weakness or change request asked otherwise), no new encodings on a `-basic` spec, and no added code whose effect does not show in the render +7. ✅ If regenerating: same data scenario and variant as before (unless a weakness or change request asked otherwise), no new encodings on a `-basic` spec, and no added code whose effect does not show in the render. For Python (`LANGUAGE=python`), converting the file to the Data block convention is the exception: it may add code while the render stays identical +8. ✅ For Python (`LANGUAGE=python`): Step 3c ran (at most three attempts) and the commit body carries its `Adaptability self-check:` line; for every language, the file follows the zones of `prompts/adaptable-code.md` If any of these failed, DO NOT report success. diff --git a/prompts/workflow-prompts/impl-repair-claude.md b/prompts/workflow-prompts/impl-repair-claude.md index 6025a718996..9f0530dc76c 100644 --- a/prompts/workflow-prompts/impl-repair-claude.md +++ b/prompts/workflow-prompts/impl-repair-claude.md @@ -26,11 +26,11 @@ Read both sources to understand what needs to be fixed: Read the closing `## What a good version looks like` section of `plots/{SPEC_ID}/specification.md` (without one, use Description, Data and Notes) and decide per weakness: -- **Fix the defects that are real per the section.** A defect line names the criterion it violates, the render and the observed value (`VQ-02 (light): … → …. Likely cause: ….`). A weakness that asks for something an `Expected, not a defect:` bullet of the section names — e.g. "bubbles overlap in the dense cluster" on a bubble chart that already uses translucency and outlines — is obsolete: decline it as `W1 — obsolete (C2)` or, without W ids, with the bullet's text. The section never waives chrome rules (theme readability, canvas, palette, text legibility, auto-reject checks); those weaknesses are always real. +- **Fix the defects that are real per the section.** A defect line names the criterion it violates, the render and the observed value (`VQ-02 (light): … → …. Likely cause: ….`). A weakness that asks for something an `Expected, not a defect:` bullet of the section names — e.g. "bubbles overlap in the dense cluster" on a bubble chart that already uses translucency and outlines — is obsolete: decline it as `W1 — obsolete (C2)` or, without W ids, with the bullet's text. The section never waives chrome rules (theme readability, canvas, palette, text legibility, auto-reject checks); those weaknesses are always real. So are the lines of the adaptability self-check for Python: a `CQ-01 (code)` line or a failed perturbation is a real defect, like a canvas line. - **Never act on a `Suggestion:` line.** It names no violated rule and costs no points; decline it as "suggestion, not taken". A line in neither format comes from an older review: act on it only when it names something visibly wrong under the current criteria. - **The checklist is context, not a list of orders.** A deducted item tells you where to look; the defect lines say what to fix. - **Keep the data scenario and the variant.** Same domain, same story, same encodings unless a weakness names the scenario itself as the problem. On a `-basic` spec, add no new encodings or elements (derived color channels, trend or reference lines, highlights, callouts, annotation layers). -- **Don't add code for changes that don't show.** Every edit should be visible in the render or fix a named code-quality item. +- **Don't add code for changes that don't show.** Every edit should be visible in the render or fix a named code-quality item. For Python (`LANGUAGE=python`): a line of the adaptability self-check is such an item; fix it and keep the render the same. - **Never move marks off their data values** to fix overlap — see the overlap line under "Visual-sizing fixes" below for the endorsed fixes and the SC-03 exemptions. - **`Expected, not a defect:` bullets are permissions, not targets.** Never shape the data to produce them, and never treat making one disappear as a fix: a weakness that only asks for less of a permitted thing (less overlap, fewer crossings) is declined, as the first bullet says. @@ -39,6 +39,7 @@ Read the closing `## What a good version looks like` section of `plots/{SPEC_ID} 1. `prompts/library/{LIBRARY}.md` - Library-specific rules + theme-adaptive chrome mapping 2. `prompts/default-style-guide.md` - Canonical Imprint palette + theme tokens (re-read if VQ-07 or VQ-04 failed), plus "Visual Sizing Defaults" and "Proportional Sizing" sections (re-read if VQ-01 / VQ-02 / VQ-05 failed) 3. `plots/{SPEC_ID}/specification.md` - The specification +4. `prompts/adaptable-code.md` - The Data block convention the file follows: one `# Data` block that ends in `df`, columns named once as constants, plot code that derives everything from `df`. Keep it intact while you repair, and read it before you fix a line of the Step 5b self-check (Python) **Visual-sizing fixes (when VQ-01 / VQ-02 / VQ-05 failed):** - Title too big / overflows → reduce fontsize or shorten title text @@ -85,7 +86,7 @@ Based on the AI feedback, fix the defects you kept in "Which weaknesses to fix" - Code quality issues - Spec compliance issues -Keep the data scenario and variant, add no code without a visible effect, and never move marks off their data values beyond the SC-03 exemptions. +Keep the data scenario and variant, add no code without a visible effect (for Python, a self-check line from Step 5b is the exception), and never move marks off their data values beyond the SC-03 exemptions. Keep the file's Data block intact: a fix below the block reads `df` and the column constants, never a typed sample value. ## Step 5: Test the fix (BOTH themes) @@ -126,6 +127,21 @@ ANYPLOT_THEME=dark node "$GITHUB_WORKSPACE/automation/js-render/render.mjs" {LI Both renders must succeed. +### Step 5b: Adaptability self-check (Python only) + +**Python (`LANGUAGE=python`) only; R, Julia and JavaScript skip this step.** After both renders succeed, check that the repaired file still follows the Data block convention (`prompts/adaptable-code.md`) and still renders on other data. Run it from the repository root, and give the Bash call a 10-minute timeout, because bokeh and plotly start Chrome for every render: + +```bash +cd "$GITHUB_WORKSPACE" +source .venv/bin/activate +python -m core.smoke_run selfcheck plots/{SPEC_ID}/implementations/python/{LIBRARY}.py --log /tmp/anyplot-selfcheck.jsonl +``` + +It prints one `CQ-01 (code): …` line per failing rule class, then one line per perturbation (renamed columns, more and fewer categories, more and fewer rows, scaled values, a missing value), and exits 0 when both checks are clean. + +- Fix every line and every failed perturbation that is not tagged `(advisory)`, then re-render both themes (Step 5) and run this step again. Lines tagged `(advisory)` never fail the check. +- At most three attempts in total, the first run included. The self-check never blocks the repair: if it still fails after the third attempt, keep the version that best fixes the review's defects and renders correctly in both themes, and go on to Step 6. Never trade a defect in the render for a clean check. + ## Step 6: Visual self-check View `plot-light.png` AND `plot-dark.png`. Verify the failed criteria are now fixed in both renders — and confirm the data colors are identical across themes (only chrome should flip). @@ -160,6 +176,8 @@ git push origin {BRANCH} Commit only the implementation file, and use only the packages the runner already has: never run `Pkg.add`, `uv add`, `pip install` or `npm install`. `impl-merge.yml` refuses to merge a PR that changes `Manifest.toml`, `Project.toml`, `uv.lock`, `pyproject.toml`, `package.json`, `package-lock.json` or `.python-version`. +For Python (`LANGUAGE=python`), add the result of the last Step 5b run to the commit body, in the form Step 3c of `impl-generate-claude.md` uses: `Adaptability self-check: pass, attempt 1`, or `fail` with the remaining rule classes and failed perturbations, such as `Adaptability self-check: fail (palette, more_categories), attempt 3`. + If you declined any weakness in Step 1, add a `Declined:` list to the commit body, one line per weakness with its reason, for example: ``` diff --git a/tests/unit/automation/scripts/test_adapt_record.py b/tests/unit/automation/scripts/test_adapt_record.py new file mode 100644 index 00000000000..172f5b84286 --- /dev/null +++ b/tests/unit/automation/scripts/test_adapt_record.py @@ -0,0 +1,549 @@ +"""automation/scripts/adapt_record.py: impl-review's adapt records, one per side of a Python pair. + +The record format is the interface to the regen gate (`regen_gate.py adapt-route` and +`decide --adapt-new/--adapt-prev`), so the tests pin the derived status table, the +fields the gate reads, the notice line, and the two copies the module shares with +other code: `code_sha256` with `core.smoke_run` and `RULE_ORDER` with `core.adaptability`. +""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from automation.scripts import adapt_record as ar +from automation.scripts import regen_gate +from core import adaptability, smoke_run + + +REPO_ROOT = Path(__file__).parent.parent.parent.parent.parent +SCRIPT = REPO_ROOT / "automation" / "scripts" / "adapt_record.py" +MIRROR = REPO_ROOT / "automation" / "scripts" / "adaptability_check.py" +FIXTURE = REPO_ROOT / "tests" / "unit" / "core" / "fixtures" / "adaptability" / "bar_grouped_matplotlib.py" +CATALOGUE = REPO_ROOT / "plots" / "bar-grouped" / "implementations" / "python" / "matplotlib.py" +SPEC = REPO_ROOT / "plots" / "bar-grouped" / "specification.md" +SOURCE = '"""anyplot.ai\nbar-grouped: Grouped Bar Chart\nQuality: pending | Created: 2026-10-23\n"""\n\nimport os\n' +STATIC_PASS = { + "version": 1, + "status": "pass", + "shape": "table", + "color_role": "group", + "block": {"start": 31, "end": 40}, + "constants": {"category": "QUARTER", "group": "PRODUCT_LINE", "value": "SALES"}, + "findings": [{"rule": 4, "class": "data-coordinate", "line": 77, "observed": "o", "cause": "c", "advisory": True}], + "lines": ["CQ-01 (code): (advisory) line 77 ..."], +} + + +def _reference_sha(text: str) -> str: + """Contract section 1.1, verbatim: the rule core.smoke_run implements.""" + body = text.lstrip() + if body.startswith('"""'): + end = body.find('"""', 3) + if end != -1: + body = body[end + 3 :] + return hashlib.sha256(body.strip().encode("utf-8")).hexdigest() + + +RESULT_PASS = { + "name": "renamed", + "description": "d", + "advisory": False, + "status": "pass", + "rows": 12, + "theme": None, + "error": None, + "line": None, + "code": None, + "canvas": {}, + "stale_labels": [], + "seconds": 4.2, +} +RESULT_ADVISORY_FAIL = { + **RESULT_PASS, + "name": "missing_nullable", + "advisory": True, + "status": "fail", + "theme": "light", + "error": "TypeError: boolean value of NA is ambiguous", + "line": 58, + "code": "if gaps[tie] <= NEAR_TIE * top:", + "canvas": None, + "seconds": 1.9, +} + + +def _smoke(source_text: str, **overrides: Any) -> dict[str, Any]: + data: dict[str, Any] = { + "v": 1, + "tool": "core.smoke_run run", + "code_sha256": ar.code_sha256(source_text), + "spec": "bar-grouped", + "library": "matplotlib", + "status": "pass", + "why": "-", + "results": [RESULT_PASS, RESULT_ADVISORY_FAIL], + "failed": [], + "advisory_failed": ["missing_nullable"], + } + data.update(overrides) + return data + + +def _write(path: Path, data: Any) -> Path: + path.write_text(data if isinstance(data, str) else json.dumps(data), encoding="utf-8") + return path + + +def _record( + tmp_path: Path, + *, + side: str = "new", + static: Any = STATIC_PASS, + exit_code: int = 0, + smoke: Any = None, + selfcheck: str | None = None, + source: str | None = SOURCE, +) -> dict[str, Any]: + src = tmp_path / "matplotlib.py" + if source is not None: + src.write_text(source, encoding="utf-8") + return ar.build_record( + side=side, + spec="bar-grouped", + lib="matplotlib", + language="python", + source=src, + static=_write(tmp_path / "static.json", static) if static is not None else tmp_path / "absent.json", + static_exit=exit_code, + smoke=_write(tmp_path / "smoke.json", smoke) if smoke is not None else tmp_path / "absent-smoke.json", + selfcheck=_write(tmp_path / "selfcheck.jsonl", selfcheck) if selfcheck is not None else None, + ) + + +class TestSharedCopies: + @pytest.mark.parametrize( + "text", + [ + SOURCE, + '\n\n """anyplot.ai\nQuality: 91/100 | Updated: 2026-10-24\n"""\nx = 1\n', + "x = 1\n", + '"""unterminated header\nx = 1\n', + "", + '"""a"""', + ], + ) + def test_code_sha256_is_the_contracts_rule(self, text): + assert ar.code_sha256(text) == _reference_sha(text) + + def test_the_header_does_not_count(self): + reviewed = SOURCE.replace("Quality: pending | Created: 2026-10-23", "Quality: 94/100 | Updated: 2026-10-24") + assert ar.code_sha256(reviewed) == ar.code_sha256(SOURCE) + assert ar.code_sha256(SOURCE + "x = 1\n") != ar.code_sha256(SOURCE) + + def test_code_sha256_matches_core_smoke_run(self): + """The staged smoke runs carry core.smoke_run's hash; a drift would make every one stale.""" + smoke_run = pytest.importorskip("core.smoke_run", reason="core.smoke_run arrives with PR 2 part B") + for text in (SOURCE, CATALOGUE.read_text(encoding="utf-8"), "x = 1\n", ""): + assert ar.code_sha256(text) == smoke_run.code_sha256(text) + + def test_rule_order_is_the_checkers(self): + assert ar.RULE_ORDER == adaptability.RULE_ORDER + + def test_selfcheck_skip_prefixes_are_core_smoke_runs(self): + """The first-attempt pass rate agrees with selfcheck's exit code.""" + assert ar.SELFCHECK_SKIP_PREFIXES == smoke_run.SELFCHECK_SKIP_PREFIXES + + def test_the_gates_derivation_is_derive_status(self): + """The gate rejects a record whose status does not follow from its blocks, by this very table.""" + assert ar.SMOKE_NOT_APPLICABLE == regen_gate.ADAPT_SMOKE_NOT_APPLICABLE + statics = (*ar.STATIC_STATUSES, "error", "weird") + smokes = (*ar.STAGED_SMOKE_STATUSES, "missing", "stale", "weird") + whys = ("-", "shape:generated", "shape:network", "shape:excluded", "no_perturbations:table", "runner_error") + for static in statics: + for smoke in smokes: + for why in whys: + expected = ar.derive_status(static, smoke, why)[0] + assert regen_gate.derived_adapt_status(static, smoke, why) == expected, (static, smoke, why) + + +class TestDerivedStatus: + @pytest.mark.parametrize( + ("static", "smoke", "why", "expected"), + [ + ("error", "pass", "-", ("unknown", "static_error")), + ("excluded", "missing", "-", ("excluded", "excluded")), + ("fail", "pass", "-", ("fail", "static_fail")), + ("fail", "missing", "-", ("fail", "static_fail")), + ("pass", "missing", "-", ("unknown", "smoke_missing")), + ("pass", "stale", "-", ("unknown", "smoke_stale")), + ("pass", "fail", "perturbation_failed", ("fail", "smoke_fail")), + ("pass", "skipped", "shape:generated", ("pass", "smoke_na")), + ("pass", "skipped", "shape:network", ("pass", "smoke_na")), + ("pass", "skipped", "shape:excluded", ("unknown", "smoke_skipped")), + ("pass", "skipped", "runner_error", ("unknown", "smoke_skipped")), + ("pass", "skipped", "no_block", ("unknown", "smoke_skipped")), + # The sample gave nothing to perturb: the smoke run checked nothing. + ("pass", "skipped", "no_perturbations:table", ("unknown", "smoke_skipped")), + ("pass", "pass", "-", ("pass", "-")), + ], + ) + def test_first_matching_rule_wins(self, static, smoke, why, expected): + assert ar.derive_status(static, smoke, why) == expected + + +class TestStatic: + def test_the_mirrors_record_verbatim_plus_exit(self, tmp_path): + record = _record(tmp_path, smoke=_smoke(SOURCE)) + assert record["static"] == {**STATIC_PASS, "exit": 0} + assert record["checker_version"] == 1 + + @pytest.mark.parametrize( + ("static", "exit_code"), + [ + (None, 2), # unreadable input: the mirror writes nothing + (None, 1), # a crash before --out + (None, 0), + ("{not json", 0), + ({"status": "maybe"}, 0), + (["pass"], 0), + (STATIC_PASS, 2), # a file the mirror did not write this time + ], + ) + def test_anything_else_is_an_error(self, tmp_path, static, exit_code): + record = _record(tmp_path, static=static, exit_code=exit_code, smoke=_smoke(SOURCE)) + assert record["static"] == {"status": "error", "exit": exit_code} + assert (record["status"], record["why"]) == ("unknown", "static_error") + assert record["checker_version"] is None + + +class TestSmoke: + def test_a_fresh_run_is_trimmed(self, tmp_path): + smoke = _record(tmp_path, smoke=_smoke(SOURCE))["smoke"] + assert smoke == { + "status": "pass", + "why": "-", + "fresh": True, + "failed": [], + "advisory_failed": ["missing_nullable"], + "results": [ + {"name": "renamed", "status": "pass", "advisory": False, "line": None, "error": None}, + { + "name": "missing_nullable", + "status": "fail", + "advisory": True, + "line": 58, + "error": "TypeError: boolean value of NA is ambiguous", + }, + ], + } + + @pytest.mark.parametrize("smoke", [None, "{not json", ["pass"], {"status": "error"}, {"code_sha256": "x"}]) + def test_no_usable_file_is_missing(self, tmp_path, smoke): + record = _record(tmp_path, smoke=smoke) + assert record["smoke"]["status"] == "missing" + assert record["smoke"]["fresh"] is None + assert (record["status"], record["why"]) == ("unknown", "smoke_missing") + + @pytest.mark.parametrize("sha", ["0" * 64, None, 7]) + def test_a_run_on_other_code_is_stale(self, tmp_path, sha): + record = _record(tmp_path, smoke=_smoke(SOURCE, code_sha256=sha, status="fail", failed=["renamed"])) + assert record["smoke"] == { + "status": "stale", + "why": "-", + "fresh": False, + "failed": [], + "advisory_failed": [], + "results": [], + } + assert (record["status"], record["why"]) == ("unknown", "smoke_stale") + + def test_an_unreadable_source_makes_any_run_stale(self, tmp_path): + record = _record(tmp_path, smoke=_smoke(SOURCE), source=None) + assert record["smoke"]["status"] == "stale" + assert record["lines"] is None and record["code_sha256"] is None + + def test_the_header_rewrite_after_the_smoke_run_is_not_stale(self, tmp_path): + reviewed = SOURCE.replace("Quality: pending", "Quality: 91/100") + record = _record(tmp_path, smoke=_smoke(SOURCE), source=reviewed) + assert record["smoke"]["fresh"] is True + + def test_untrusted_values_are_reduced_to_tokens(self, tmp_path): + """The generation runner had a model shell: nothing from the file reaches a notice unfiltered.""" + evil = "x\n::error::forged" + smoke = _smoke( + SOURCE, + status="fail", + why=evil, + failed=["renamed", evil, 3, "more_categories"], + advisory_failed=evil, + results=[{"name": evil, "status": "fail", "advisory": "yes", "line": True, "error": "a\n" * 400}, "junk"], + ) + record = _record(tmp_path, smoke=smoke) + assert record["smoke"]["why"] == "-" + assert record["smoke"]["failed"] == ["renamed", "more_categories"] + assert record["smoke"]["advisory_failed"] == [] + (result,) = record["smoke"]["results"] + assert result["name"] == "-" and result["advisory"] is False and result["line"] is None + assert "\n" not in result["error"] and len(result["error"]) == ar.MAX_ERROR_CHARS + notice = ar.notice_line(record) + assert "\n" not in notice and "forged" not in notice + + +class TestSelfcheck: + def test_attempts_and_the_first_outcome(self, tmp_path): + log = "\n".join( + json.dumps(line) + for line in ( + {"attempt": 1, "static": "fail", "classes": ["palette"], "smoke": "pass"}, + {"attempt": 2, "static": "pass", "classes": [], "smoke": "pass"}, + ) + ) + assert _record(tmp_path, smoke=_smoke(SOURCE), selfcheck=log + "\n")["selfcheck"] == { + "attempts": 2, + "first": "fail", + } + + @pytest.mark.parametrize( + ("static", "smoke", "first"), + [("pass", "pass", "pass"), ("excluded", "skipped", "pass"), ("pass", "fail", "fail"), ("fail", "pass", "fail")], + ) + def test_a_first_attempt_passes_like_selfchecks_exit_0(self, tmp_path, static, smoke, first): + log = json.dumps({"attempt": 1, "static": static, "smoke": smoke}) + assert _record(tmp_path, selfcheck=log)["selfcheck"] == {"attempts": 1, "first": first} + + @pytest.mark.parametrize( + ("smoke", "smoke_why", "first"), + [ + ("skipped", "shape:generated", "pass"), + ("skipped", "no_perturbations:table", "pass"), # selfcheck exits 0 on it too + ("skipped", "runner_error", "fail"), + ("skipped", "no_block", "fail"), + ("pass", "-", "pass"), + ], + ) + def test_a_skip_passes_only_for_the_shape(self, tmp_path, smoke, smoke_why, first): + log = json.dumps({"attempt": 1, "static": "pass", "smoke": smoke, "smoke_why": smoke_why}) + assert _record(tmp_path, selfcheck=log)["selfcheck"] == {"attempts": 1, "first": first} + + @pytest.mark.parametrize("log", [None, "", "garbage\n[1]\n"]) + def test_no_log_is_null(self, tmp_path, log): + assert _record(tmp_path, selfcheck=log)["selfcheck"] is None + + def test_garbage_lines_are_not_attempts(self, tmp_path): + log = 'oops\n{"static": "pass", "smoke": "pass"}\n\n' + assert _record(tmp_path, selfcheck=log)["selfcheck"] == {"attempts": 1, "first": "pass"} + + def test_the_prev_side_has_no_selfcheck(self, tmp_path): + record = _record(tmp_path, side="prev", selfcheck='{"static": "pass", "smoke": "pass"}') + assert "selfcheck" not in record + + +class TestRecord: + def test_the_fields_the_gate_reads(self, tmp_path): + record = _record(tmp_path, smoke=_smoke(SOURCE)) + assert record["v"] == 1 + assert (record["side"], record["spec"], record["lib"], record["language"]) == ( + "new", + "bar-grouped", + "matplotlib", + "python", + ) + assert record["checker_version"] == 1 + assert record["status"] in ar.STATUSES + assert record["static"]["status"] == "pass" + assert record["smoke"]["status"] == "pass" + assert record["lines"] == len(SOURCE.splitlines()) + assert record["code_sha256"] == ar.code_sha256(SOURCE) + assert list(record) == [ + "v", + "side", + "spec", + "lib", + "language", + "checker_version", + "status", + "why", + "lines", + "code_sha256", + "static", + "smoke", + "selfcheck", + ] + json.dumps(record) # serializable as it stands + + def test_the_contract_example(self, tmp_path): + record = _record( + tmp_path, + smoke=_smoke(SOURCE), + selfcheck='{"static": "fail", "smoke": "pass"}\n{"static": "pass", "smoke": "pass"}\n', + ) + assert (record["status"], record["why"]) == ("pass", "-") + assert ar.notice_line(record) == ( + "::notice::adapt_check spec=bar-grouped lib=matplotlib side=new status=pass static=pass smoke=pass " + "rules=- advisory=data-coordinate perturbations=- advisory_perturbations=missing_nullable version=1 " + f"lines={len(SOURCE.splitlines())} selfcheck_attempts=2 selfcheck_first=fail why=-" + ) + + def test_a_failing_predecessor(self, tmp_path): + static = { + **STATIC_PASS, + "status": "fail", + "findings": [ + {"rule": 5, "class": "palette", "line": 9, "observed": "o", "cause": "c", "advisory": False}, + {"rule": 1, "class": "layout", "line": 40, "observed": "o", "cause": "c", "advisory": False}, + {"rule": 3, "class": "interface", "line": 47, "observed": "o", "cause": "c", "advisory": False}, + {"rule": 3, "class": "interface", "line": 52, "observed": "o", "cause": "c", "advisory": False}, + {"rule": 2, "class": "columns", "line": 25, "observed": "o", "cause": "c", "advisory": False}, + {"rule": 9, "class": "unknown", "line": 1, "observed": "o", "cause": "c", "advisory": False}, + ], + } + smoke = _smoke(SOURCE, status="skipped", why="static_failed", results=[], advisory_failed=[]) + record = _record(tmp_path, side="prev", static=static, exit_code=1, smoke=smoke) + assert (record["status"], record["why"]) == ("fail", "static_fail") + assert ar.notice_line(record) == ( + "::notice::adapt_check spec=bar-grouped lib=matplotlib side=prev status=fail static=fail smoke=skipped " + "rules=layout,columns,interface,palette advisory=- perturbations=- advisory_perturbations=- version=1 " + f"lines={len(SOURCE.splitlines())} why=static_fail" + ) + + @pytest.mark.parametrize("prev_smoke", [None, "stale"]) + def test_a_static_only_regression_reaches_the_gate(self, tmp_path, prev_smoke): + """A predecessor without a fresh smoke run that passes the static check, a new file that fails it. + + The predecessor's record is `unknown` and keeps `static.status: pass`; the gate reads that as a + certain regression without any smoke run. + """ + (tmp_path / "prev").mkdir() + (tmp_path / "new").mkdir() + smoke = _smoke("other code") if prev_smoke == "stale" else None + prev = _record(tmp_path / "prev", side="prev", smoke=smoke) + why = f"smoke_{prev_smoke or 'missing'}" + assert (prev["status"], prev["why"], prev["static"]["status"]) == ("unknown", why, "pass") + static_fail = {**STATIC_PASS, "status": "fail"} + new = _record(tmp_path / "new", static=static_fail, exit_code=1, smoke=_smoke(SOURCE)) + assert (new["status"], new["static"]["status"]) == ("fail", "fail") + inp = regen_gate.GateInput( + spec_id="bar-grouped", + score=None, + regen=None, + adapt=regen_gate.AdaptInput( + library="matplotlib", language="python", new=new, prev=prev, new_error=None, prev_error=None + ), + ) + assessed = regen_gate.adapt_assess(inp, None) + assert (assessed.regression, assessed.static_regression) == (True, True) + + def test_a_record_without_any_input(self, tmp_path): + record = _record(tmp_path, static=None, exit_code=2, source=None) + assert (record["status"], record["why"]) == ("unknown", "static_error") + assert ar.notice_line(record).endswith( + "static=error smoke=missing rules=- advisory=- perturbations=- advisory_perturbations=- version=n/a " + "lines=n/a selfcheck_attempts=n/a selfcheck_first=n/a why=static_error" + ) + + +class TestCli: + def _args(self, tmp_path: Path, out: Path, *extra: str) -> list[str]: + src = tmp_path / "matplotlib.py" + src.write_text(SOURCE, encoding="utf-8") + _write(tmp_path / "static.json", STATIC_PASS) + _write(tmp_path / "smoke.json", _smoke(SOURCE)) + return [ + "record", + "--side", + "new", + "--spec", + "bar-grouped", + "--lib", + "matplotlib", + "--language", + "python", + "--source", + str(src), + "--static", + str(tmp_path / "static.json"), + "--static-exit", + "0", + "--smoke", + str(tmp_path / "smoke.json"), + "--out", + str(out), + *extra, + ] + + def test_writes_the_record_and_prints_the_notice(self, tmp_path, capsys): + out = tmp_path / "anyplot-adapt-new.json" + assert ar.main(self._args(tmp_path, out)) == 0 + record = json.loads(out.read_text(encoding="utf-8")) + assert record["status"] == "pass" and record["selfcheck"] is None + assert capsys.readouterr().out.strip() == ar.notice_line(record) + + def test_an_unwritable_record_exits_1(self, tmp_path, capsys): + assert ar.main(self._args(tmp_path, tmp_path / "no" / "such" / "dir.json")) == 1 + assert "::warning::adapt record (new) not written" in capsys.readouterr().out + + def test_a_usage_error_exits_2(self, tmp_path): + with pytest.raises(SystemExit) as exc: + ar.main(self._args(tmp_path, tmp_path / "r.json", "--side", "both")) + assert exc.value.code == 2 + + def test_runs_as_a_script_without_the_package(self, tmp_path): + """The workflow runs its regen-tools copy, where `automation` is not importable.""" + tools = tmp_path / "regen-tools" + tools.mkdir() + (tools / "adapt_record.py").write_text(SCRIPT.read_text(encoding="utf-8"), encoding="utf-8") + out = tmp_path / "r.json" + result = subprocess.run( + [sys.executable, "-I", str(tools / "adapt_record.py"), *self._args(tmp_path, out)], + cwd=tmp_path, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.startswith("::notice::adapt_check spec=bar-grouped lib=matplotlib side=new status=pass ") + + def test_imports_only_the_standard_library(self): + import ast + + tree = ast.parse(SCRIPT.read_text(encoding="utf-8")) + modules = { + alias.name.split(".")[0] for node in ast.walk(tree) if isinstance(node, ast.Import) for alias in node.names + } | {node.module.split(".")[0] for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module} + assert modules <= set(sys.stdlib_module_names) | {"__future__"} + + +class TestWithTheMirror: + """The mirror's real --out file is what the record builder reads.""" + + @pytest.mark.parametrize( + ("source", "exit_code", "status", "why"), [(FIXTURE, 0, "pass", "-"), (CATALOGUE, 1, "fail", "static_fail")] + ) + def test_a_converted_and_an_unconverted_file(self, tmp_path, source, exit_code, status, why): + src = tmp_path / "matplotlib.py" # the checker infers the library from the stem + src.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + static = tmp_path / "static.json" + command = [sys.executable, "-I", str(MIRROR), "check", str(src)] + command += ["--spec", str(SPEC), "--spec-id", "bar-grouped", "--out", str(static)] + result = subprocess.run(command, capture_output=True, text=True) + assert result.returncode == exit_code, result.stdout + result.stderr + record = ar.build_record( + side="new", + spec="bar-grouped", + lib="matplotlib", + language="python", + source=src, + static=static, + static_exit=result.returncode, + smoke=_write(tmp_path / "smoke.json", _smoke(src.read_text(encoding="utf-8"), advisory_failed=[])), + ) + assert (record["status"], record["why"]) == (status, why) + assert record["static"]["status"] == ("pass" if exit_code == 0 else "fail") + if status == "fail": + assert "rules=layout,columns,interface" in ar.notice_line(record) diff --git a/tests/unit/automation/scripts/test_regen_gate.py b/tests/unit/automation/scripts/test_regen_gate.py index 1ef5233e908..f086a103bf6 100644 --- a/tests/unit/automation/scripts/test_regen_gate.py +++ b/tests/unit/automation/scripts/test_regen_gate.py @@ -7,16 +7,24 @@ from __future__ import annotations +import ast import json import re +import subprocess +import sys from pathlib import Path +from typing import Any import pytest import yaml +from automation.scripts import regen_gate as gate_module from automation.scripts.regen_gate import ( + ADAPT_FILE_LIMIT, + ADAPT_WHY, CARRIER_CRITERIA, CRITERIA, + IDENTITY_BOUND, KEEP, KIND_EXPECTED, KIND_SHOWS, @@ -24,7 +32,14 @@ REASON_CODES, RECORD_KEYS, WRITEBACK_CODES, + AdaptAssessment, + AdaptInput, GateInput, + GateResult, + adapt_assess, + adapt_record_errors, + adapt_record_problem, + adapt_summary_line, build_record, call_count, characteristic_kind, @@ -34,11 +49,14 @@ decide, is_cq04_code_defect, is_removal, + load_adapt_file, load_checklist_scores, load_weakness_classes, load_weakness_texts, main, + normalize_regen, nothing_to_repair, + pair_problem, parse_characteristics, parse_record_markers, permission_refs, @@ -53,7 +71,9 @@ validate_regen, weakness_class, weakness_ids, + within_line_bound, ) +from core.render_identity import DEFAULT_BOUND, RenderIdentity KNOWN = frozenset({"W1", "W2", "W3"}) @@ -2112,6 +2132,7 @@ def test_marker_roundtrip_and_schema_unchanged(self): "coerced", "at", "writeback", + "adapt", } ) @@ -3051,3 +3072,1072 @@ def test_the_claim_must_show_in_the_scores(self, tmp_path, capsys): tmp_path, capsys, checklist={**NEW_CHECKLIST, "CQ-04": 1}, regen=_code_regen(), prev=[CODE_LINE] ) assert "code improvement 1 (W1) claims CQ-04, but your checklist for the new render gives CQ-04 1" in out + + +# --------------------------------------------------------------------------- +# The adapt route and the adapt regression (docs/concepts/adaptable-code.md, +# "The adapt route"; PR 2 contract section 5) +# --------------------------------------------------------------------------- + +ADAPT_SPEC = "bar-grouped" +ADAPT_LIB = "matplotlib" +# 100 lines before, 110 after: inside max(1.25 × 100, 100 + 15) = 125. +ADAPT_PREV_SOURCE = "".join(f"value_{i} = {i}\n" for i in range(100)) +ADAPT_NEW_SOURCE = "".join(f"value_{i} = {i}\n" for i in range(110)) +# The static and smoke statuses impl-review derives each record status from. +_SIDE_STATUSES = { + "pass": ("pass", "pass"), + "fail": ("fail", "skipped"), + "unknown": ("pass", "missing"), + "excluded": ("excluded", "skipped"), +} +_FILE = object() # the fixture's default file + + +def _adapt_record(side: str, status: str, **overrides: Any) -> dict: + """An adapt record as impl-review's record step writes it (contract section 4).""" + static, smoke = _SIDE_STATUSES[status] + record: dict[str, Any] = { + "v": 1, + "side": side, + "spec": ADAPT_SPEC, + "lib": ADAPT_LIB, + "language": "python", + "checker_version": 1, + "status": status, + "why": "-", + "lines": 100 if side == "prev" else 110, + # The same hashes as `_pair`'s two sides: the pair rendered these sources. + "code_sha256": "51aa07" if side == "prev" else "9b2f3c", + "static": { + "status": static, + "exit": 0 if static == "pass" else 1, + "version": 1, + "shape": "table", + "color_role": "group", + "block": {"start": 31, "end": 40}, + "constants": {"category": "QUARTER"}, + "findings": [], + "lines": [], + }, + "smoke": {"status": smoke, "why": "-", "fresh": True, "failed": [], "advisory_failed": [], "results": []}, + } + if side == "new": + record["selfcheck"] = {"attempts": 2, "first": "fail"} + record.update(overrides) + return record + + +def _identity(fraction: float = 0.0011, *, size=(3200, 1800), new_size=None, passed: Any = None) -> dict: + """One theme of a pair record: ``core.render_identity.RenderIdentity.to_dict()``, as ``pair`` writes it.""" + entry = RenderIdentity(size, new_size or size, round(fraction * size[0] * size[1]), DEFAULT_BOUND).to_dict() + if passed is not None: + entry["passed"] = passed + return entry + + +def _pair(light: dict | None = None, dark: dict | None = None, **overrides: Any) -> dict: + """``core.smoke_run pair`` output for two identical renders (contract section 1.3).""" + render = {"status": "ok", "size": [3200, 1800], "seconds": 2.1} + pair: dict[str, Any] = { + "v": 1, + "tool": "core.smoke_run pair", + "library": ADAPT_LIB, + "themes": ["light", "dark"], + "new": {"code_sha256": "9b2f3c", "renders": {"light": render, "dark": render}, "error": None, "line": None}, + "prev": {"code_sha256": "51aa07", "renders": {"light": render, "dark": render}, "error": None, "line": None}, + "identity": {"light": light or _identity(0.0011), "dark": dark or _identity(0.0009)}, + "status": "identical", + "why": "-", + } + pair.update(overrides) + return pair + + +def _adapt( + *, new: Any = "pass", prev: Any = "fail", pair: Any = _FILE, language: str | None = "python", **fields: Any +) -> AdaptInput: + """A status string builds that record, a dict is the parsed file as it is, None is a missing file.""" + + def load(value: Any, side: str) -> tuple[Any, str | None]: + if value is None: + return None, "missing" + return (_adapt_record(side, value), None) if isinstance(value, str) else (value, None) + + new_value, new_error = load(new, "new") + prev_value, prev_error = load(prev, "prev") + pair_value, pair_error = (_pair(), None) if pair is _FILE else load(pair, "pair") + return AdaptInput( + **{ + "library": ADAPT_LIB, + "language": language, + "new": new_value, + "prev": prev_value, + "pair": pair_value, + "new_error": new_error, + "prev_error": prev_error, + "pair_error": pair_error, + **fields, + } + ) + + +def _adapt_inp( + score: int | None = 85, regen: Any = _FILE, *, adapt: AdaptInput | None = None, **overrides +) -> GateInput: + """A regeneration whose review lists no improvement: kept as no_visible_improvement unless the route counts.""" + fields: dict[str, Any] = { + "spec_id": ADAPT_SPEC, + "score": score, + "regen": _regen(improvements=[]) if regen is _FILE else regen, + "known_weakness_ids": KNOWN, + "new_checklist": NEW_CHECKLIST, + "prev_source": ADAPT_PREV_SOURCE, + "new_source": ADAPT_NEW_SOURCE, + "adapt": adapt or _adapt(), + } + fields.update(overrides) + return GateInput(**fields) + + +def _review(regen: Any = _FILE) -> dict: + """The normalized review the route's floor reads.""" + return normalize_regen(_regen(improvements=[]) if regen is _FILE else regen)[0] + + +def _why(inp: GateInput, review: Any = None) -> str: + return adapt_assess(inp, review).why + + +class TestAdaptRoute: + """``adapt_assess``: every condition, in the contract's order (section 5.3).""" + + def test_the_route_holds_without_a_review(self): + adapt = adapt_assess(_adapt_inp(score=None, regen=None), None) + assert (adapt.why, adapt.counted, adapt.review, adapt.regression) == ("route", True, "skipped", False) + assert adapt.prev == {"status": "fail", "static": "fail", "smoke": "skipped"} + assert adapt.new == {"status": "pass", "static": "pass", "smoke": "pass"} + assert adapt.identity == {"light": 0.0011, "dark": 0.0009} + assert (adapt.version, adapt.prev_lines, adapt.new_lines) == (1, 100, 110) + + def test_the_route_holds_with_a_review(self): + adapt = adapt_assess(_adapt_inp(), _review()) + assert (adapt.why, adapt.counted, adapt.review) == ("route", True, "ran") + + @pytest.mark.parametrize( + ("adapt", "why"), + [ + ({"language": "r"}, "not_python"), + ({"language": "javascript"}, "not_python"), + ({"new": None}, "no_record"), + ({"prev": None}, "no_record"), + ({"new": _adapt_record("new", "pass", v=2)}, "record_invalid"), + ({"new": _adapt_record("prev", "pass")}, "record_invalid"), + ({"prev": _adapt_record("new", "fail")}, "record_invalid"), + ({"new": _adapt_record("new", "pass", spec="bar-basic")}, "record_invalid"), + ({"prev": _adapt_record("prev", "fail", lib="seaborn")}, "record_invalid"), + ({"new": _adapt_record("new", "pass", language="r")}, "record_invalid"), + ({"new": {**_adapt_record("new", "pass"), "status": "maybe"}}, "record_invalid"), + ({"new": _adapt_record("new", "pass", v=True)}, "record_invalid"), + ({"prev": ["fail"]}, "record_invalid"), + # The status without the two blocks it is derived from fails closed. + ({"new": {k: v for k, v in _adapt_record("new", "pass").items() if k != "static"}}, "record_invalid"), + ({"new": {k: v for k, v in _adapt_record("new", "pass").items() if k != "smoke"}}, "record_invalid"), + ({"new": {**_adapt_record("new", "pass"), "smoke": {"status": "ok"}}}, "record_invalid"), + ({"prev": {**_adapt_record("prev", "fail"), "static": "fail"}}, "record_invalid"), + ({"new": "excluded"}, "excluded"), + ({"prev": "excluded"}, "excluded"), + ({"new": "unknown"}, "new_unknown"), + ({"prev": "unknown"}, "prev_unknown"), + ({"prev": "pass"}, "prev_passes"), + ({"new": "fail"}, "new_fails"), + ({"pair": None}, "pair_missing"), + ({"pair": ["identical"]}, "pair_missing"), + ({"pair": _pair(v=2)}, "pair_missing"), + ({"pair": _pair(status="same")}, "pair_missing"), + # A pair rendered from another version of either source measures other code. + ({"pair": {**_pair(), "new": {**_pair()["new"], "code_sha256": "0000aa"}}}, "pair_stale"), + ({"pair": {**_pair(), "prev": {**_pair()["prev"], "code_sha256": "0000aa"}}}, "pair_stale"), + ({"pair": {**_pair(), "prev": {**_pair()["prev"], "code_sha256": None}}}, "pair_stale"), + ({"pair": {k: v for k, v in _pair().items() if k != "new"}}, "pair_stale"), + ({"new": {**_adapt_record("new", "pass"), "code_sha256": None}}, "pair_stale"), + ({"pair": _pair(library="seaborn")}, "pair_missing"), + ({"pair": _pair(identity=None)}, "pair_missing"), + ({"pair": _pair(identity={"light": _identity()})}, "pair_missing"), + ({"pair": _pair(light={**_identity(), "old_size": [3200]})}, "pair_missing"), + ({"pair": _pair(light={**_identity(), "passed": "yes"})}, "pair_missing"), + ({"pair": _pair(dark={**_identity(), "fraction": "0.001"})}, "pair_missing"), + ({"pair": _pair(dark={**_identity(), "fraction": True})}, "pair_missing"), + ({"pair": _pair(status="error", why="prev_render_failed", identity=None)}, "pair_error"), + ({"pair": _pair(status="error", why="new_render_failed", identity=None)}, "pair_error"), + ({"pair": _pair(light=_identity(new_size=(3160, 1760)))}, "size_changed"), + ({"pair": _pair(dark=_identity(size=(2400, 2400), new_size=(2400, 2399)))}, "size_changed"), + ({"pair": _pair(light=_identity(0.006))}, "identity_light"), + ({"pair": _pair(dark=_identity(0.0051))}, "identity_dark"), + ({"pair": _pair(light=_identity(0.02), dark=_identity(0.3))}, "identity_both"), + # The gate's own bound, whatever the pair says passed. + ({"pair": _pair(light=_identity(0.006, passed=True))}, "identity_light"), + ({"pair": _pair(dark=_identity(0.001, passed=False))}, "identity_dark"), + ({"pair": _pair(light={**_identity(), "fraction": float("nan")})}, "identity_light"), + ({"pair": _pair(light={**_identity(), "fraction": None})}, "identity_light"), + ({"pair": _pair(light={**_identity(), "fraction": -0.1})}, "identity_light"), + ], + ) + def test_conditions_before_the_sources(self, adapt, why): + assert _why(_adapt_inp(adapt=_adapt(**adapt))) == why + assert _why(_adapt_inp(adapt=_adapt(**adapt)), _review()) == why + + def test_unreadable_files(self): + """A file that does not parse is malformed, not missing: ``record_invalid`` and ``pair_missing``.""" + base = _adapt().__dict__ + for side in ("new", "prev"): + adapt = AdaptInput(**{**base, side: None, f"{side}_error": "unreadable"}) + assert _why(_adapt_inp(adapt=adapt)) == "record_invalid", side + assert AdaptInput(**{f"{side}_error": "unreadable"}).requested + pair = AdaptInput(**{**base, "pair": None, "pair_error": "unreadable"}) + assert _why(_adapt_inp(adapt=pair)) == "pair_missing" + assert not AdaptInput().requested and AdaptInput(review_skipped=True).requested + + def test_without_any_adapt_input_the_route_is_off(self): + inp = _adapt_inp(adapt=AdaptInput()) + adapt = adapt_assess(inp, _review()) + assert (adapt.why, adapt.counted, adapt.requested, adapt.prev, adapt.new, adapt.identity) == ( + "no_record", + False, + False, + None, + None, + None, + ) + + @pytest.mark.parametrize( + ("overrides", "why"), + [ + ({"prev_source": None}, "no_sources"), + ({"new_source": None}, "no_sources"), + ({"new_source": ADAPT_NEW_SOURCE + "x = 1\n" * 16}, "lines_over"), + ({"canvas_failed": True}, "canvas_failed"), + ], + ) + def test_conditions_after_the_identity(self, overrides, why): + assert _why(_adapt_inp(**overrides)) == why + + @pytest.mark.parametrize( + ("prev", "new", "within"), + [ + (100, 125, True), + (100, 126, False), + (40, 55, True), + (40, 56, False), + (200, 250, True), + (200, 251, False), + (0, 15, True), + (0, 16, False), + (100, 60, True), + ], + ) + def test_line_bound_is_the_larger_of_a_quarter_and_15_lines(self, prev, new, within): + assert within_line_bound(prev, new) is within + + def test_the_first_failing_condition_wins(self): + assert _why(_adapt_inp(adapt=_adapt(prev="pass", pair=None))) == "prev_passes" + assert _why(_adapt_inp(adapt=_adapt(new="fail", pair=None))) == "new_fails" + assert _why(_adapt_inp(adapt=_adapt(new="unknown", prev="pass"))) == "new_unknown" + assert _why(_adapt_inp(adapt=_adapt(language="r", new=None))) == "not_python" + assert _why(_adapt_inp(canvas_failed=True, prev_source=None)) == "no_sources" + light = _pair(light=_identity(0.3, new_size=(3200, 1799)), dark=_identity(0.3)) + assert _why(_adapt_inp(adapt=_adapt(pair=light))) == "size_changed" + + @pytest.mark.parametrize( + ("score", "review", "overrides", "why"), + [ + (0, None, {}, "score_zero"), + (None, None, {}, "score_zero"), + (85, {"regressions": [{"what": "legend lost", "where_visible": "both"}]}, {}, "review_regression"), + (85, {"scenario_changed": True}, {}, "scenario_changed"), + (85, {"encodings_added": ["size by revenue"]}, {}, "encodings_added"), + (85, {"encodings_added": [" "]}, {}, "route"), + (85, {"prev_checklist": {}}, {}, "no_prev_checklist"), + (85, None, {"new_checklist": {**NEW_CHECKLIST, "SC-02": 3}}, "technical_below"), + (85, None, {"new_checklist": {k: v for k, v in NEW_CHECKLIST.items() if k != "CQ-01"}}, "technical_below"), + (85, None, {"new_checklist": {}}, "technical_below"), + # The judgment items (DE, LM) are no floor. + (85, None, {"new_checklist": {**NEW_CHECKLIST, "DE-01": 2, "LM-02": 1}}, "route"), + # No tolerance: a score far below the re-score still counts. + (40, None, {}, "route"), + ], + ) + def test_the_review_floor(self, score, review, overrides, why): + inp = _adapt_inp(score=score, **overrides) + assert _why(inp, _review(_regen(improvements=[], **(review or {})))) == why + assert _why(inp, None) == "route" # without a review the floor does not apply + + def test_scenario_and_encodings_on_a_basic_spec_are_review_regressions(self): + spec = "bar-basic" + adapt = _adapt(new=_adapt_record("new", "pass", spec=spec), prev=_adapt_record("prev", "fail", spec=spec)) + inp = _adapt_inp(spec_id=spec, adapt=adapt) + assert _why(inp, _review(_regen(improvements=[], scenario_changed=True))) == "review_regression" + assert _why(inp, _review(_regen(improvements=[], encodings_added=["size"]))) == "review_regression" + + def test_an_empty_review_fails_the_floor(self): + """A review that ran but left no valid review_regen.json: nothing to hold the floor against.""" + assert _why(_adapt_inp(), {}) == "no_prev_checklist" + + def test_the_regression_is_set_whenever_both_records_are_valid(self): + regression = adapt_assess(_adapt_inp(adapt=_adapt(prev="pass", new="fail", pair=None)), None) + assert (regression.regression, regression.static_regression, regression.why) == (True, False, "prev_passes") + # The fixture's `unknown` predecessor passes the static check and has no smoke run. + static = adapt_assess(_adapt_inp(adapt=_adapt(prev="unknown", new="fail")), None) + assert (static.regression, static.static_regression, static.why) == (True, True, "prev_unknown") + for adapt in ( + _adapt(prev="pass", new="fail", language="r"), + _adapt(prev="pass", new=_adapt_record("new", "fail", v=2)), + _adapt(prev="pass", new=None), + _adapt(prev="pass", new="unknown"), + _adapt(prev=None, new="fail"), + ): + assert adapt_assess(_adapt_inp(adapt=adapt), None).regression is False + + def test_a_missing_static_or_smoke_block_fails_closed(self): + """The derived status needs both blocks; without them the record is malformed, never a pass.""" + record = _adapt_record("prev", "fail") + del record["smoke"] + record["static"] = {"status": "weird"} + adapt = adapt_assess(_adapt_inp(adapt=_adapt(prev=record)), None) + assert (adapt.why, adapt.counted, adapt.prev) == ("record_invalid", False, None) + + def test_a_crafted_pass_over_a_missing_smoke_run_is_invalid(self): + """The record the second opinion crafted: `pass` with a smoke run that never ran used to route.""" + crafted = {**_adapt_record("new", "pass"), "static": {"status": "pass"}, "smoke": {"status": "missing"}} + assert adapt_record_problem(crafted, "new", ADAPT_SPEC, ADAPT_LIB) == ( + "status does not follow from static and smoke" + ) + adapt = adapt_assess(_adapt_inp(adapt=_adapt(new=crafted)), None) + assert (adapt.why, adapt.counted, adapt.new) == ("record_invalid", False, None) + + @pytest.mark.parametrize( + ("side", "status", "static", "smoke", "smoke_why"), + [ + ("prev", "fail", "pass", "pass", "-"), # a predecessor that passes, relabelled fail + ("prev", "fail", "pass", "missing", "-"), + ("new", "pass", "fail", "pass", "-"), + ("new", "pass", "pass", "stale", "-"), + ("new", "pass", "pass", "skipped", "runner_error"), + ("new", "pass", "pass", "skipped", "no_perturbations:table"), + ("new", "pass", "pass", "fail", "perturbation_failed"), + ("new", "pass", "error", "pass", "-"), + ("new", "pass", "excluded", "skipped", "shape:excluded"), + ("new", "unknown", "pass", "pass", "-"), + ("new", "excluded", "pass", "pass", "-"), + ], + ) + def test_a_status_that_does_not_follow_from_its_blocks_is_invalid(self, side, status, static, smoke, smoke_why): + record = _adapt_record(side, status) + record["static"] = {**record["static"], "status": static} + record["smoke"] = {**record["smoke"], "status": smoke, "why": smoke_why} + problem = adapt_record_problem(record, side, ADAPT_SPEC, ADAPT_LIB) + assert problem == "status does not follow from static and smoke" + assert _why(_adapt_inp(adapt=_adapt(**{side: record}))) == "record_invalid" + + @pytest.mark.parametrize( + ("status", "static", "smoke", "smoke_why"), + [ + ("pass", "pass", "pass", "-"), + ("pass", "pass", "skipped", "shape:generated"), + ("pass", "pass", "skipped", "shape:network"), + ("fail", "fail", "pass", "-"), + ("fail", "fail", "skipped", "static_failed"), + ("fail", "fail", "missing", "-"), + ("fail", "pass", "fail", "perturbation_failed"), + ("excluded", "excluded", "skipped", "shape:excluded"), + ("excluded", "excluded", "missing", "-"), + ("unknown", "error", "pass", "-"), + ("unknown", "pass", "missing", "-"), + ("unknown", "pass", "stale", "-"), + ("unknown", "pass", "skipped", "runner_error"), + ("unknown", "pass", "skipped", "no_perturbations:table"), + ], + ) + def test_every_consistent_record_is_read(self, status, static, smoke, smoke_why): + record = _adapt_record("prev", status) + record["static"] = {**record["static"], "status": static} + record["smoke"] = {**record["smoke"], "status": smoke, "why": smoke_why} + assert adapt_record_problem(record, "prev", ADAPT_SPEC, ADAPT_LIB) is None + assert adapt_assess(_adapt_inp(adapt=_adapt(prev=record)), None).prev == { + "status": status, + "static": static, + "smoke": smoke, + } + + def test_identity_fractions_for_the_record(self): + pair = _pair(light=_identity(new_size=(3160, 1760)), dark=_identity(0.123456789)) + adapt = adapt_assess(_adapt_inp(adapt=_adapt(pair=pair)), None) + assert adapt.identity == {"light": None, "dark": 0.123457} + assert adapt_assess(_adapt_inp(adapt=_adapt(pair=None)), None).identity is None + error = _pair(status="error", identity=None) + assert adapt_assess(_adapt_inp(adapt=_adapt(pair=error)), None).identity is None + + def test_a_non_integer_checker_version_is_null(self): + adapt = _adapt(new=_adapt_record("new", "pass", checker_version="1")) + assert adapt_assess(_adapt_inp(adapt=adapt), None).version is None + + +class TestAdaptDecision: + """Where the route enters ``decide`` (contract section 5.4).""" + + def test_review_free_merge_before_the_score_checks(self): + result = decide(_adapt_inp(score=None, regen=None, adapt=_adapt(review_skipped=True))) + assert (result.verdict, result.code) == (MERGE, "adapt_identity") + assert result.reason.startswith("adapt route, no review: the predecessor fails the adaptability check") + assert "0.11 % (light) and 0.09 % (dark)" in result.reason and "100 → 110 lines" in result.reason + assert result.adapt is not None and result.adapt.review == "skipped" + + @pytest.mark.parametrize( + "adapt", [{"pair": _pair(dark=_identity(0.01))}, {"prev": "pass"}, {"new": None}, {"pair": None}] + ) + def test_review_free_without_the_route_keeps_for_the_missing_score(self, adapt): + result = decide(_adapt_inp(score=None, regen=None, adapt=_adapt(review_skipped=True, **adapt))) + assert (result.verdict, result.code) == (KEEP, "no_score") + assert result.adapt is not None and not result.adapt.counted + + def test_review_free_canvas_failure_comes_first(self): + result = decide(_adapt_inp(score=None, regen=None, canvas_failed=True, adapt=_adapt(review_skipped=True))) + assert (result.verdict, result.code) == (KEEP, "canvas_failed") + assert result.adapt is not None and result.adapt.why == "canvas_failed" + + def test_the_review_free_exit_needs_the_flag(self): + result = decide(_adapt_inp(score=None, regen=None)) + assert (result.verdict, result.code) == (KEEP, "no_score") + assert result.adapt is not None and result.adapt.why == "score_zero" + + def test_a_score_next_to_the_flag_means_the_review_ran(self): + """The fail-safe direction: the stricter, reviewed path.""" + merged = decide(_adapt_inp(adapt=_adapt(review_skipped=True))) + assert (merged.verdict, merged.code) == (MERGE, "adapt_identity") + assert merged.adapt is not None and merged.adapt.review == "ran" + floor = decide(_adapt_inp(adapt=_adapt(review_skipped=True), new_checklist={**NEW_CHECKLIST, "VQ-01": 7})) + assert (floor.verdict, floor.code) == (KEEP, "no_visible_improvement") + assert floor.adapt is not None and floor.adapt.why == "technical_below" + + def test_no_visible_improvement_becomes_adapt_identity(self): + result = decide(_adapt_inp()) + assert (result.verdict, result.code) == (MERGE, "adapt_identity") + assert result.reason.startswith("adapt route: the predecessor fails") + assert result.reason.endswith("(without the route: no_visible_improvement)") + assert decide(_adapt_inp(adapt=AdaptInput())).code == "no_visible_improvement" + + def test_no_defect_improvement_becomes_adapt_identity(self): + suggestion = [{"ref": "P1", "rule": "DE-02", "what": "fonts aligned", "where_visible": "both renders"}] + regen = _regen(improvements=suggestion) + assert decide(_adapt_inp(regen=regen, adapt=AdaptInput())).code == "no_defect_improvement" + result = decide(_adapt_inp(regen=regen)) + assert (result.verdict, result.code) == (MERGE, "adapt_identity") + assert result.reason.endswith("(without the route: no_defect_improvement)") + + def test_below_tolerance_never_applies_to_the_route(self): + carrier = _regen() # W2 carries; the re-score is 85 + assert decide(_adapt_inp(score=80, regen=carrier, adapt=AdaptInput())).code == "below_tolerance" + result = decide(_adapt_inp(score=80, regen=carrier)) + assert (result.verdict, result.code) == (MERGE, "adapt_identity") + assert result.reason.endswith("(without the route: below_tolerance)") + + def test_a_carrier_merge_keeps_its_code_and_records_the_route(self): + result = decide(_adapt_inp(regen=_regen())) + assert (result.verdict, result.code) == (MERGE, "merge") + assert result.adapt is not None and result.adapt.counted + record = build_record(result, spec_id=ADAPT_SPEC, library=ADAPT_LIB, score=85, prev_stored=94) + assert record["adapt"]["counted"] is True and record["code"] == "merge" + + def test_the_regression_exit_is_unchanged(self): + regen = _regen(improvements=[], regressions=[{"what": "legend lost", "where_visible": "both renders"}]) + result = decide(_adapt_inp(regen=regen)) + assert (result.verdict, result.code) == (KEEP, "regression") + assert result.adapt is not None and result.adapt.why == "review_regression" + + def test_the_route_does_not_lift_a_keep_before_the_judgement(self): + for overrides, code in ( + ({"prev_renders": False}, "prev_renders_missing"), + ({"context_ok": False}, "context_failed"), + ({"regen": None, "regen_error": "missing"}, "regen_json_missing"), + ({"regen": _regen(prev_rescored="many")}, "regen_json_invalid"), + ({"score": 0}, "score_zero"), + ): + result = decide(_adapt_inp(**overrides)) + assert (result.verdict, result.code) == (KEEP, code), overrides + + def test_the_route_is_off_with_an_invalid_review_file(self): + result = decide(_adapt_inp(regen=_regen(prev_rescored="many"))) + assert result.adapt is not None and result.adapt.why == "no_prev_checklist" + + def test_the_cq04_path_keeps_its_tolerance_without_the_route(self): + regen = _code_regen(prev_rescored=85) + prev_passes = _adapt(prev="pass") + result = decide(_code_inp(score=80, regen=regen, spec_id=ADAPT_SPEC, adapt=prev_passes)) + assert (result.verdict, result.code) == (KEEP, "below_tolerance") + merged = decide(_code_inp(regen=regen, spec_id=ADAPT_SPEC, adapt=prev_passes)) + assert (merged.verdict, merged.code) == (MERGE, "merge") + + def test_older_inputs_decide_exactly_as_before(self): + """No adapt input: every fixture of the earlier tests keeps its verdict, code and reason.""" + for inp in (_inp(), _inp(score=80), _inp(regen=_regen(improvements=[])), _code_inp(), _inp(score=None)): + result = decide(inp) + assert result.adapt is not None and result.adapt.why == "no_record" + assert result.code not in ("adapt_identity", "adapt_regression") + + +class TestAdaptOverruled: + """A route that held before the review and that the review's floor turned off (``regen:adapt-overruled``).""" + + @pytest.mark.parametrize( + ("score", "review", "overrides", "why"), + [ + (0, None, {}, "score_zero"), + (85, {"regressions": [{"what": "legend lost", "where_visible": "both"}]}, {}, "review_regression"), + (85, {"scenario_changed": True}, {}, "scenario_changed"), + (85, {"encodings_added": ["size by revenue"]}, {}, "encodings_added"), + (85, {"prev_checklist": {}}, {}, "no_prev_checklist"), + (85, None, {"new_checklist": {**NEW_CHECKLIST, "VQ-01": 7}}, "technical_below"), + ], + ) + def test_every_review_floor_reason_overrules_a_held_route(self, score, review, overrides, why): + inp = _adapt_inp(score=score, adapt=_adapt(route_held=True), **overrides) + adapt = adapt_assess(inp, _review(_regen(improvements=[], **(review or {})))) + assert (adapt.why, adapt.counted, adapt.route_pre_review, adapt.overruled) == (why, False, True, True) + assert why in gate_module.ADAPT_REVIEW_WHY + + def test_the_decision_records_an_overruled_route(self): + result = decide(_adapt_inp(adapt=_adapt(route_held=True), new_checklist={**NEW_CHECKLIST, "VQ-01": 7})) + assert (result.verdict, result.code) == (KEEP, "no_visible_improvement") + assert result.adapt is not None and result.adapt.overruled + record = build_record(result, spec_id=ADAPT_SPEC, library=ADAPT_LIB, score=85, prev_stored=94) + assert (record["adapt"]["why"], record["adapt"]["route_pre_review"]) == ("technical_below", True) + line = next(line for line in render_summary(result, "94", 85).splitlines() if "Adapt route" in line) + assert line.endswith("; the route held before the review, so the review overruled it") + + def test_a_carrier_merge_can_overrule_the_route_too(self): + """The review counted a carrier but its floor failed: merged, not counted, overruled.""" + result = decide( + _adapt_inp(regen=_regen(), adapt=_adapt(route_held=True), new_checklist={**NEW_CHECKLIST, "VQ-01": 7}) + ) + assert (result.verdict, result.code) == (MERGE, "merge") + assert result.adapt is not None and (result.adapt.counted, result.adapt.overruled) == (False, True) + + @pytest.mark.parametrize( + ("adapt", "overrides"), + [ + (_adapt(route_held=True), {}), # counted: nothing overruled + (_adapt(), {"new_checklist": {**NEW_CHECKLIST, "VQ-01": 7}}), # no route before the review + # A reason the review does not give (the reviewed file grew past the bound): not the review's verdict. + (_adapt(route_held=True), {"new_source": ADAPT_NEW_SOURCE + "x = 1\n" * 16}), + (_adapt(route_held=True, review_skipped=True), {"score": None, "regen": None}), + ], + ) + def test_nothing_else_is_overruled(self, adapt, overrides): + result = decide(_adapt_inp(adapt=adapt, **overrides)) + assert result.adapt is not None and not result.adapt.overruled + line = adapt_summary_line(result.adapt) + assert "overruled" not in line + + +class TestAdaptRegression: + """A predecessor that passes and a new source that fails: keep, on every spec (contract section 5.4).""" + + def test_a_carrier_merge_becomes_keep(self): + result = decide(_adapt_inp(regen=_regen(), adapt=_adapt(prev="pass", new="fail"))) + assert (result.verdict, result.code) == (KEEP, "adapt_regression") + assert result.reason.startswith("adapt regression: the predecessor passes the adaptability check") + assert result.reason.endswith("(without it: merge)") + + def test_a_code_path_merge_becomes_keep(self): + result = decide(_code_inp(spec_id=ADAPT_SPEC, adapt=_adapt(prev="pass", new="fail"))) + assert (result.verdict, result.code) == (KEEP, "adapt_regression") + + def test_on_a_basic_spec_too(self): + spec = "bar-basic" + adapt = _adapt(new=_adapt_record("new", "fail", spec=spec), prev=_adapt_record("prev", "pass", spec=spec)) + result = decide(_adapt_inp(spec_id=spec, regen=_regen(), adapt=adapt)) + assert (result.verdict, result.code) == (KEEP, "adapt_regression") + + def test_a_keep_keeps_its_own_code(self): + result = decide(_adapt_inp(adapt=_adapt(prev="pass", new="fail"))) + assert (result.verdict, result.code) == (KEEP, "no_visible_improvement") + assert result.adapt is not None and result.adapt.regression + + @pytest.mark.parametrize( + "adapt", + [ + {"prev": "pass", "new": "pass"}, + {"prev": "fail", "new": "fail"}, + {"prev": "pass", "new": "unknown"}, + {"prev": "pass", "new": None}, + {"prev": "pass", "new": "fail", "language": "julia"}, + {"prev": _adapt_record("prev", "pass", lib="seaborn"), "new": "fail"}, + {"prev": "unknown", "new": "unknown"}, + {"prev": "unknown", "new": "pass"}, + ], + ) + def test_no_regression_without_two_valid_records_saying_so(self, adapt): + result = decide(_adapt_inp(regen=_regen(), adapt=_adapt(**adapt))) + assert (result.verdict, result.code) == (MERGE, "merge") + + @pytest.mark.parametrize(("smoke", "smoke_why"), [("missing", "-"), ("stale", "-"), ("skipped", "runner_error")]) + def test_a_static_only_regression_needs_no_smoke_run(self, smoke, smoke_why): + """The predecessor's static check passes, its smoke run is not known, the new static check fails.""" + prev = _adapt_record("prev", "unknown") + prev["smoke"] = {**prev["smoke"], "status": smoke, "why": smoke_why} + result = decide(_adapt_inp(regen=_regen(), adapt=_adapt(prev=prev, new="fail"))) + assert (result.verdict, result.code) == (KEEP, "adapt_regression") + assert result.reason.startswith( + "adapt regression: the predecessor passes the static adaptability check (its smoke run is not known) " + "and the new source fails it, so the live implementation stays" + ) + assert result.reason.endswith("(without it: merge)") + assert "**Adapt route:** regression: the predecessor passes the static adaptability check" in ( + render_summary(result, "94", 85) + ) + + def test_a_static_only_regression_turns_a_code_path_merge_into_keep(self): + result = decide(_code_inp(spec_id=ADAPT_SPEC, adapt=_adapt(prev="unknown", new="fail"))) + assert (result.verdict, result.code) == (KEEP, "adapt_regression") + + @pytest.mark.parametrize( + ("prev", "new"), + [ + # The new source passes the static check; only its smoke run fails, against an unknown one. + (("unknown", "pass", "missing"), ("fail", "pass", "fail")), + # The predecessor's static check errored: nothing is known about it. + (("unknown", "error", "pass"), ("fail", "fail", "skipped")), + # The predecessor's own smoke run failed: it passes nothing. + (("fail", "pass", "fail"), ("fail", "fail", "skipped")), + ], + ) + def test_no_static_only_regression_without_certainty(self, prev, new): + records = {} + for side, (status, static, smoke) in (("prev", prev), ("new", new)): + record = _adapt_record(side, status) + record["static"] = {**record["static"], "status": static} + record["smoke"] = {**record["smoke"], "status": smoke} + records[side] = record + result = decide(_adapt_inp(regen=_regen(), adapt=_adapt(**records))) + assert (result.verdict, result.code) == (MERGE, "merge") + assert result.adapt is not None and not result.adapt.regression + + +class TestAdaptRecord: + """The record's ``adapt`` block (contract section 5.5).""" + + def _record(self, result, **fields) -> dict: + return build_record( + result, spec_id=ADAPT_SPEC, library=ADAPT_LIB, at="2026-10-24T09:30:11Z", **{"score": None, **fields} + ) + + def test_the_review_free_merge(self): + result = decide(_adapt_inp(score=None, regen=None, adapt=_adapt(review_skipped=True, route_held=True))) + record = self._record(result, prev_stored=94, pr=12301) + assert (record["verdict"], record["code"], record["prev_rescored"], record["new"]) == ( + "merge", + "adapt_identity", + None, + None, + ) + assert record["adapt"] == { + "version": 1, + "prev": {"status": "fail", "static": "fail", "smoke": "skipped"}, + "new": {"status": "pass", "static": "pass", "smoke": "pass"}, + "identity": {"light": 0.0011, "dark": 0.0009}, + "counted": True, + "why": "route", + "review": "skipped", + "route_pre_review": True, + } + assert validate_record(record) == [] + assert parse_record_markers(render_record_marker(record)) == [record] + + def test_without_adapt_input(self): + record = self._record(decide(_inp()), score=85, prev_stored=90) + assert record["adapt"] == { + "version": None, + "prev": None, + "new": None, + "identity": None, + "counted": False, + "why": "no_record", + "review": "ran", + "route_pre_review": None, + } + assert validate_record(record) == [] + + @pytest.mark.parametrize( + ("adapt", "expected"), + [ + (_adapt(route_held=True), True), + (_adapt(), False), # adapt input without the flag: the route did not hold before the review + (AdaptInput(route_held=True), True), + (AdaptInput(), None), + (AdaptInput(language="r"), None), + ], + ) + def test_route_pre_review(self, adapt, expected): + record = self._record(decide(_adapt_inp(adapt=adapt)), score=85, prev_stored=90) + assert record["adapt"]["route_pre_review"] is expected + assert validate_record(record) == [] + + @pytest.mark.parametrize("value", ["yes", 1, 0, "true", {"held": True}]) + def test_validate_rejects_a_route_pre_review_that_is_no_boolean(self, value): + record = self._record(decide(_adapt_inp()), score=85, prev_stored=90) + record["adapt"]["route_pre_review"] = value + assert "record.adapt.route_pre_review must be true, false or null" in validate_record(record) + + def test_a_block_without_route_pre_review_still_validates(self): + """The key came after the block's first form: additive, RECORD_VERSION stays 1.""" + record = self._record(decide(_adapt_inp()), score=85, prev_stored=90) + del record["adapt"]["route_pre_review"] + assert validate_record(record) == [] + + def test_a_result_without_an_assessment_records_no_record(self): + record = build_record(GateResult(KEEP, "x", "no_score"), spec_id="s", library="b", score=None, prev_stored=None) + assert record["adapt"]["why"] == "no_record" and validate_record(record) == [] + + def test_not_python(self): + result = decide(_inp(adapt=AdaptInput(language="r"))) + assert self._record(result, score=85, prev_stored=90)["adapt"]["why"] == "not_python" + + def test_a_size_change_records_a_null_fraction(self): + pair = _pair(light=_identity(new_size=(3160, 1760))) + record = self._record(decide(_adapt_inp(adapt=_adapt(pair=pair))), score=85, prev_stored=90) + assert record["adapt"]["identity"] == {"light": None, "dark": 0.0009} + assert record["adapt"]["why"] == "size_changed" + assert validate_record(record) == [] + + def test_every_reason_validates(self): + base = self._record(decide(_adapt_inp()), score=85, prev_stored=90) + for why in ADAPT_WHY: + record = {**base, "adapt": {**base["adapt"], "why": why, "counted": why == "route"}} + assert validate_record(record) == [], why + + @pytest.mark.parametrize( + ("change", "message"), + [ + ({"extra": 1}, "unknown key 'extra'"), + ({"version": "1"}, "version must be an integer or null"), + ({"prev": ["fail"]}, "prev must be null or an object"), + ({"prev": {"status": "fail", "static": "fail"}}, "prev must be null or an object"), + ({"new": {"status": "great", "static": "pass", "smoke": "pass"}}, "new.status is not a known status"), + ({"new": {"status": None, "static": "pass", "smoke": "pass"}}, "new.status is not a known status"), + ({"new": {"status": "pass", "static": "fine", "smoke": "pass"}}, "new.static is not a known status"), + ({"identity": {"light": 0.001}}, "identity must be null or an object with light and dark"), + ({"identity": {"light": 1.5, "dark": 0.0}}, "identity.light must be a fraction"), + ({"identity": {"light": float("nan"), "dark": 0.0}}, "identity.light must be a fraction"), + ({"identity": {"light": True, "dark": 0.0}}, "identity.light must be a fraction"), + ({"identity": [0.001, 0.002]}, "identity must be null or an object"), + ({"counted": "yes"}, "counted must be true or false"), + ({"why": "because"}, "why is not one of ADAPT_WHY"), + ({"counted": False}, "counted must be true exactly when why is route"), + ({"counted": True, "why": "identity_dark"}, "counted must be true exactly when why is route"), + ({"review": "maybe"}, "review must be ran or skipped"), + ], + ) + def test_validate_rejects_a_bad_block(self, change, message): + record = self._record(decide(_adapt_inp()), score=85, prev_stored=90) + record["adapt"] = {**record["adapt"], **change} + errors = validate_record(record) + assert any(message in e for e in errors), errors + + def test_validate_rejects_a_missing_key_and_a_non_object(self): + record = self._record(decide(_adapt_inp()), score=85, prev_stored=90) + del record["adapt"]["review"] + assert "record.adapt lacks 'review'" in validate_record(record) + assert validate_record({**record, "adapt": ["route"]})[0] == "record.adapt is not an object" + assert adapt_record_errors(None) == ["record.adapt is not an object"] + + def test_records_written_before_the_route_still_validate(self): + record = self._record(decide(_adapt_inp()), score=85, prev_stored=90) + del record["adapt"] + assert validate_record(record) == [] + assert parse_record_markers(render_record_marker(record)) == [record] + + def test_both_codes_are_declared(self): + assert {"adapt_identity", "adapt_regression"} <= set(REASON_CODES) + + +class TestAdaptSummary: + def test_counted_merge(self): + summary = render_summary( + decide(_adapt_inp(score=None, regen=None, adapt=_adapt(review_skipped=True))), "94", None + ) + line = next(line for line in summary.splitlines() if line.startswith("**Adapt route:**")) + assert line.startswith("**Adapt route:** counted: the predecessor fails the adaptability check (static fail,") + assert line.endswith("100 → 110 lines; no review ran") + + def test_not_counted_names_the_reason(self): + result = decide(_adapt_inp(adapt=_adapt(pair=_pair(dark=_identity(0.01))))) + assert ( + "**Adapt route:** not counted (identity_dark): the dark renders differ in more than 0.50 % of their pixels" + in render_summary(result, "94", 85) + ) + + def test_regression(self): + result = decide(_adapt_inp(regen=_regen(), adapt=_adapt(prev="pass", new="fail"))) + assert "**Adapt route:** regression: the predecessor passes the adaptability check" in render_summary( + result, "94", 85 + ) + + def test_no_line_without_adapt_input(self): + assert "Adapt route" not in render_summary(decide(_inp()), "90", 85) + assert "Adapt route" not in render_summary(decide(_inp(adapt=AdaptInput(language="r"))), "90", 85) + + def test_every_reason_has_a_text(self): + for why in ADAPT_WHY: + line = adapt_summary_line(AdaptAssessment(why=why)) + expected = "**Adapt route:** counted: " if why == "route" else f"**Adapt route:** not counted ({why}): " + assert line.startswith(expected), why + assert "@" not in line + assert set(gate_module.ADAPT_WHY_TEXT) == set(ADAPT_WHY) - {"route"} + + +class TestAdaptFiles: + def test_load_adapt_file(self, tmp_path): + assert load_adapt_file("") == (None, "absent") + assert load_adapt_file(str(tmp_path / "absent.json")) == (None, "missing") + bad = tmp_path / "bad.json" + bad.write_text("{not json", encoding="utf-8") + assert load_adapt_file(str(bad)) == (None, "unreadable") + deep = tmp_path / "deep.json" + deep.write_text("[" * 200_000 + "]" * 200_000, encoding="utf-8") + assert load_adapt_file(str(deep)) == (None, "unreadable") + huge = tmp_path / "huge.json" + huge.write_text(" " * (ADAPT_FILE_LIMIT + 1) + "{}", encoding="utf-8") + assert load_adapt_file(str(huge)) == (None, "unreadable") + good = tmp_path / "good.json" + good.write_text(json.dumps(_pair()), encoding="utf-8") + assert load_adapt_file(str(good)) == (_pair(), None) + assert load_adapt_file(str(tmp_path)) == (None, "missing") # a directory is no file + + def test_identity_bound_is_render_identitys_default(self): + assert IDENTITY_BOUND == DEFAULT_BOUND + + def test_render_identity_entries_parse(self): + """The identity block is core.render_identity's own dict, a size change included.""" + assert pair_problem(_pair(), ADAPT_LIB) is None + changed = _pair(light=RenderIdentity((3200, 1800), (3160, 1760), 0, DEFAULT_BOUND).to_dict()) + assert pair_problem(changed, ADAPT_LIB) is None + assert changed["identity"]["light"]["fraction"] is None + assert _why(_adapt_inp(adapt=_adapt(pair=changed))) == "size_changed" + assert pair_problem(_pair(status="error", identity=None), ADAPT_LIB) is None + assert pair_problem(_pair(), "seaborn") == "library is missing or names another library" + + @pytest.mark.parametrize("library", [None, "", ["matplotlib"]]) + def test_a_pair_must_name_its_library(self, library): + """``core.smoke_run pair`` always writes ``library``; a record without it was not written there.""" + pair = _pair() + if library is None: + del pair["library"] + else: + pair["library"] = library + assert pair_problem(pair, ADAPT_LIB) == "library is missing or names another library" + assert pair_problem({**pair, "status": "error", "identity": None}, ADAPT_LIB) is not None + assert _why(_adapt_inp(adapt=_adapt(pair=pair))) == "pair_missing" + + +def _adapt_files(tmp_path: Path, *, new: str = "pass", prev: str = "fail", pair: dict | None = None) -> list[str]: + """The adapt flags with their files, and the two sources, as impl-review passes them.""" + (tmp_path / "adapt-new.json").write_text(json.dumps(_adapt_record("new", new)), encoding="utf-8") + (tmp_path / "adapt-prev.json").write_text(json.dumps(_adapt_record("prev", prev)), encoding="utf-8") + (tmp_path / "pair.json").write_text(json.dumps(pair or _pair()), encoding="utf-8") + (tmp_path / "anyplot-prev-impl.py").write_text(ADAPT_PREV_SOURCE, encoding="utf-8") + (tmp_path / f"{ADAPT_LIB}.py").write_text(ADAPT_NEW_SOURCE, encoding="utf-8") + return [ + "--adapt-new", + str(tmp_path / "adapt-new.json"), + "--adapt-prev", + str(tmp_path / "adapt-prev.json"), + "--adapt-pair", + str(tmp_path / "pair.json"), + "--prev-impl", + str(tmp_path / "anyplot-prev-impl.py"), + "--new-impl", + str(tmp_path / f"{ADAPT_LIB}.py"), + ] + + +def _outputs(path: Path) -> dict[str, str]: + return dict(line.split("=", 1) for line in path.read_text(encoding="utf-8").splitlines()) + + +class TestAdaptCli: + def _decide(self, tmp_path, monkeypatch, capsys, extra: list[str], *, score: str = "85", adapt: bool = True): + out = tmp_path / "gh_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(out)) + (tmp_path / "review_regen.json").write_text(json.dumps(_regen(improvements=[])), encoding="utf-8") + (tmp_path / "review_checklist.json").write_text(json.dumps(_checklist_json(NEW_CHECKLIST)), encoding="utf-8") + args = ["decide", "--spec-id", ADAPT_SPEC, "--library", ADAPT_LIB, "--score", score, "--prev-stored", "94"] + args += ["--regen-json", str(tmp_path / "review_regen.json"), "--prev-renders", "available"] + args += ["--record-out", str(tmp_path / "record.json"), *extra] + if adapt: + args += _adapt_files(tmp_path) + assert main(args) == 0 + notice = next(line for line in capsys.readouterr().out.splitlines() if line.startswith("::notice::regen_gate")) + return notice, _outputs(out), json.loads((tmp_path / "record.json").read_text(encoding="utf-8")) + + def test_decide_with_the_adapt_flags(self, tmp_path, monkeypatch, capsys): + notice, outputs, record = self._decide(tmp_path, monkeypatch, capsys, []) + expected = ( + " code=adapt_identity prev_lines=100 new_lines=110 adapt_counted=true adapt_why=route" + " adapt_overruled=false reason=adapt route:" + ) + assert expected in notice + assert (outputs["verdict"], outputs["code"], outputs["adapt_counted"], outputs["adapt_why"]) == ( + "merge", + "adapt_identity", + "true", + "route", + ) + assert outputs["adapt_overruled"] == "false" + assert record["adapt"]["counted"] is True and record["adapt"]["review"] == "ran" + assert record["adapt"]["route_pre_review"] is False # no --adapt-route-held + assert validate_record(record) == [] + + def test_decide_with_a_held_route_that_the_review_overrules(self, tmp_path, monkeypatch, capsys): + checklist = _checklist_json({**NEW_CHECKLIST, "VQ-01": 7}) + (tmp_path / "x").mkdir() + (tmp_path / "x" / "review_checklist.json").write_text(json.dumps(checklist), encoding="utf-8") + (tmp_path / "x" / "review_regen.json").write_text(json.dumps(_regen(improvements=[])), encoding="utf-8") + # The review's files sit apart, so its checklist (one item below the re-score) is the one the gate reads. + extra = ["--adapt-route-held", "--regen-json", str(tmp_path / "x" / "review_regen.json")] + notice, outputs, record = self._decide(tmp_path, monkeypatch, capsys, extra) + assert " adapt_counted=false adapt_why=technical_below adapt_overruled=true reason=" in notice + assert (outputs["verdict"], outputs["adapt_why"], outputs["adapt_overruled"]) == ( + "keep", + "technical_below", + "true", + ) + assert (record["adapt"]["route_pre_review"], record["adapt"]["counted"]) == (True, False) + assert validate_record(record) == [] + + def test_decide_with_a_held_route_that_counts(self, tmp_path, monkeypatch, capsys): + _, outputs, record = self._decide(tmp_path, monkeypatch, capsys, ["--adapt-route-held"]) + assert (outputs["adapt_counted"], outputs["adapt_overruled"]) == ("true", "false") + assert record["adapt"]["route_pre_review"] is True + + def test_decide_without_a_review(self, tmp_path, monkeypatch, capsys): + notice, outputs, record = self._decide(tmp_path, monkeypatch, capsys, ["--review-skipped"], score="") + assert " prev_rescored=n/a new=n/a verdict=merge code=adapt_identity " in notice + assert (outputs["code"], outputs["adapt_why"]) == ("adapt_identity", "route") + assert (record["new"], record["prev_rescored"], record["adapt"]["review"]) == (None, None, "skipped") + + def test_decide_with_the_review_skipped_and_no_route(self, tmp_path, monkeypatch, capsys): + (tmp_path / "x").mkdir() + files = _adapt_files(tmp_path / "x", prev="pass") + notice, outputs, record = self._decide( + tmp_path, monkeypatch, capsys, ["--review-skipped", *files], score="", adapt=False + ) + assert " verdict=keep code=no_score " in notice and " adapt_counted=false adapt_why=prev_passes " in notice + assert record["adapt"]["why"] == "prev_passes" + + def test_older_argument_sets_print_the_old_notice(self, tmp_path, monkeypatch, capsys): + notice, outputs, record = self._decide(tmp_path, monkeypatch, capsys, [], adapt=False) + assert "adapt_" not in notice and " code=no_visible_improvement reason=" in notice + assert (outputs["adapt_counted"], outputs["adapt_why"], outputs["adapt_overruled"]) == ( + "false", + "no_record", + "false", + ) + assert record["adapt"]["why"] == "no_record" and record["adapt"]["route_pre_review"] is None + + def test_a_non_python_pair_records_not_python(self, tmp_path, monkeypatch, capsys): + (tmp_path / "prev.R").write_text("x <- 1\n", encoding="utf-8") + (tmp_path / "ggplot2.R").write_text("x <- 2\n", encoding="utf-8") + extra = ["--prev-impl", str(tmp_path / "prev.R"), "--new-impl", str(tmp_path / "ggplot2.R")] + notice, outputs, record = self._decide(tmp_path, monkeypatch, capsys, extra, adapt=False) + assert "adapt_" not in notice + assert outputs["adapt_why"] == "not_python" and record["adapt"]["why"] == "not_python" + + def _route(self, tmp_path, monkeypatch, capsys, args: list[str]) -> tuple[str, dict[str, str]]: + out = tmp_path / "gh_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(out)) + assert main(["adapt-route", "--spec-id", ADAPT_SPEC, "--library", ADAPT_LIB, *args]) == 0 + notice = next(line for line in capsys.readouterr().out.splitlines() if line.startswith("::notice::adapt_route")) + return notice, _outputs(out) + + def test_adapt_route(self, tmp_path, monkeypatch, capsys): + notice, outputs = self._route(tmp_path, monkeypatch, capsys, _adapt_files(tmp_path)) + assert notice == ( + "::notice::adapt_route spec=bar-grouped lib=matplotlib prev=fail new=pass light=0.0011 dark=0.0009 " + "lines=100->110 route=true why=route" + ) + assert outputs == {"adapt_route": "true", "adapt_why": "route", "adapt_prev": "fail", "adapt_new": "pass"} + + def test_adapt_route_canvas_failed(self, tmp_path, monkeypatch, capsys): + _, outputs = self._route(tmp_path, monkeypatch, capsys, [*_adapt_files(tmp_path), "--canvas-failed"]) + assert (outputs["adapt_route"], outputs["adapt_why"]) == ("false", "canvas_failed") + + def test_adapt_route_without_files(self, tmp_path, monkeypatch, capsys): + notice, outputs = self._route(tmp_path, monkeypatch, capsys, []) + assert notice.endswith("prev=n/a new=n/a light=n/a dark=n/a lines=n/a->n/a route=false why=no_record") + assert outputs == {"adapt_route": "false", "adapt_why": "no_record", "adapt_prev": "n/a", "adapt_new": "n/a"} + + def test_adapt_route_with_an_unreadable_pair(self, tmp_path, monkeypatch, capsys): + args = _adapt_files(tmp_path) + (tmp_path / "pair.json").write_text("{", encoding="utf-8") + _, outputs = self._route(tmp_path, monkeypatch, capsys, args) + assert (outputs["adapt_route"], outputs["adapt_why"]) == ("false", "pair_missing") + + def test_adapt_route_never_fails(self, tmp_path, monkeypatch, capsys): + def boom(*_args, **_kwargs): + raise RuntimeError("bug") + + monkeypatch.setattr(gate_module, "adapt_assess", boom) + out = tmp_path / "gh_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(out)) + assert main(["adapt-route", "--spec-id", ADAPT_SPEC, "--library", ADAPT_LIB, *_adapt_files(tmp_path)]) == 0 + assert "::warning::adapt-route could not assess the pair, so the route is off: bug" in capsys.readouterr().out + assert _outputs(out)["adapt_route"] == "false" + + def test_adapt_route_usage_error_exits_2(self): + with pytest.raises(SystemExit) as exc: + main(["adapt-route", "--library", ADAPT_LIB]) + assert exc.value.code == 2 + + +class TestAdaptRunnerCopy: + """The runner runs a single-file copy with its own Python, where ``core`` is not importable.""" + + GATE = Path(gate_module.__file__) + + def test_the_gate_imports_only_the_standard_library(self): + """PyYAML is the one exception, imported inside ``cmd_context`` only.""" + tree = ast.parse(self.GATE.read_text(encoding="utf-8")) + context = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "cmd_context") + in_context = {id(n) for n in ast.walk(context)} + modules = { + (node.module if isinstance(node, ast.ImportFrom) else alias.name).split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.Import | ast.ImportFrom) and id(node) not in in_context + for alias in node.names + } + assert modules <= set(sys.stdlib_module_names) | {"__future__"}, modules - set(sys.stdlib_module_names) + + def test_adapt_route_and_decide_run_without_core(self, tmp_path): + copy = tmp_path / "tools" / "regen_gate.py" + copy.parent.mkdir() + copy.write_text(self.GATE.read_text(encoding="utf-8"), encoding="utf-8") + work = tmp_path / "work" + work.mkdir() + files = _adapt_files(work) + out = work / "gh_output" + env = {"GITHUB_OUTPUT": str(out), "PATH": "/usr/bin:/bin"} + route = subprocess.run( + [sys.executable, "-I", str(copy), "adapt-route", "--spec-id", ADAPT_SPEC, "--library", ADAPT_LIB, *files], + cwd=work, + env=env, + capture_output=True, + text=True, + ) + assert route.returncode == 0, route.stderr + assert "adapt_route=true" in out.read_text(encoding="utf-8").splitlines() + args = ["decide", "--spec-id", ADAPT_SPEC, "--library", ADAPT_LIB, "--score", "", "--review-skipped", *files] + decided = subprocess.run( + [sys.executable, "-I", str(copy), *args], cwd=work, env=env, capture_output=True, text=True + ) + assert decided.returncode == 0, decided.stderr + assert "code=adapt_identity" in out.read_text(encoding="utf-8").splitlines() diff --git a/tests/unit/automation/scripts/test_regen_writeback.py b/tests/unit/automation/scripts/test_regen_writeback.py index afeb99e1642..b2cace904ad 100644 --- a/tests/unit/automation/scripts/test_regen_writeback.py +++ b/tests/unit/automation/scripts/test_regen_writeback.py @@ -375,6 +375,14 @@ def test_a_review_without_a_verdict_is_stored_as_approved(self): del review["verdict"] assert wb.apply_review({}, review, 80)["review"]["verdict"] == "APPROVED" + def test_a_carried_review_loses_its_mark(self): + """A review-free adapt merge carried the stored review over; a re-score is a review that ran.""" + old = yaml.safe_load(METADATA_TEXT) + old["review"]["carried"] = True + new = wb.apply_review(old, _prev_review(), 80, "claude-opus-5", "qc-a.aqr-b.sg-c.lib-d") + assert "carried" not in new["review"] + assert old["review"]["carried"] is True # the input is not mutated + def test_cli_writes_the_writer_format(self, tmp_path, capsys): meta, impl, prev = tmp_path / "meta.yaml", tmp_path / "impl.R", tmp_path / "review_prev.json" meta.write_text(METADATA_TEXT, encoding="utf-8") @@ -606,6 +614,27 @@ def test_a_non_integer_score(self): problems = wb.metadata_diff_problems(METADATA_TEXT, wb.dump_metadata(data)) assert any("quality_score at the head must be an integer" in p for p in problems) + def test_the_carried_mark_may_go_but_never_come(self): + carried = yaml.safe_load(METADATA_TEXT) + carried["review"]["carried"] = True + carried_text = wb.dump_metadata(carried) + written = wb.dump_metadata(wb.apply_review(carried, _prev_review(), 80, "claude-opus-5", "")) + assert "carried" not in yaml.safe_load(written)["review"] + assert wb.metadata_diff_problems(carried_text, written) == [] + # A write-back that adds the mark, or keeps it with another value, is refused. + added = yaml.safe_load(_written_back()) + added["review"]["carried"] = True + assert any( + "review key 'carried' changed" in p + for p in wb.metadata_diff_problems(METADATA_TEXT, wb.dump_metadata(added)) + ) + flipped = yaml.safe_load(written) + flipped["review"]["carried"] = False + assert any( + "review key 'carried' changed" in p + for p in wb.metadata_diff_problems(carried_text, wb.dump_metadata(flipped)) + ) + # --------------------------------------------------------------------------- # check-pr diff --git a/tests/unit/core/test_perturbation.py b/tests/unit/core/test_perturbation.py index ec97add6aa5..9f2d0b9600a 100644 --- a/tests/unit/core/test_perturbation.py +++ b/tests/unit/core/test_perturbation.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import datetime from pathlib import Path import numpy as np @@ -10,7 +11,16 @@ import pytest from core.adaptability import check_source -from core.perturbation import MAX_GROUPS, ROW_FACTOR, SCALE, SHIFT, perturbations, single_group_allowed, smoke_source +from core.perturbation import ( + MAX_GROUPS, + ROW_FACTOR, + SCALE, + SHIFT, + loader_dtypes, + perturbations, + single_group_allowed, + smoke_source, +) from core.spec_roles import parse_roles @@ -47,6 +57,7 @@ def test_the_set_and_its_order(self, sample: pd.DataFrame) -> None: "fewer_rows", "more_rows", "scaled", + "missing_nullable", # integer sales: no float64 column, so no missing_number ] def test_the_sample_is_never_changed(self, sample: pd.DataFrame) -> None: @@ -118,7 +129,7 @@ def test_no_category_perturbations_without_categorical_columns(self) -> None: df = pd.DataFrame({"x": np.arange(10.0), "y": np.arange(10.0) ** 2}) roles = parse_roles(["`x` (numeric) - x", "`y` (numeric) - y"]) names = [item.name for item in perturbations(df, {"x": "x", "y": "y"}, roles, "table")] - assert names == ["renamed", "fewer_rows", "more_rows", "scaled"] + assert names == ["renamed", "fewer_rows", "more_rows", "scaled", "missing_number"] def test_generated_excluded_and_network_specs_have_none(self, sample: pd.DataFrame) -> None: """A network block may bind `nodes` beside the edges; its smoke run waits for network eligibility (Copilot).""" @@ -155,6 +166,122 @@ def test_numeric_group_codes_stay_groups_in_more_rows(self) -> None: assert more["more_rows"].df["x"].nunique() == 2 * ROW_FACTOR +MIXED_COLUMNS = {"category": "Quarter", "group": "Product line", "value": "Sales"} + + +class TestMissingValues: + """The loader's dtypes with one missing cell per column of a kind (contract section 1.4, design "Smoke run").""" + + @pytest.fixture + def mixed(self) -> pd.DataFrame: + return pd.DataFrame( + { + "Quarter": ["Q1", "Q2", "Q3", "Q4"] * 3, + "Product line": pd.Categorical(["Electronics"] * 4 + ["Clothing"] * 4 + ["Home & Garden"] * 4), + "Sales": np.linspace(100.0, 210.0, 12), + "Units": np.arange(12), + "Promoted": [True, False] * 6, + "Day": pd.date_range("2024-01-01", periods=12, freq="D"), + } + ) + + def test_the_loader_dtypes(self, mixed: pd.DataFrame) -> None: + cast = loader_dtypes(mixed) + assert {name: str(dtype) for name, dtype in cast.dtypes.items()} == { + "Quarter": "string", + "Product line": "string", + "Sales": "float64", + "Units": "Int64", + "Promoted": "boolean", + "Day": str(mixed["Day"].dtype), # datetimes stay as they are + } + assert not cast.isna().any().any() + + def test_object_columns_read_as_their_values(self) -> None: + df = pd.DataFrame( + { + "ints": pd.Series([1, 2, 3], dtype=object), + "floats": pd.Series([1.5, 2, 3], dtype=object), + "dates": [datetime.date(2024, 1, day) for day in (1, 2, 3)], + "words": pd.Series(["a", "b", None], dtype=object), + } + ) + dtypes = {name: str(dtype) for name, dtype in loader_dtypes(df).dtypes.items()} + assert dtypes == {"ints": "Int64", "floats": "float64", "dates": "object", "words": "string"} + + def test_missing_number_puts_nan_in_every_float_column(self, mixed: pd.DataFrame) -> None: + mixed["Price"] = mixed["Sales"] / 10 + items = {item.name: item for item in perturbations(mixed, MIXED_COLUMNS, ROLES, "table", "group")} + item = items["missing_number"] + assert not item.advisory + missing = item.df.isna() + assert missing.sum().to_dict() == { + "Quarter": 0, + "Product line": 0, + "Sales": 1, + "Units": 0, + "Promoted": 0, + "Day": 0, + "Price": 1, + } + # the i-th float column gets row (len // 2 + i) % len: rows 6 and 7 + assert missing.index[missing["Sales"]].tolist() == [6] and missing.index[missing["Price"]].tolist() == [7] + assert str(item.df["Sales"].dtype) == "float64" + + def test_missing_nullable_is_advisory_and_spreads_over_rows(self, mixed: pd.DataFrame) -> None: + items = {item.name: item for item in perturbations(mixed, MIXED_COLUMNS, ROLES, "table", "group")} + item = items["missing_nullable"] + assert item.advisory + missing = item.df.isna() + assert missing.sum().to_dict() == { + "Quarter": 1, + "Product line": 1, + "Sales": 0, + "Units": 1, + "Promoted": 1, + "Day": 0, + } + rows = { + name: missing.index[missing[name]].tolist()[0] for name in ("Quarter", "Product line", "Units", "Promoted") + } + assert rows == {"Quarter": 6, "Product line": 7, "Units": 8, "Promoted": 9} + assert not missing.all(axis=1).any() + assert item.df["Units"].isna().iloc[8] and item.df["Units"].iloc[8] is pd.NA + assert all(not item.advisory for item in items.values() if item.name != "missing_nullable") + + def test_each_is_left_out_without_a_column_of_its_kind(self) -> None: + roles = parse_roles(["`x` (numeric) - x", "`y` (numeric) - y"]) + floats = pd.DataFrame({"x": np.arange(4.0), "y": np.arange(4.0)}) + ints = pd.DataFrame({"x": np.arange(4), "y": np.arange(4)}) + assert [i.name for i in perturbations(floats, {"x": "x", "y": "y"}, roles, "table")][-1] == "missing_number" + names = [i.name for i in perturbations(ints, {"x": "x", "y": "y"}, roles, "table")] + assert "missing_number" not in names and names[-1] == "missing_nullable" + dates = pd.DataFrame({"d": pd.date_range("2024-01-01", periods=3)}) + assert not {"missing_number", "missing_nullable"} & { + i.name for i in perturbations(dates, {"x": "d"}, roles, "table") + } + + def test_the_sample_is_never_changed(self, mixed: pd.DataFrame) -> None: + before = mixed.copy() + perturbations(mixed, MIXED_COLUMNS, ROLES, "table", "group") + pd.testing.assert_frame_equal(mixed, before) + + def test_matrices_get_no_missing_values(self) -> None: + labels = ["a", "b", "c"] + df = pd.DataFrame(np.eye(3), index=labels, columns=labels) + names = {item.name for item in perturbations(df, {}, [], "matrix")} + assert not names & {"missing_number", "missing_nullable"} + + def test_the_smoke_source_reads_the_nullable_dtypes_back(self, tmp_path: Path, mixed: pd.DataFrame) -> None: + item = {i.name: i for i in perturbations(mixed, MIXED_COLUMNS, ROLES, "table", "group")}["missing_nullable"] + path = tmp_path / "data.pkl" + item.df.to_pickle(path) + source = (FIXTURES / "bar_grouped_matplotlib.py").read_text(encoding="utf-8") + code = smoke_source(source, item, str(path)) + assert f"df = pd.read_pickle({str(path)!r})" in code + pd.testing.assert_frame_equal(pd.read_pickle(path), item.df) + + class TestSingleGroupRule: def test_a_free_group_role(self) -> None: assert single_group_allowed(ROLES, "table", "group") diff --git a/tests/unit/core/test_smoke_run.py b/tests/unit/core/test_smoke_run.py new file mode 100644 index 00000000000..ef537854031 --- /dev/null +++ b/tests/unit/core/test_smoke_run.py @@ -0,0 +1,473 @@ +"""Tests for core/smoke_run.py: the perturbation smoke run, the render pair and the generator's self-check. + +The fixtures are the bar-grouped example of docs/concepts/adaptable-code.md (it passes) and a +variant with a fixed bar width, an unguarded index into the second-largest bar and a callout +typed as "Near-tie in Q3" (it fails). Every run uses the light theme only and a minimal spec, +so the module stays at a few seconds and needs no network. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path + +import pytest + +from core import smoke_run +from core.smoke_run import ( + SmokeRun, + code_sha256, + execute, + execution_env, + line_map, + main, + pair, + pair_exit_code, + run, + selfcheck, + stale_labels, + traceback_error, +) + + +FIXTURES = Path(__file__).parent / "fixtures" / "adaptability" +PASSING = (FIXTURES / "bar_grouped_matplotlib.py").read_text(encoding="utf-8") +SPEC = """# bar-grouped: Grouped Bar Chart + +## Data + +- `category` (categorical) - Labels for each group of bars on the x-axis +- `group` (categorical) - The different series being compared (creates side-by-side bars) +- `value` (numeric) - Heights of the bars + +## Notes + +- Minimal spec for the smoke-run tests. +""" +LIGHT = ("light",) + + +def _replace(source: str, old: str, new: str) -> str: + assert source.count(old) == 1, old + return source.replace(old, new) + + +FAILING = _replace( + _replace( + _replace(PASSING, "width = 0.8 / len(table.columns)", "width = 0.25"), + "if len(table.columns) > 1:\n", + "if True:\n", + ), + 'f"Near-tie in {table.index[tie]}"', + '"Near-tie in Q3"', +) +SECOND_LARGEST_LINE = FAILING.splitlines().index(" gaps = ranked[:, -1] - ranked[:, -2]") + 1 +STATIC_FAILING = _replace( + FAILING, + 'IMPRINT = ["#009E73", "#C475FD", "#4467A3", "#BD8233", "#AE3030", "#2ABCCD", "#954477", "#99B314"]', + 'IMPRINT = ["#009E73", "#C475FD", "#4467A3"]', +) +"""The failing variant with three palette positions, which the static palette rule rejects.""" + + +def _catalogue(root: Path, source: str, library: str = "matplotlib", spec_id: str = "bar-grouped") -> Path: + """`plots//implementations/python/.py` with its specification.md under `root`.""" + spec_dir = root / "plots" / spec_id + impl = spec_dir / "implementations" / "python" / f"{library}.py" + impl.parent.mkdir(parents=True, exist_ok=True) + (spec_dir / "specification.md").write_text(SPEC, encoding="utf-8") + impl.write_text(source, encoding="utf-8") + return impl + + +@pytest.fixture(scope="module") +def passing_run(tmp_path_factory: pytest.TempPathFactory) -> SmokeRun: + return run(_catalogue(tmp_path_factory.mktemp("pass"), PASSING), themes=LIGHT) + + +@pytest.fixture(scope="module") +def failing_run(tmp_path_factory: pytest.TempPathFactory) -> SmokeRun: + return run(_catalogue(tmp_path_factory.mktemp("fail"), FAILING), themes=LIGHT) + + +def _result(smoke: SmokeRun, name: str) -> dict[str, object]: + (found,) = [result for result in smoke.results if result["name"] == name] + return found + + +class TestRun: + def test_the_example_passes_every_binding_perturbation(self, passing_run: SmokeRun) -> None: + record = passing_run.to_dict() + assert (record["status"], record["why"], record["failed"]) == ("pass", "-", []) + assert passing_run.exit_code() == 0 + assert record["shape"] == "table" and record["color_role"] == "group" and record["static_status"] == "pass" + assert record["sample"] == { + "status": "pass", + "rows": 12, + "columns": {"category": "Quarter", "group": "Product line", "value": "Sales (thousand USD)"}, + "error": None, + "line": None, + } + names = [result["name"] for result in passing_run.results] + assert names[:7] == [ + "renamed", + "more_categories", + "fewer_categories", + "single_group", + "fewer_rows", + "more_rows", + "scaled", + ] + assert set(names[7:]) <= {"missing_number", "missing_nullable"} + assert _result(passing_run, "renamed")["canvas"] == {"light": "3200x1800"} + assert _result(passing_run, "renamed")["stale_labels"] == [] + + def test_the_record_carries_the_pinned_fields(self, passing_run: SmokeRun) -> None: + record = passing_run.to_dict() + assert record["v"] == 1 and record["tool"] == "core.smoke_run run" and record["checker_version"] == 1 + assert (record["spec"], record["library"], record["language"], record["themes"]) == ( + "bar-grouped", + "matplotlib", + "python", + ["light"], + ) + assert record["code_sha256"] == code_sha256(PASSING) + for result in passing_run.results: + assert set(result) == { + "name", + "description", + "advisory", + "status", + "rows", + "theme", + "error", + "line", + "code", + "canvas", + "stale_labels", + "seconds", + } + json.dumps(record) # serializable as it stands + + def test_an_unguarded_second_largest_index_fails_the_single_group(self, failing_run: SmokeRun) -> None: + # The design's point: this draft passes every static rule, only the smoke run finds the crash. + assert failing_run.to_dict()["static_status"] == "pass" + assert (failing_run.status, failing_run.why) == ("fail", "perturbation_failed") + assert failing_run.exit_code() == 1 + single = _result(failing_run, "single_group") + assert single["status"] == "fail" and single["theme"] == "light" + assert str(single["error"]).startswith("IndexError") + assert single["line"] == SECOND_LARGEST_LINE # the original source's line, not the smoke file's + assert single["code"] == "gaps = ranked[:, -1] - ranked[:, -2]" + assert "single_group" in failing_run.failed + + def test_a_typed_callout_is_a_stale_label_on_renamed_data(self, failing_run: SmokeRun) -> None: + renamed = _result(failing_run, "renamed") + assert renamed["status"] == "fail" and renamed["line"] is None + assert renamed["stale_labels"] == ["Near-tie in Q3"] + assert renamed["error"] == "stale label: 'Near-tie in Q3' shows the retired category 'Q3'" + assert renamed["canvas"] == {"light": "3200x1800"} # the render itself was fine + + def test_the_summary_names_each_failure(self, failing_run: SmokeRun) -> None: + lines = failing_run.summary() + assert lines[0].startswith("smoke_run bar-grouped matplotlib: fail (perturbation_failed): ") + (single,) = [line for line in lines if line.startswith(" single_group")] + assert f"FAIL line {SECOND_LARGEST_LINE}: IndexError" in single + assert single.endswith("| gaps = ranked[:, -1] - ranked[:, -2]") + + def test_a_block_without_df_fails_the_sample(self, tmp_path: Path) -> None: + source = _replace(PASSING, "df = pd.DataFrame(\n", "frame = pd.DataFrame(\n") + smoke = run(_catalogue(tmp_path, source), themes=LIGHT) + assert (smoke.status, smoke.why, smoke.results) == ("fail", "sample_failed", []) + assert smoke.sample is not None and smoke.sample["status"] == "fail" + assert "binds no df" in str(smoke.sample["error"]) + + def test_a_raising_block_reports_its_line(self, tmp_path: Path) -> None: + source = _replace(PASSING, "df = pd.DataFrame(\n", "raise ValueError('no sample')\ndf = pd.DataFrame(\n") + smoke = run(_catalogue(tmp_path, source), themes=LIGHT) + assert smoke.why == "sample_failed" and smoke.sample is not None + assert smoke.sample["error"] == "ValueError: no sample" + assert smoke.sample["line"] == source.splitlines().index("raise ValueError('no sample')") + 1 + + def test_skip_if_static_fails_runs_nothing(self, tmp_path: Path) -> None: + smoke = run(_catalogue(tmp_path, STATIC_FAILING), themes=LIGHT, skip_if_static_fails=True) + assert (smoke.status, smoke.why, smoke.results, smoke.sample) == ("skipped", "static_failed", [], None) + assert smoke.to_dict()["static_status"] == "fail" and smoke.exit_code() == 0 + + @pytest.mark.parametrize( + ("spec_id", "why"), [("maze-circular", "shape:generated"), ("choropleth-basic", "shape:excluded")] + ) + def test_shapes_without_a_table_skip_without_probing(self, tmp_path: Path, spec_id: str, why: str) -> None: + impl = _catalogue(tmp_path, PASSING) + smoke = run(impl, spec_id=spec_id, themes=LIGHT) + assert (smoke.status, smoke.why, smoke.sample) == ("skipped", why, None) + + def test_a_table_sample_with_nothing_to_perturb_has_its_own_reason( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Not `shape:table`: the shape has tables, only this sample yields none.""" + monkeypatch.setattr(smoke_run, "perturbations", lambda *_args, **_kwargs: []) + smoke = run(_catalogue(tmp_path, PASSING), themes=LIGHT) + assert (smoke.status, smoke.why, smoke.results) == ("skipped", "no_perturbations:table", []) + assert smoke.sample is not None and smoke.sample["status"] == "pass" # the probe ran + assert smoke.exit_code() == 0 + + def test_no_block_is_skipped(self, tmp_path: Path) -> None: + source = _replace(PASSING, "# Plot: reads only df", "# Drawing: reads only df") + smoke = run(_catalogue(tmp_path, source), themes=LIGHT) + assert (smoke.status, smoke.why) == ("skipped", "no_block") + + def test_other_languages_are_not_python(self, tmp_path: Path) -> None: + impl = tmp_path / "ggplot2.R" + impl.write_text("library(ggplot2)\n", encoding="utf-8") + smoke = run(impl) + assert (smoke.status, smoke.why, smoke.exit_code()) == ("skipped", "not_python", 0) + assert smoke.to_dict()["language"] == "r" + + @pytest.mark.parametrize( + ("file_name", "library", "python", "message"), + [ + ("missing.py", "matplotlib", None, "cannot read the source"), + ("plot.py", None, None, "pass --library"), + ("plot.py", "excel", None, "unknown library"), + ("plot.py", "matplotlib", "/nonexistent/python", "no Python interpreter"), + ], + ) + def test_a_run_that_cannot_happen_is_a_runner_error( + self, tmp_path: Path, file_name: str, library: str | None, python: str | None, message: str + ) -> None: + spec = _catalogue(tmp_path, PASSING).parents[2] / "specification.md" + source = tmp_path / file_name + if file_name == "plot.py": + source.write_text(PASSING, encoding="utf-8") + smoke = run(source, spec_path=spec, library=library, python=python) + assert (smoke.status, smoke.why, smoke.exit_code()) == ("skipped", "runner_error", 2) + assert message in str(smoke.error) + assert smoke.to_dict()["error"] == smoke.error + + def test_the_cli_writes_the_record_and_exits_by_status( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + out = tmp_path / "out" / "smoke.json" + impl = tmp_path / "ggplot2.R" + impl.write_text("library(ggplot2)\n", encoding="utf-8") + assert main(["run", str(impl), "--out", str(out)]) == 0 + assert json.loads(out.read_text(encoding="utf-8"))["why"] == "not_python" + assert main(["run", str(tmp_path / "nothing.py"), "--library", "matplotlib", "--out", str(out)]) == 2 + record = json.loads(out.read_text(encoding="utf-8")) + assert (record["status"], record["why"]) == ("skipped", "runner_error") + assert "smoke_run" in capsys.readouterr().out + + +class TestPair: + def test_the_same_source_renders_identical(self, tmp_path: Path) -> None: + new, prev = tmp_path / "new" / "matplotlib.py", tmp_path / "prev" / "matplotlib.py" + for path in (new, prev): + path.parent.mkdir() + path.write_text(PASSING, encoding="utf-8") + out = tmp_path / "adapt" + record = pair(new, prev, out, themes=LIGHT) + assert (record["status"], record["why"], pair_exit_code(record)) == ("identical", "-", 0) + assert (out / "new" / "plot-light.png").is_file() and (out / "prev" / "plot-light.png").is_file() + identity = record["identity"] + assert isinstance(identity, dict) and identity["light"]["fraction"] == 0.0 and identity["light"]["passed"] + new_side = record["new"] + assert isinstance(new_side, dict) and new_side["code_sha256"] == code_sha256(PASSING) + assert new_side["renders"]["light"]["status"] == "ok" and new_side["renders"]["light"]["size"] == [3200, 1800] + + def test_another_palette_is_different(self, tmp_path: Path) -> None: + new, prev = tmp_path / "matplotlib.py", tmp_path / "prev" / "matplotlib.py" + new.write_text(PASSING, encoding="utf-8") + prev.parent.mkdir() + prev.write_text(_replace(PASSING, 'IMPRINT = ["#009E73"', 'IMPRINT = ["#000000"'), encoding="utf-8") + record = pair(new, prev, tmp_path / "adapt", themes=LIGHT) + assert (record["status"], record["why"], pair_exit_code(record)) == ("different", "identity_light", 1) + + def test_a_failing_predecessor_is_an_error(self, tmp_path: Path) -> None: + new, prev = tmp_path / "matplotlib.py", tmp_path / "prev" / "matplotlib.py" + new.write_text(PASSING, encoding="utf-8") + prev.parent.mkdir() + prev.write_text("raise SystemExit('broken predecessor')\n", encoding="utf-8") + record = pair(new, prev, tmp_path / "adapt", themes=LIGHT) + assert (record["status"], record["why"], record["identity"]) == ("error", "prev_render_failed", None) + assert pair_exit_code(record) == 2 + prev_side = record["prev"] + assert isinstance(prev_side, dict) and "broken predecessor" in str(prev_side["error"]) + assert not (tmp_path / "adapt" / "prev" / "plot-light.png").exists() + + def test_a_non_python_library_is_an_error(self, tmp_path: Path) -> None: + record = pair(tmp_path / "ggplot2.R", tmp_path / "prev.R", tmp_path / "adapt", themes=LIGHT) + assert (record["status"], record["why"], pair_exit_code(record)) == ("error", "new_render_failed", 2) + + +class TestSelfcheck: + def test_attempts_are_logged_and_counted(self, tmp_path: Path, passing_run: SmokeRun) -> None: + impl = _catalogue(tmp_path, PASSING) + log, static, smoke = tmp_path / "log.jsonl", tmp_path / "static.json", tmp_path / "smoke.json" + for attempt in (1, 2): + code, lines = selfcheck(impl, log=log, static_out=static, smoke_out=smoke, runner=lambda _: passing_run) + assert code == 0 + assert lines[-1] == f"selfcheck attempt {attempt}: pass" + entries = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + assert [entry["attempt"] for entry in entries] == [1, 2] + assert entries[0]["static"] == "pass" and entries[0]["smoke"] == "pass" and entries[0]["failed"] == [] + assert entries[0]["code_sha256"] == code_sha256(PASSING) + assert set(entries[0]) == { + "at", + "attempt", + "code_sha256", + "static", + "classes", + "advisory", + "smoke", + "smoke_why", + "failed", + "advisory_failed", + } + assert entries[0]["smoke_why"] == "-" + assert json.loads(static.read_text(encoding="utf-8"))["status"] == "pass" + assert json.loads(smoke.read_text(encoding="utf-8"))["status"] == "pass" + + def test_a_static_failure_fails_the_selfcheck(self, tmp_path: Path, passing_run: SmokeRun) -> None: + impl = _catalogue(tmp_path, STATIC_FAILING) + log = tmp_path / "log.jsonl" + code, lines = selfcheck( + impl, + log=log, + static_out=tmp_path / "static.json", + smoke_out=tmp_path / "smoke.json", + runner=lambda _: passing_run, + ) + assert code == 1 + entry = json.loads(log.read_text(encoding="utf-8")) + assert entry["static"] == "fail" and "palette" in entry["classes"] + assert any(line.startswith("CQ-01 (code):") for line in lines) + + def test_a_skipped_shape_passes_but_a_failed_smoke_run_does_not( + self, tmp_path: Path, failing_run: SmokeRun + ) -> None: + impl = _catalogue(tmp_path, PASSING) + paths = {"log": tmp_path / "log.jsonl", "static_out": tmp_path / "s.json", "smoke_out": tmp_path / "m.json"} + skipped = SmokeRun(impl, "matplotlib", LIGHT, status="skipped", why="shape:generated") + assert selfcheck(impl, runner=lambda _: skipped, **paths)[0] == 0 + assert selfcheck(impl, runner=lambda _: failing_run, **paths)[0] == 1 + + @pytest.mark.parametrize( + ("why", "code"), + [ + ("shape:network", 0), + ("no_perturbations:table", 0), # nothing the model can fix + ("no_perturbations:matrix", 0), + ("no_block", 1), + ("static_failed", 1), + ("runner_error", 2), + ], + ) + def test_which_skips_pass_the_selfcheck(self, tmp_path: Path, why: str, code: int) -> None: + impl = _catalogue(tmp_path, PASSING) + paths = {"log": tmp_path / "log.jsonl", "static_out": tmp_path / "s.json", "smoke_out": tmp_path / "m.json"} + skipped = SmokeRun(impl, "matplotlib", LIGHT, status="skipped", why=why) + assert selfcheck(impl, runner=lambda _: skipped, **paths)[0] == code + assert json.loads(paths["log"].read_text(encoding="utf-8"))["smoke_why"] == why + + def test_an_unknown_file_name_is_an_input_error(self, tmp_path: Path) -> None: + assert selfcheck(tmp_path / "plot.py", log=tmp_path / "log.jsonl")[0] == 2 + + +class TestHelpers: + def test_code_sha256_ignores_the_header(self) -> None: + rescored = PASSING.replace("Quality: pending | Created: 2026-10-23", "Quality: 91/100 | Updated: 2026-11-02") + assert rescored != PASSING and code_sha256(rescored) == code_sha256(PASSING) + assert code_sha256(PASSING + "\nprint(1)\n") != code_sha256(PASSING) + assert code_sha256("x = 1\n") == code_sha256("\n\nx = 1") + + def test_the_traceback_names_the_innermost_line_of_the_script(self, tmp_path: Path) -> None: + script = tmp_path / "matplotlib.py" + stderr = ( + "Traceback (most recent call last):\n" + f' File "{script}", line 12, in \n' + " draw()\n" + f' File "{script}", line 7, in draw\n' + " values[3]\n" + " ~~~~~~^^^\n" + ' File "/usr/lib/python3/site-packages/numpy/core.py", line 99, in f\n' + "IndexError: list index out of range\n" + "Exception ignored in: \n" + "Traceback (most recent call last):\n" + ' File "/site-packages/selenium/service.py", line 3, in __del__\n' + "AttributeError: 'NoneType' object has no attribute 'close'\n" + ) + assert traceback_error(stderr, script) == ("IndexError: list index out of range", 7) + + def test_a_syntax_error_has_no_traceback_header(self, tmp_path: Path) -> None: + script = tmp_path / "matplotlib.py" + stderr = f" File \"{script}\", line 3\n x = (\n ^\nSyntaxError: '(' was never closed\n" + assert traceback_error(stderr, script) == ("SyntaxError: '(' was never closed", 3) + + def test_without_a_traceback_the_last_line_is_the_error(self, tmp_path: Path) -> None: + assert traceback_error("warning: x\nsmoke_run: the Data block binds no df\n", tmp_path / "a.py") == ( + "smoke_run: the Data block binds no df", + None, + ) + + def test_line_map_shifts_the_lines_below_the_block(self) -> None: + original = "A = 1 # role: x\n# Data\nx = 1\ny = 2\nz = 3\n# Plot\nplot()\nsave()\n" + executed = "A = 2 # role: x\n# Data\ndf = read()\n\n# Plot\nplot()\nsave()\n" + mapping = line_map(original, executed) + assert mapping[6] == 7 and mapping[7] == 8 # plot() and save() + assert mapping[2] == 2 and mapping[1] == 1 + assert 3 <= mapping[3] <= 5 # inside the replaced block + + def test_stale_labels_skip_prose_numbers_and_the_title(self) -> None: + retired = frozenset({"Q3", "Total", "2024", "A", "Home & Garden"}) + texts = ["Near-tie in Q3", "Q30 forecast", "Total sales", "2024", "A", "Home & Garden", "x · anyplot.ai Q3"] + assert stale_labels(texts, retired) == [("Near-tie in Q3", "Q3"), ("Home & Garden", "Home & Garden")] + + def test_the_execution_env_is_the_pipelines(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHONPATH", "/runner/tools") + env = execution_env("dark") + assert "PYTHONPATH" not in env + assert { + key: env[key] for key in ("ANYPLOT_THEME", "MPLBACKEND", "PYTHONHASHSEED", "PYTHONDONTWRITEBYTECODE") + } == {"ANYPLOT_THEME": "dark", "MPLBACKEND": "Agg", "PYTHONHASHSEED": "0", "PYTHONDONTWRITEBYTECODE": "1"} + + def test_the_actions_runtime_never_reaches_model_written_code(self, monkeypatch: pytest.MonkeyPatch) -> None: + dropped = { + "GITHUB_TOKEN": "ghs_secret", + "GITHUB_REPOSITORY": "owner/repo", + "GITHUB_OUTPUT": "/runner/_temp/out", + "ACTIONS_RUNTIME_TOKEN": "secret", + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://token", + "RUNNER_TEMP": "/runner/_temp", + "GH_TOKEN": "gho_secret", + "github_token": "lower-case too", + } + kept = {"HOME": "/home/runner", "DISPLAY": ":99", "LANG": "C.UTF-8", "MPLCONFIGDIR": "/tmp/mpl", "CI": "true"} + for key, value in {**dropped, **kept}.items(): + monkeypatch.setenv(key, value) + env = execution_env("light") + assert not set(dropped) & set(env) + assert {key: env[key] for key in kept} == kept + assert env["PATH"] == os.environ["PATH"] + + def test_an_execution_sees_no_token(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_TOKEN", "ghs_secret") + code = "import os, sys; sys.exit(1 if any(k.startswith(('GITHUB_', 'GH_')) for k in os.environ) else 0)" + execution = execute([sys.executable, "-c", code], tmp_path, execution_env("light"), timeout=30) + assert execution.returncode == 0, execution.stderr + + def test_a_timeout_kills_the_process_group(self, tmp_path: Path) -> None: + marker = tmp_path / "survived" + child = f"import time, pathlib; time.sleep(1); pathlib.Path({str(marker)!r}).touch()" + code = f"import subprocess, sys, time; subprocess.Popen([sys.executable, '-c', {child!r}]); time.sleep(30)" + execution = execute([sys.executable, "-c", code], tmp_path, execution_env("light"), timeout=0.5) + assert execution.timed_out and execution.seconds < 10 + time.sleep(1.5) + assert not marker.exists() # the grandchild died with the group + + def test_workers_are_fewer_for_chrome_libraries(self) -> None: + assert smoke_run.LIBRARY_WORKERS == {"bokeh": 2, "plotly": 2} + assert smoke_run.DEFAULT_WORKERS == 4 diff --git a/tests/unit/prompts/test_prompts.py b/tests/unit/prompts/test_prompts.py index 2dc7606a8a8..9c1ea483c77 100644 --- a/tests/unit/prompts/test_prompts.py +++ b/tests/unit/prompts/test_prompts.py @@ -18,6 +18,8 @@ from automation.scripts.regen_gate import DEFECT_RE, SUGGESTION_RE from automation.scripts.spec_characteristics_lint import check_contract +from core.adaptability import LIBRARIES as PYTHON_LIBRARIES +from core.adaptability import check_source, spec_context from core.constants import INTERACTIVE_LIBRARIES, LANGUAGE_FILE_EXTENSIONS, LIBRARIES_METADATA, SUPPORTED_LANGUAGES @@ -28,6 +30,7 @@ # Expected files EXPECTED_BASE_PROMPTS = [ "plot-generator.md", + "adaptable-code.md", "quality-criteria.md", "quality-evaluator.md", "spec-id-generator.md", @@ -1275,3 +1278,156 @@ def test_stale_distribution_is_gone_and_the_anchors_stay(self) -> None: assert "When in doubt whether evidence raises a design or library item, keep the default" in criteria assert "Median implementation should score 72-78" in criteria assert "median implementation should score 72-78" in REVIEW_PROMPT.read_text() + + +ADAPTABLE_CODE = PROMPTS_DIR / "adaptable-code.md" +GENERATE_PROMPT = WORKFLOW_PROMPTS_DIR / "impl-generate-claude.md" +REPAIR_PROMPT = WORKFLOW_PROMPTS_DIR / "impl-repair-claude.md" +ALL_LIBRARY_PROMPTS = sorted(path.name for path in LIBRARY_PROMPTS_DIR.glob("*.md")) +PYTHON_AMENDMENT = "For Python (`LANGUAGE=python`)" + + +def _python_block_after(content: str, heading: str) -> str: + """The first ```python block after `heading`.""" + return re.search(r"```python\n(.*?)\n```", content[content.index(heading) :], re.DOTALL).group(1) + + +def _section(content: str, heading: str) -> str: + """The text under a `## ` heading, up to the next `## ` heading.""" + return content.split(f"\n{heading}\n", 1)[1].split("\n## ", 1)[0] + + +class TestAdaptableCode: + """PR 2 of docs/concepts/adaptable-code.md: one copy of the convention, the + generator's examples in its six zones, one Data block idiom per library, + the Python self-check (Step 3c), and the Python-only regen amendments.""" + + def test_the_convention_names_the_six_zones_and_the_markers(self) -> None: + content = ADAPTABLE_CODE.read_text() + for zone in ( + "**Head.**", + "**`# Constants`, optional.**", + "**Column constants.**", + "**Exactly one `# Data` block.**", + "**Plot code, below a `# Plot` marker.**", + "**Output.**", + ): + assert zone in content, zone + assert "# role: " in content + assert "`// Data`, `// Plot`" in content # JavaScript's markers + assert 'never with `bbox_inches="tight"`' in content + + @pytest.mark.parametrize( + ("path", "heading", "spec_id"), + [ + (ADAPTABLE_CODE, "## Example", "bar-grouped"), + (PROMPTS_DIR / "plot-generator.md", "## Output", "scatter-basic"), + ], + ids=["convention-example", "generator-template"], + ) + def test_the_full_examples_pass_the_checker(self, path: Path, heading: str, spec_id: str) -> None: + """The examples the generator copies are the ones the checker passes (core/adaptability.py).""" + source = _python_block_after(path.read_text(), heading) + _, roles, shape = spec_context(PLOTS_DIR / spec_id / "specification.md") + record = check_source(source, roles, shape, library="matplotlib") + assert record.status == "pass", record.defect_lines() + assert set(record.constants) == {role.name for role in roles if role.required} + + @pytest.mark.parametrize( + "path", + [PROMPTS_DIR / "plot-generator.md", PROMPTS_DIR / "default-style-guide.md", ADAPTABLE_CODE] + + [LIBRARY_PROMPTS_DIR / name for name in ALL_LIBRARY_PROMPTS], + ids=lambda p: p.name, + ) + def test_no_prompt_recommends_bbox_inches_tight(self, path: Path) -> None: + """It trims the saved PNG below the fixed canvas; every mention says not to use it.""" + for line in path.read_text().splitlines(): + if re.search(r"bbox_inches\s*=\s*['\"]tight", line): + assert re.search(r"\b(never|not|no|don't)\b", line, re.IGNORECASE), line + + @pytest.mark.parametrize("filename", ALL_LIBRARY_PROMPTS) + def test_every_library_prompt_has_its_data_block_idiom(self, filename: str) -> None: + section = _section((LIBRARY_PROMPTS_DIR / filename).read_text(), "## Data block") + assert "prompts/adaptable-code.md" in section + if filename in JS_LIBRARY_PROMPTS: + assert "// role:" in section and "// Data" in section and "// Plot" in section + assert "const data = window.ANYPLOT_DATA;" in section + assert "d[QUARTER]" in section + elif filename in ("ggplot2.md", "makie.md"): + assert "# role:" in section and "# Data" in section and "# Plot" in section + else: # Python: the zones and the role comments are the convention's example; the section is the idiom + assert "`df`" in section + + def test_fifteen_library_prompts(self) -> None: + python = {f"{library}.md" for library in PYTHON_LIBRARIES} + assert python | set(JS_LIBRARY_PROMPTS) | {"ggplot2.md", "makie.md"} == set(ALL_LIBRARY_PROMPTS) + + def test_r_and_julia_idioms(self) -> None: + ggplot2 = _section((LIBRARY_PROMPTS_DIR / "ggplot2.md").read_text(), "## Data block") + assert "tibble::tibble(" in ggplot2 and ".data[[QUARTER]]" in ggplot2 and "labs(" in ggplot2 + makie = _section((LIBRARY_PROMPTS_DIR / "makie.md").read_text(), "## Data block") + assert "DataFrame(" in makie and "df[!, QUARTER]" in makie + + def test_altair_escapes_field_names(self) -> None: + section = _section((LIBRARY_PROMPTS_DIR / "altair.md").read_text(), "## Data block") + assert 'ESCAPE = str.maketrans({".": "\\\\.", "[": "\\\\[", "]": "\\\\]", ":": "\\\\:"})' in section + assert "title=QUARTER" in section + + def test_no_palette_slice_sized_to_the_sample(self) -> None: + for filename in ("matplotlib.md", "seaborn.md"): + assert "IMPRINT_PALETTE[:N]" not in (LIBRARY_PROMPTS_DIR / filename).read_text(), filename + style = (PROMPTS_DIR / "default-style-guide.md").read_text() + restraint = style[style.index("### Color Restraint") : style.index("### Optional outline pattern")] + assert "list all eight hues and use the first n" in restraint + + def test_the_generator_reads_the_convention(self) -> None: + reading_list = GENERATE_PROMPT.read_text().split("## Step 1:", 1)[1].split("### If regenerating", 1)[0] + assert "`prompts/adaptable-code.md`" in reading_list + generator = (PROMPTS_DIR / "plot-generator.md").read_text() + allowed = generator.split("**Allowed inputs for this implementation:**", 1)[1].split("**Forbidden:**", 1)[0] + assert "`prompts/adaptable-code.md`" in allowed + storytelling = generator.split("**Data storytelling (for DE-03 score):**", 1)[1].split("## Output Files", 1)[0] + assert "**Callout targets come from a rule over the data:**" in storytelling + + def test_step_3c_follows_3b_and_never_blocks(self) -> None: + content = GENERATE_PROMPT.read_text() + start = content.index("### Step 3c: Adaptability self-check (Python only)") + assert content.index("### Step 3b:") < start < content.index("## Step 4:") + step = content[start : content.index("## Step 4:")] + assert ( + "python -m core.smoke_run selfcheck plots/{SPEC_ID}/implementations/python/{LIBRARY}.py " + "--log /tmp/anyplot-selfcheck.jsonl" + ) in step + assert "10-minute timeout" in step + assert "**At most three attempts in total,**" in step + assert "Adaptability never blocks the implementation" in step + assert "python -m core.smoke_run pair --new" in step + assert "Adaptability self-check: pass, attempt 1" in content + + @pytest.mark.parametrize( + "anchor", + [ + "**Keep the data scenario and the variant.**", + "**Don't add code for changes that don't show.**", + "Do NOT discard working structure / data generation / layout choices", + "No defect listed and nothing in Step 0 or the base style to align", + "7. ✅ If regenerating:", + ], + ) + def test_the_five_regen_amendments_are_python_only(self, anchor: str) -> None: + """The same prompt serves every language; R, Julia and JavaScript keep today's regen mindset.""" + line = next(line for line in GENERATE_PROMPT.read_text().splitlines() if anchor in line) + assert PYTHON_AMENDMENT in line + + def test_repair_treats_checker_lines_as_real_defects(self) -> None: + content = REPAIR_PROMPT.read_text() + weaknesses = content[content.index("### Which weaknesses to fix") : content.index("## Step 2")] + assert "is a real defect, like a canvas line" in weaknesses + line = next( + line for line in weaknesses.splitlines() if "**Don't add code for changes that don't show.**" in line + ) + assert PYTHON_AMENDMENT in line + step = content[ + content.index("### Step 5b: Adaptability self-check (Python only)") : content.index("## Step 6:") + ] + assert "python -m core.smoke_run selfcheck" in step and "At most three attempts" in step diff --git a/tests/unit/workflows/test_adapt_generate_steps.py b/tests/unit/workflows/test_adapt_generate_steps.py new file mode 100644 index 00000000000..ce27adb7e94 --- /dev/null +++ b/tests/unit/workflows/test_adapt_generate_steps.py @@ -0,0 +1,487 @@ +"""The adaptability smoke-run steps of impl-generate.yml and impl-repair.yml (PR 2 contract, sections 1.6 and 3). + +Both steps are record-only: they run `core.smoke_run` from the workflow's own ref, write +their JSON under `$RUNNER_TEMP/adapt`, and never fail the job. The upload steps stage the +results privately under `staging/.../adapt/`, after clearing an earlier run's. The step +scripts run here against a scratch origin, with a stub `core/smoke_run.py` in the workflow +ref, a different one on the implementation branch (the generator could have written it), +and a stub `gsutil`, under `bash -e` (the Actions default) and `bash -eo pipefail`. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest +import yaml + + +WORKFLOWS_DIR = Path(__file__).resolve().parents[3] / ".github" / "workflows" +SMOKE_STEP = "Adaptability smoke run (record-only)" +SPEC, LIB = "bar-grouped", "matplotlib" +IMPL = f"plots/{SPEC}/implementations/python/{LIB}.py" +SPEC_FILE = f"plots/{SPEC}/specification.md" +SHELLS = [["bash", "-e", "-c"], ["bash", "-eo", "pipefail", "-c"]] + +STUB_RUNNER = '''"""Stub core.smoke_run: logs its call, writes --out and the pair renders, exits with STUB_EXIT.""" +import json +import os +import pathlib +import sys + +args = sys.argv[1:] +call = {"args": args, "pythonpath": os.environ.get("PYTHONPATH"), "module": __file__} +if "--prev" in args: + call["prev_text"] = pathlib.Path(args[args.index("--prev") + 1]).read_text() +with open(os.environ["STUB_LOG"], "a") as handle: + handle.write(json.dumps(call) + "\\n") +if "--out" in args: + pathlib.Path(args[args.index("--out") + 1]).write_text(json.dumps({"stub": args[0]})) +if args[0] == "pair": + root = pathlib.Path(args[args.index("--out-dir") + 1]) + for side in ("new", "prev"): + (root / side).mkdir(parents=True, exist_ok=True) + (root / side / "plot-light.png").write_bytes(b"png") +sys.exit(int(os.environ.get("STUB_EXIT", "0"))) +''' +BRANCH_RUNNER = """import os, sys +open(os.environ["STUB_LOG"], "a").write('{"args": ["BRANCH-CORE"]}\\n') +sys.exit(0) +""" +FAKE_GSUTIL = """#!/bin/sh +printf '%s\\n' "$*" >> "$GSUTIL_LOG" +case "$*" in + *"cp "*"adapt/new-smoke.json"*) [ -n "$GSUTIL_FAIL_CP" ] && exit 1 ;; + *"rsync "*) [ -n "$GSUTIL_FAIL_CP" ] && exit 1 ;; + *"rm "*) [ -n "$GSUTIL_RM_MISSES" ] && exit 1 ;; +esac +# `rsync -r SRC DST` mirrors the tree: one object per file, named DST/. +for arg in "$@"; do src="$dst"; dst="$arg"; done +case "$*" in + *"rsync -r "*) (cd "$src" && find . -type f | sed "s|^\\./|$dst/|") >> "$GSUTIL_OBJECTS" ;; +esac +exit 0 +""" + + +def _workflow(filename: str) -> dict[str, Any]: + return yaml.safe_load((WORKFLOWS_DIR / filename).read_text(encoding="utf-8")) + + +def _steps(filename: str) -> list[dict[str, Any]]: + return [step for job in _workflow(filename)["jobs"].values() for step in job.get("steps", [])] + + +def _step(filename: str, name: str) -> dict[str, Any]: + matches = [step for step in _steps(filename) if step.get("name") == name] + assert len(matches) == 1, f"expected exactly one step named {name!r} in {filename}" + return matches[0] + + +def _names(filename: str) -> list[str]: + return [str(step.get("name")) for step in _steps(filename)] + + +def _env(**extra: str) -> dict[str, str]: + """No user git config and no inherited GIT_* state.""" + env = {key: value for key, value in os.environ.items() if not key.startswith(("GIT_", "PYTHONPATH"))} + env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, **extra) + return env + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false", *args], + capture_output=True, + text=True, + env=_env(), + check=True, + ) + return result.stdout.strip() + + +def _write(root: Path, files: dict[str, str]) -> None: + for rel, text in files.items(): + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text(text, encoding="utf-8") + + +class Runner: + """A workspace on the implementation branch, an origin/main, and the workflow ref holding the stub runner.""" + + def __init__(self, tmp: Path, *, regeneration: bool) -> None: + self.tmp = tmp + seed = tmp / "seed" + seed.mkdir() + _git(seed, "init", "-q", "-b", "main") + main_files = {"core/__init__.py": "", "core/smoke_run.py": STUB_RUNNER, SPEC_FILE: "# spec\n"} + if regeneration: + main_files[IMPL] = "PREDECESSOR\n" + _write(seed, main_files) + _git(seed, "add", "-A") + _git(seed, "commit", "-q", "-m", "main") + self.sha = _git(seed, "rev-parse", "HEAD") + origin = tmp / "origin.git" + subprocess.run(["git", "clone", "-q", "--bare", str(seed), str(origin)], check=True, env=_env()) + self.ws = tmp / "ws" + subprocess.run(["git", "clone", "-q", str(origin), str(self.ws)], check=True, env=_env()) + _git(self.ws, "checkout", "-q", "-b", f"implementation/{SPEC}/{LIB}") + _write(self.ws, {IMPL: "GENERATED\n", "core/smoke_run.py": BRANCH_RUNNER}) + _git(self.ws, "add", "-A") + _git(self.ws, "commit", "-q", "-m", "feat: implement") + python = self.ws / ".venv" / "bin" / "python" + python.parent.mkdir(parents=True) + python.write_text(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n', encoding="utf-8") + python.chmod(0o755) + self.runner_temp = tmp / "runner" + self.runner_temp.mkdir() + self.log = tmp / "stub.jsonl" + self.selfcheck = tmp / "anyplot-selfcheck.jsonl" + + @property + def out(self) -> Path: + return self.runner_temp / "adapt" + + def calls(self) -> list[dict[str, Any]]: + if not self.log.exists(): + return [] + return [json.loads(line) for line in self.log.read_text(encoding="utf-8").splitlines()] + + def run( + self, workflow: str, shell: list[str], *, sha: str | None = None, stub_exit: int = 0 + ) -> subprocess.CompletedProcess[str]: + step = _step(workflow, SMOKE_STEP) + script = str(step["run"]) + assert "${{" not in script + script = script.replace("/tmp/anyplot-selfcheck.jsonl", str(self.selfcheck)) + return subprocess.run( + [*shell, script], + cwd=self.ws, + env=_env( + RUNNER_TEMP=str(self.runner_temp), + GITHUB_SHA=sha or self.sha, + SPEC_ID=SPEC, + LIBRARY=LIB, + STUB_LOG=str(self.log), + STUB_EXIT=str(stub_exit), + ), + capture_output=True, + text=True, + ) + + +# --- wiring ------------------------------------------------------------------------------------ + + +class TestWiring: + def test_generate_runs_it_after_the_metadata_commit_and_before_the_png_processing(self) -> None: + names = _names("impl-generate.yml") + assert names.count(SMOKE_STEP) == 1 + index = names.index(SMOKE_STEP) + assert names[index - 1] == "Create library metadata file" + assert names[index + 1] == "Process plot images (light + dark)" + assert index < names.index("Upload to GCS Staging") + + def test_repair_runs_it_after_the_repaired_image_and_before_the_upload(self) -> None: + names = _names("impl-repair.yml") + index = names.index(SMOKE_STEP) + assert names[index - 1] == "Process repaired image" + assert index < names.index("Upload repaired plot to GCS staging") + + @pytest.mark.parametrize( + ("workflow", "condition"), + [ + ("impl-generate.yml", "steps.inputs.outputs.language == 'python'"), + ("impl-repair.yml", "success() && steps.lang.outputs.language == 'python'"), + ], + ) + def test_the_steps_are_record_only_and_python_only(self, workflow: str, condition: str) -> None: + step = _step(workflow, SMOKE_STEP) + assert step["id"] == "adapt_smoke" + assert step["if"] == condition + assert step["continue-on-error"] is True + assert step["timeout-minutes"] == 20 + script = str(step["run"]) + assert "exit 1" not in script # never a raw exit 1: the step records, it does not gate + assert 'git archive "$GITHUB_SHA" core' in script # the runner comes from the workflow's ref + assert 'PYTHONPATH="$TOOLS" .venv/bin/python -P -m core.smoke_run' in script + logical = script.replace("\\\n", " ").splitlines() + invocations = [ + line + for line in logical + if ('"${RUN[@]}"' in line or "-m core.smoke_run" in line) and not line.strip().startswith("RUN=") + ] + assert invocations + assert all(line.rstrip().endswith("|| true") for line in invocations), invocations + + def test_repair_runs_no_pair_and_no_predecessor(self) -> None: + script = str(_step("impl-repair.yml", SMOKE_STEP)["run"]) + assert " pair " not in script and "origin/main" not in script and "prev" not in script + + def test_the_metadata_step_is_untouched(self) -> None: + """Four tests read "Create library metadata file"; the smoke run lives in its own step (contract 9.2).""" + assert "smoke_run" not in str(_step("impl-generate.yml", "Create library metadata file")["run"]) + + +# --- the impl-generate step -------------------------------------------------------------------- + + +@pytest.mark.parametrize("shell", SHELLS, ids=["bash-e", "pipefail"]) +class TestGenerateStep: + def test_a_regeneration_runs_new_pair_and_predecessor_from_the_workflow_ref( + self, tmp_path: Path, shell: list[str] + ) -> None: + runner = Runner(tmp_path, regeneration=True) + runner.selfcheck.write_text('{"attempt": 1}\n', encoding="utf-8") + result = runner.run("impl-generate.yml", shell) + assert result.returncode == 0, result.stdout + result.stderr + calls = runner.calls() + assert [call["args"][0] for call in calls] == ["run", "pair", "run"] + tools = str(runner.runner_temp / "adapt-tools") + assert all(call["pythonpath"] == tools for call in calls) + assert all(call["module"] == f"{tools}/core/smoke_run.py" for call in calls) # never the branch's core/ + out = str(runner.out) + new, pair, prev = (call["args"] for call in calls) + assert new == [ + "run", + IMPL, + "--spec", + SPEC_FILE, + "--spec-id", + SPEC, + "--library", + LIB, + "--out", + f"{out}/new-smoke.json", + ] + assert pair[:5] == ["pair", "--new", IMPL, "--prev", f"{out}/prev-src/{LIB}.py"] + assert pair[5:] == ["--library", LIB, "--out-dir", out, "--out", f"{out}/pair.json"] + assert calls[1]["prev_text"] == "PREDECESSOR\n" # origin/main's file, not the generated one + assert prev == [ + "run", + f"{out}/prev-src/{LIB}.py", + "--spec", + SPEC_FILE, + "--spec-id", + SPEC, + "--library", + LIB, + "--skip-if-static-fails", + "--out", + f"{out}/prev-smoke.json", + ] + staged = sorted(str(path.relative_to(runner.out)) for path in runner.out.rglob("*") if path.is_file()) + assert staged == [ + "new-smoke.json", + "new/plot-light.png", + "pair.json", + "prev-smoke.json", + "prev/plot-light.png", + "selfcheck.jsonl", + ] + assert (runner.out / "selfcheck.jsonl").read_text(encoding="utf-8") == '{"attempt": 1}\n' + assert "::notice::adapt_smoke spec=bar-grouped lib=matplotlib staged=" in result.stdout + + def test_a_first_generation_runs_only_the_new_source(self, tmp_path: Path, shell: list[str]) -> None: + runner = Runner(tmp_path, regeneration=False) + result = runner.run("impl-generate.yml", shell) + assert result.returncode == 0, result.stdout + result.stderr + assert [call["args"][0] for call in runner.calls()] == ["run"] + assert sorted(path.name for path in runner.out.iterdir()) == ["new-smoke.json"] # no log, no prev-src + + @pytest.mark.parametrize("stub_exit", [1, 2]) + def test_a_failing_or_erroring_runner_never_fails_the_step( + self, tmp_path: Path, shell: list[str], stub_exit: int + ) -> None: + runner = Runner(tmp_path, regeneration=True) + result = runner.run("impl-generate.yml", shell, stub_exit=stub_exit) + assert result.returncode == 0, result.stdout + result.stderr + assert len(runner.calls()) == 3 + assert not (runner.out / "prev-src").exists() + + def test_without_the_workflow_ref_nothing_runs(self, tmp_path: Path, shell: list[str]) -> None: + runner = Runner(tmp_path, regeneration=True) + result = runner.run("impl-generate.yml", shell, sha="0" * 40) + assert result.returncode == 0, result.stdout + result.stderr + assert "::warning::adapt_smoke could not copy core/" in result.stdout + assert runner.calls() == [] and not runner.out.exists() + + +# --- the impl-repair step ---------------------------------------------------------------------- + + +@pytest.mark.parametrize("shell", SHELLS, ids=["bash-e", "pipefail"]) +class TestRepairStep: + def test_the_repaired_file_gets_one_run(self, tmp_path: Path, shell: list[str]) -> None: + runner = Runner(tmp_path, regeneration=True) + result = runner.run("impl-repair.yml", shell, stub_exit=1) + assert result.returncode == 0, result.stdout + result.stderr + (call,) = runner.calls() + out = str(runner.out) + assert call["args"] == [ + "run", + IMPL, + "--spec", + SPEC_FILE, + "--spec-id", + SPEC, + "--library", + LIB, + "--out", + f"{out}/new-smoke.json", + ] + assert call["module"] == f"{runner.runner_temp}/adapt-tools/core/smoke_run.py" + assert sorted(path.name for path in runner.out.iterdir()) == ["new-smoke.json"] + + def test_without_the_workflow_ref_nothing_runs(self, tmp_path: Path, shell: list[str]) -> None: + runner = Runner(tmp_path, regeneration=True) + result = runner.run("impl-repair.yml", shell, sha="0" * 40) + assert result.returncode == 0 and runner.calls() == [] + + +# --- staging ----------------------------------------------------------------------------------- + + +class Staging: + """An implementation directory with both theme renders, `$RUNNER_TEMP/adapt`, and a stub gsutil.""" + + def __init__(self, tmp: Path, language: str = "python") -> None: + self.ws = tmp / "ws" + self.impl_dir = self.ws / "plots" / SPEC / "implementations" / language + self.impl_dir.mkdir(parents=True) + for theme in ("light", "dark"): + (self.impl_dir / f"plot-{theme}.png").write_bytes(b"png") + self.runner_temp = tmp / "runner" + self.adapt = self.runner_temp / "adapt" + self.adapt.mkdir(parents=True) + bin_dir = tmp / "bin" + bin_dir.mkdir() + (bin_dir / "gsutil").write_text(FAKE_GSUTIL, encoding="utf-8") + (bin_dir / "gsutil").chmod(0o755) + self.path = f"{bin_dir}{os.pathsep}{os.environ['PATH']}" + self.log = tmp / "gsutil.log" + self.objects = tmp / "gsutil.objects" + self.output = tmp / "github_output" + self.language = language + self.staging = f"gs://anyplot-images/staging/{SPEC}/{language}/{LIB}" + + def run(self, workflow: str, step: str, shell: list[str], **extra: str) -> list[str]: + script = str(_step(workflow, step)["run"]) + assert "${{" not in script + result = subprocess.run( + [*shell, script], + cwd=self.ws, + env=_env( + PATH=self.path, + RUNNER_TEMP=str(self.runner_temp), + GSUTIL_LOG=str(self.log), + GSUTIL_OBJECTS=str(self.objects), + GITHUB_OUTPUT=str(self.output), + SPEC_ID=SPEC, + LIBRARY=LIB, + LANGUAGE=self.language, + **extra, + ), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + self.stdout = result.stdout + return self.log.read_text(encoding="utf-8").splitlines() if self.log.exists() else [] + + def object_names(self) -> list[str]: + """The objects an `rsync -r` created, relative to the pair's staging folder.""" + if not self.objects.exists(): + return [] + lines = self.objects.read_text(encoding="utf-8").splitlines() + return sorted(line.removeprefix(f"{self.staging}/") for line in lines) + + +@pytest.mark.parametrize("shell", SHELLS, ids=["bash-e", "pipefail"]) +class TestGenerateUpload: + STEP = "Upload to GCS Staging" + + def test_stale_artifacts_go_first_then_this_runs_are_staged_privately( + self, tmp_path: Path, shell: list[str] + ) -> None: + staging = Staging(tmp_path) + (staging.adapt / "new-smoke.json").write_text("{}", encoding="utf-8") + (staging.adapt / "pair.json").write_text("{}", encoding="utf-8") + for side in ("new", "prev"): + (staging.adapt / side).mkdir() + for theme in ("light", "dark"): + (staging.adapt / side / f"plot-{theme}.png").write_bytes(b"png") + calls = staging.run("impl-generate.yml", self.STEP, shell) + assert calls[0] == f"-m rm -r {staging.staging}/adapt" + (upload,) = [call for call in calls if "rsync" in call] + assert upload == f"-m -h Cache-Control:no-store rsync -r {staging.adapt} {staging.staging}/adapt" + # new/ and prev/ keep their names: the two sides' renders never collide. + assert staging.object_names() == [ + "adapt/new-smoke.json", + "adapt/new/plot-dark.png", + "adapt/new/plot-light.png", + "adapt/pair.json", + "adapt/prev/plot-dark.png", + "adapt/prev/plot-light.png", + ] + assert not [call for call in calls if "acl" in call and "adapt" in call] # never public + assert any("plot-light" in call for call in calls) # the renders still go up + + def test_a_cut_off_step_never_stages_the_predecessor_source(self, tmp_path: Path, shell: list[str]) -> None: + staging = Staging(tmp_path) + (staging.adapt / "new-smoke.json").write_text("{}", encoding="utf-8") + (staging.adapt / "prev-src").mkdir() + (staging.adapt / "prev-src" / f"{LIB}.py").write_text("PREDECESSOR\n", encoding="utf-8") + calls = staging.run("impl-generate.yml", self.STEP, shell) + assert not [call for call in calls if "prev-src" in call] + assert not (staging.adapt / "prev-src").exists() + + def test_without_artifacts_only_the_stale_ones_go(self, tmp_path: Path, shell: list[str]) -> None: + staging = Staging(tmp_path) + calls = staging.run("impl-generate.yml", self.STEP, shell, GSUTIL_RM_MISSES="1") + assert calls[0] == f"-m rm -r {staging.staging}/adapt" + assert not [call for call in calls if "adapt/" in call] + + def test_a_failed_upload_warns_and_the_step_goes_on(self, tmp_path: Path, shell: list[str]) -> None: + staging = Staging(tmp_path) + (staging.adapt / "new-smoke.json").write_text("{}", encoding="utf-8") + calls = staging.run("impl-generate.yml", self.STEP, shell, GSUTIL_FAIL_CP="1") + assert "::warning::adapt artifacts not staged; impl-review records them as missing" in staging.stdout + assert any("plot-light" in call for call in calls) + + +@pytest.mark.parametrize("shell", SHELLS, ids=["bash-e", "pipefail"]) +class TestRepairUpload: + STEP = "Upload repaired plot to GCS staging" + + def test_the_repaired_files_record_replaces_the_staged_one(self, tmp_path: Path, shell: list[str]) -> None: + staging = Staging(tmp_path) + (staging.adapt / "new-smoke.json").write_text("{}", encoding="utf-8") + calls = staging.run("impl-repair.yml", self.STEP, shell) + assert ( + f"-h Cache-Control:no-store cp {staging.adapt}/new-smoke.json {staging.staging}/adapt/new-smoke.json" + in calls + ) + assert not [call for call in calls if call.startswith("rm ")] + + def test_without_a_record_the_stale_one_goes(self, tmp_path: Path, shell: list[str]) -> None: + staging = Staging(tmp_path) + calls = staging.run("impl-repair.yml", self.STEP, shell, GSUTIL_RM_MISSES="1") + assert calls[-1] == f"rm {staging.staging}/adapt/new-smoke.json" + + def test_a_failed_upload_removes_the_stale_record(self, tmp_path: Path, shell: list[str]) -> None: + staging = Staging(tmp_path) + (staging.adapt / "new-smoke.json").write_text("{}", encoding="utf-8") + calls = staging.run("impl-repair.yml", self.STEP, shell, GSUTIL_FAIL_CP="1") + assert calls[-1] == f"rm {staging.staging}/adapt/new-smoke.json" + assert "::warning::adapt/new-smoke.json not staged" in staging.stdout + + def test_other_languages_touch_no_adapt_record(self, tmp_path: Path, shell: list[str]) -> None: + staging = Staging(tmp_path, language="r") + calls = staging.run("impl-repair.yml", self.STEP, shell) + assert calls and not [call for call in calls if "adapt" in call] diff --git a/tests/unit/workflows/test_adapt_review_steps.py b/tests/unit/workflows/test_adapt_review_steps.py new file mode 100644 index 00000000000..5e4fc8d29d8 --- /dev/null +++ b/tests/unit/workflows/test_adapt_review_steps.py @@ -0,0 +1,1011 @@ +"""impl-review's adaptability steps (PR 2 of docs/concepts/adaptable-code.md), run for real where they can be. + +- "Adaptability check (record-only)" runs against a throwaway repository whose + `origin/main` holds the catalogue's bar-grouped matplotlib file (it fails the + check) and whose detached PR head holds the converted fixture of PR 1 (it + passes), with the real checker mirror and record builder in `$RUNNER_TEMP/ + regen-tools/`, a fake `gsutil` serving a local "staging" folder, and a stub + `regen_gate.py` for `adapt-route` (part D of PR 2 implements the real one; one + test runs it once it is there). The step's fixed `/tmp/anyplot-*` paths are + moved under the test's own directory. +- The review-free path: which steps it runs, the carried quality label, the + `regen:adapt` count label, the carried metadata, and what impl-merge reads. + +Every path ends with the step's exit code 0 and `review_skipped` as its last +output: anything short of that leaves the review running. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from automation.scripts import adapt_record as ar +from tests.unit.workflows.test_pipeline_retry_gaps import _calls, _code_only, _run, _step, _steps + + +REPO_ROOT = Path(__file__).parent.parent.parent.parent +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "impl-review.yml" +TOOLS = REPO_ROOT / "automation" / "scripts" +FIXTURE = REPO_ROOT / "tests" / "unit" / "core" / "fixtures" / "adaptability" / "bar_grouped_matplotlib.py" +SPEC_ID, LIB = "bar-grouped", "matplotlib" +IMPL = f"plots/{SPEC_ID}/implementations/python/{LIB}.py" +SPEC = f"plots/{SPEC_ID}/specification.md" +ADAPT = "Adaptability check (record-only)" +SETUP_PYTHON = "Set up Python" +CARRIED_LABEL = "Add carried quality score label" +EARLY = "Add preliminary verdict label (early)" +METADATA = "Update metadata and implementation header" +RESCUE = "Re-dispatch review after a post-score failure" +VERDICT = "Add verdict label and take action" +REVIEW_FREE = "steps.adapt.outputs.review_skipped == 'true'" +PINNED_RECORDS = ("/tmp/anyplot-adapt-new.json", "/tmp/anyplot-adapt-prev.json") + +FAKE_GSUTIL = """\ +printf '%s\\n' "$*" >> "$GSUTIL_LOG" +case "${GSUTIL_MODE:-ok}" in + nomatch) echo "CommandException: No URLs matched: $4" >&2; exit 1 ;; + fail) echo "ServiceException: 503 Service Unavailable" >&2; exit 1 ;; +esac +for dest in "$@"; do :; done +cp -r "$GSUTIL_BUCKET"/. "$dest" +""" + +# adapt-route as part D specifies it (contract section 5.2): exit 0, adapt_* keys to $GITHUB_OUTPUT. +STUB_GATE = """\ +import json, os, sys +with open(os.environ["STUB_LOG"], "a") as f: + f.write(json.dumps(sys.argv[1:]) + "\\n") +if sys.argv[1] != "adapt-route" or os.environ.get("STUB_RC", "0") != "0": + sys.exit(int(os.environ.get("STUB_RC", "2"))) +with open(os.environ["GITHUB_OUTPUT"], "a") as f: + route, why = os.environ.get("STUB_ROUTE", "true"), os.environ.get("STUB_WHY", "route") + f.write(f"adapt_route={route}\\nadapt_why={why}\\nadapt_prev=fail\\nadapt_new=pass\\n") +print(f"::notice::adapt_route spec=s lib=l route={os.environ.get('STUB_ROUTE', 'true')}") +""" + + +def _env(tmp_path: Path) -> dict[str, str]: + empty = tmp_path / "gitconfig" + empty.touch() + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update(GIT_CONFIG_GLOBAL=str(empty), GIT_CONFIG_NOSYSTEM="1") + return env + + +def _git(tmp_path: Path, repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t", *args], + capture_output=True, + text=True, + env=_env(tmp_path), + check=True, + ) + return result.stdout.strip() + + +def _outputs(path: Path) -> list[tuple[str, str]]: + """The `key=value` lines of a $GITHUB_OUTPUT file, in order.""" + if not path.exists(): + return [] + pairs = [line.partition("=") for line in path.read_text(encoding="utf-8").splitlines() if "=" in line] + return [(key, value) for key, _, value in pairs] + + +class Pair: + """A PR head (the converted fixture) over origin/main (the catalogue's file), plus staging.""" + + def __init__(self, tmp_path: Path) -> None: + self.tmp = tmp_path + self.work = tmp_path / "work" + self.runner = tmp_path / "runner" + self.bucket = tmp_path / "bucket" + self.output = tmp_path / "github_output" + self.stub_log = tmp_path / "stub_log" + self.gsutil_log = tmp_path / "gsutil_log" + self.work.mkdir() + _git(tmp_path, self.work, "init", "-q", "-b", "main") + for rel in (IMPL, SPEC): + path = self.work / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text((REPO_ROOT / rel).read_text(encoding="utf-8"), encoding="utf-8") + _git(tmp_path, self.work, "add", "-A") + _git(tmp_path, self.work, "commit", "-q", "-m", "main") + _git(tmp_path, self.work, "update-ref", "refs/remotes/origin/main", "HEAD") + _git(tmp_path, self.work, "checkout", "-q", "--detach") + (self.work / IMPL).write_text(FIXTURE.read_text(encoding="utf-8"), encoding="utf-8") + _git(tmp_path, self.work, "commit", "-q", "-am", "converted") + self.new_text = (self.work / IMPL).read_text(encoding="utf-8") + self.prev_text = (REPO_ROOT / IMPL).read_text(encoding="utf-8") + # The workflow-ref copies "Checkout PR code" takes, with the adapt-route stub. + tools = self.runner / "regen-tools" + tools.mkdir(parents=True) + for name in ("adaptability_check.py", "adapt_record.py"): + shutil.copyfile(TOOLS / name, tools / name) + (tools / "regen_gate.py").write_text(STUB_GATE, encoding="utf-8") + # What impl-generate staged for a regeneration. + (self.bucket / "prev").mkdir(parents=True) + (self.bucket / "new").mkdir() + self.stage("new-smoke.json", self.smoke(self.new_text)) + self.stage("prev-smoke.json", self.smoke(self.prev_text, status="skipped", why="static_failed")) + self.stage("pair.json", {"v": 1, "status": "identical", "why": "-"}) + self.stage("selfcheck.jsonl", '{"static": "fail", "smoke": "pass"}\n{"static": "pass", "smoke": "pass"}\n') + for side in ("prev", "new"): + for theme in ("light", "dark"): + (self.bucket / side / f"plot-{theme}.png").write_bytes(b"\x89PNG") + # Regen context's sanitized predecessor (adapt-route reads its line count). + self.tmp_file("anyplot-prev-impl.py").write_text(self.prev_text, encoding="utf-8") + + def tmp_file(self, name: str) -> Path: + return self.tmp / name + + @staticmethod + def smoke(text: str, **overrides: Any) -> dict[str, Any]: + data = { + "v": 1, + "code_sha256": ar.code_sha256(text), + "status": "pass", + "why": "-", + "results": [{"name": "renamed", "status": "pass", "advisory": False, "line": None, "error": None}], + "failed": [], + "advisory_failed": [], + } + data.update(overrides) + return data + + def stage(self, name: str, data: Any) -> None: + text = data if isinstance(data, str) else json.dumps(data) + (self.bucket / name).write_text(text, encoding="utf-8") + + def run(self, **env: str) -> tuple[subprocess.CompletedProcess[str], dict[str, str], list[tuple[str, str]]]: + script = _step("impl-review.yml", ADAPT)["run"] + for path in (*PINNED_RECORDS, "/tmp/anyplot-prev-impl.py", "/tmp/anyplot-canvas-gate.txt"): + assert path in script, path + script = script.replace("/tmp/anyplot-", f"{self.tmp}/anyplot-") + bin_dir = self.tmp / "bin" + bin_dir.mkdir(exist_ok=True) + (bin_dir / "gsutil").write_text("#!/bin/sh\n" + FAKE_GSUTIL, encoding="utf-8") + (bin_dir / "gsutil").chmod(0o755) + # The step's `python3` is "Set up Python"'s interpreter; here it is this one. + python3 = bin_dir / "python3" + if not python3.exists(): + python3.symlink_to(sys.executable) + defaults = { + "SPEC_ID": SPEC_ID, + "LIBRARY": LIB, + "LANGUAGE": "python", + "IS_REGEN": "true", + "GCS_OK": "true", + "ADAPT_REVIEW_FREE": "true", + "CONTEXT_OUTCOME": "success", + "PREV_STORED": "94", + "CHANGE_REQUEST_PRESENT": "false", + "RUNNER_TEMP": str(self.runner), + "GITHUB_OUTPUT": str(self.output), + "GSUTIL_BUCKET": str(self.bucket), + "GSUTIL_LOG": str(self.gsutil_log), + "STUB_LOG": str(self.stub_log), + } + result, _ = _run(script, self.work, self.tmp, **{**defaults, **env}) + outputs = _outputs(self.output) + return result, dict(outputs), outputs + + def record(self, side: str) -> dict[str, Any] | None: + path = self.tmp_file(f"anyplot-adapt-{side}.json") + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + def route_calls(self) -> list[list[str]]: + if not self.stub_log.exists(): + return [] + return [json.loads(line) for line in self.stub_log.read_text(encoding="utf-8").splitlines()] + + +@pytest.fixture +def pair(tmp_path: Path) -> Pair: + return Pair(tmp_path) + + +def _done(result: subprocess.CompletedProcess[str], ordered: list[tuple[str, str]]) -> None: + """The step ended cleanly, and review_skipped was its last write.""" + assert result.returncode == 0, result.stdout + result.stderr + assert ordered[-1][0] == "review_skipped", ordered + + +class TestAdaptStepRuns: + def test_a_conversion_inside_the_route_skips_the_review(self, pair): + result, out, ordered = pair.run() + _done(result, ordered) + assert out == { + "new_status": "pass", + "prev_status": "fail", + "route": "true", + "why": "route", + "review_skipped": "true", + } + # The route's own outputs went to its file, not to this step's. + assert not any(key.startswith("adapt_") for key, _ in ordered) + assert ( + "::notice::adapt_review spec=bar-grouped lib=matplotlib switch=on route=true review=skipped why=route" + in (result.stdout) + ) + new, prev = pair.record("new"), pair.record("prev") + assert new is not None and prev is not None + assert (new["side"], new["status"], new["static"]["status"], new["smoke"]["status"]) == ( + "new", + "pass", + "pass", + "pass", + ) + assert new["selfcheck"] == {"attempts": 2, "first": "fail"} + assert new["lines"] == len(pair.new_text.splitlines()) + assert (prev["side"], prev["status"], prev["static"]["status"], prev["smoke"]["status"]) == ( + "prev", + "fail", + "fail", + "skipped", + ) + assert prev["lines"] == len(pair.prev_text.splitlines()) + assert "selfcheck" not in prev + assert "side=new status=pass static=pass smoke=pass" in result.stdout + assert "side=prev status=fail static=fail smoke=skipped rules=layout,columns,interface" in result.stdout + + def test_adapt_route_gets_the_contracts_arguments(self, pair): + pair.run() + (call,) = pair.route_calls() + assert call == [ + "adapt-route", + "--spec-id", + SPEC_ID, + "--library", + LIB, + "--adapt-new", + str(pair.tmp_file("anyplot-adapt-new.json")), + "--adapt-prev", + str(pair.tmp_file("anyplot-adapt-prev.json")), + "--adapt-pair", + str(pair.runner / "adapt" / "pair.json"), + "--prev-impl", + str(pair.tmp_file("anyplot-prev-impl.py")), + "--new-impl", + IMPL, + ] + + def test_the_artifacts_stay_under_runner_temp_and_the_source_copy_goes(self, pair): + pair.run() + adapt = pair.runner / "adapt" + for name in ("new-smoke.json", "prev-smoke.json", "pair.json", "static-new.json", "static-prev.json"): + assert (adapt / name).is_file(), name + assert (adapt / "prev" / "plot-light.png").is_file() + # The predecessor's source carries its stored score in the header. + assert not (adapt / "prev-src").exists() + assert sorted(p.name for p in pair.tmp.glob("anyplot-adapt-*")) == [ + "anyplot-adapt-new.json", + "anyplot-adapt-prev.json", + ] + + def test_a_first_generation_has_no_predecessor_and_no_route(self, pair): + pair.tmp_file("anyplot-adapt-prev.json").write_text('{"stale": true}', encoding="utf-8") + result, out, ordered = pair.run(IS_REGEN="false") + _done(result, ordered) + assert out == { + "new_status": "pass", + "prev_status": "n/a", + "route": "n/a", + "why": "n/a", + "review_skipped": "false", + } + assert pair.record("prev") is None # a stale record never stands in + assert pair.route_calls() == [] + assert "route=n/a review=run why=first_generation" in result.stdout + + @pytest.mark.parametrize( + ("env", "why"), + [ + ({"ADAPT_REVIEW_FREE": "false"}, "switch_off"), + ({"ADAPT_REVIEW_FREE": "TRUE"}, "switch_off"), # only the literal `true` + ({"ADAPT_REVIEW_FREE": ""}, "switch_off"), + ({"STUB_ROUTE": "false", "STUB_WHY": "identity_dark"}, "no_route"), + ({"CONTEXT_OUTCOME": "failure"}, "context_failed"), + ({"PREV_STORED": "n/a"}, "no_stored_score"), + ({"PREV_STORED": "0"}, "no_stored_score"), + ({"PREV_STORED": "101"}, "no_stored_score"), + ({"PREV_STORED": "9x"}, "no_stored_score"), + ({"CHANGE_REQUEST_PRESENT": "true"}, "change_request"), + ], + ) + def test_anything_short_of_the_full_set_reviews(self, pair, env, why): + result, out, ordered = pair.run(**env) + _done(result, ordered) + assert out["review_skipped"] == "false" + assert f"review=run why={why}" in result.stdout + if why == "no_route": + assert (out["route"], out["why"]) == ("false", "identity_dark") + + def test_a_failed_canvas_gate_reviews_and_reaches_the_route(self, pair): + pair.tmp_file("anyplot-canvas-gate.txt").write_text("VQ-05 (both): ...", encoding="utf-8") + result, out, ordered = pair.run() + _done(result, ordered) + assert out["review_skipped"] == "false" + assert "review=run why=canvas_failed" in result.stdout + assert pair.route_calls()[0][-1] == "--canvas-failed" + + @pytest.mark.parametrize( + ("env", "why"), + [ + ({"STUB_RC": "2"}, "route_error"), + ({"STUB_RC": "1"}, "route_error"), + ({"STUB_ROUTE": "maybe", "STUB_WHY": "route"}, "route_error"), + ({"STUB_ROUTE": "", "STUB_WHY": ""}, "route_error"), + ({"STUB_WHY": "x y"}, "route_error"), + ({"STUB_WHY": "Route;rm"}, "route_error"), + ({"STUB_WHY": "prev_passes"}, "prev_passes"), # `true` must come with why=route + ], + ) + def test_an_unclear_route_answer_is_no_route(self, pair, env, why): + result, out, ordered = pair.run(**env) + _done(result, ordered) + assert (out["route"], out["why"], out["review_skipped"]) == ("false", why, "false") + + def test_no_staged_artifacts_read_as_missing(self, pair): + # The real gate answers new_unknown for a record without a smoke run. + result, out, ordered = pair.run(GSUTIL_MODE="nomatch", STUB_ROUTE="false", STUB_WHY="new_unknown") + _done(result, ordered) + assert len(pair.gsutil_log.read_text(encoding="utf-8").splitlines()) == 1 # a state, not a blip + assert "no adapt artifacts staged" in result.stdout + new = pair.record("new") + assert new is not None and new["smoke"]["status"] == "missing" and new["selfcheck"] is None + assert (out["new_status"], out["review_skipped"]) == ("unknown", "false") + + def test_a_download_that_keeps_failing_is_retried_and_never_fatal(self, pair): + result, out, ordered = pair.run(GSUTIL_MODE="fail", STUB_ROUTE="false", STUB_WHY="no_record") + _done(result, ordered) + assert len(pair.gsutil_log.read_text(encoding="utf-8").splitlines()) == 3 + assert result.stdout.count("::warning::adapt artifact download failed") == 3 + assert out["new_status"] == "unknown" + + def test_without_gcp_nothing_is_downloaded(self, pair): + result, out, ordered = pair.run(GCS_OK="false", STUB_ROUTE="false", STUB_WHY="no_record") + _done(result, ordered) + assert not pair.gsutil_log.exists() + assert out["new_status"] == "unknown" + + def test_staged_files_never_stand_in_for_the_mirror(self, pair): + """Staging is writable from the generation runner; only this step writes static-*.json and prev-src.""" + pair.stage("static-prev.json", {"version": 1, "status": "pass", "findings": []}) + (pair.bucket / "prev-src").mkdir() + (pair.bucket / "prev-src" / f"{LIB}.py").write_text(pair.new_text, encoding="utf-8") + result, out, ordered = pair.run() + _done(result, ordered) + prev = pair.record("prev") + assert prev is not None and prev["static"]["status"] == "fail" + assert prev["lines"] == len(pair.prev_text.splitlines()) + + def test_a_smoke_run_of_other_code_is_stale(self, pair): + pair.stage("new-smoke.json", pair.smoke(pair.new_text + "# edited after the smoke run\n")) + result, out, ordered = pair.run(STUB_ROUTE="false", STUB_WHY="new_unknown") + _done(result, ordered) + new = pair.record("new") + assert new is not None and (new["status"], new["why"]) == ("unknown", "smoke_stale") + + def test_a_crashing_mirror_is_an_unknown_record(self, pair): + (pair.runner / "regen-tools" / "adaptability_check.py").write_text("import sys\nsys.exit(1)\n") + result, out, ordered = pair.run(STUB_ROUTE="false", STUB_WHY="new_unknown") + _done(result, ordered) + new = pair.record("new") + assert new is not None and new["static"] == {"status": "error", "exit": 1} + assert (out["new_status"], out["prev_status"]) == ("unknown", "unknown") + + def test_a_crashing_record_builder_leaves_no_record_and_reviews(self, pair): + (pair.runner / "regen-tools" / "adapt_record.py").write_text("raise SystemExit(3)\n") + result, out, ordered = pair.run(STUB_ROUTE="false", STUB_WHY="no_record") + _done(result, ordered) + assert pair.record("new") is None and pair.record("prev") is None + assert (out["new_status"], out["prev_status"]) == ("n/a", "n/a") + assert "adapt record (new) not written" in result.stdout + + def test_a_predecessor_missing_on_main_is_unknown(self, pair): + _git(pair.tmp, pair.work, "update-ref", "-d", "refs/remotes/origin/main") + result, out, ordered = pair.run(STUB_ROUTE="false", STUB_WHY="prev_unknown") + _done(result, ordered) + prev = pair.record("prev") + assert prev is not None and prev["static"] == {"status": "error", "exit": 2} and prev["lines"] is None + assert out["prev_status"] == "unknown" + + @staticmethod + def _stage_real_pair(pair, *, new_text: str, prev_text: str) -> None: + """The real gate, and a pair.json as `core.smoke_run pair` writes it for these two sources.""" + shutil.copyfile(TOOLS / "regen_gate.py", pair.runner / "regen-tools" / "regen_gate.py") + identity = {"old_size": [3200, 1800], "new_size": [3200, 1800], "differing": 0, "fraction": 0.0011} + render = {"status": "ok", "size": [3200, 1800], "seconds": 2.0} + side = {"renders": {"light": render, "dark": render}, "error": None, "line": None} + pair.stage( + "pair.json", + { + "v": 1, + "tool": "core.smoke_run pair", + "library": LIB, + "themes": ["light", "dark"], + "new": {**side, "code_sha256": ar.code_sha256(new_text)}, + "prev": {**side, "code_sha256": ar.code_sha256(prev_text)}, + "identity": { + "light": {**identity, "bound": 0.005, "passed": True}, + "dark": {**identity, "bound": 0.005, "passed": True}, + }, + "status": "identical", + "why": "-", + }, + ) + + def test_the_real_gate_routes_this_pair(self, pair): + """Part D's adapt-route on the records this step writes and the pair impl-generate staged.""" + self._stage_real_pair(pair, new_text=pair.new_text, prev_text=pair.prev_text) + result, out, ordered = pair.run() + _done(result, ordered) + assert (out["route"], out["why"]) == ("true", "route"), result.stdout + + def test_the_real_gate_refuses_a_pair_of_other_sources(self, pair): + """A pair rendered from an earlier head measures other code: no route, the review runs.""" + self._stage_real_pair(pair, new_text=pair.new_text + "\n# an earlier head\n", prev_text=pair.prev_text) + result, out, ordered = pair.run() + _done(result, ordered) + assert (out["route"], out["why"], out["review_skipped"]) == ("false", "pair_stale", "false"), result.stdout + + +class TestAdaptStepWiring: + def test_order_canvas_gate_then_mirror_then_the_review(self): + names = [s.get("name") for s in _steps("impl-review.yml")] + canvas = names.index("Canvas dimension gate") + assert names[canvas + 1 : canvas + 3] == [ADAPT, "React with eyes emoji"] + assert names.index("Regen context") < names.index(ADAPT) + assert names.index("Download plot images from staging") < names.index(ADAPT) + assert names.index(ADAPT) < names.index("Hide the previous review") < names.index("Run AI Quality Review") + + def test_python_only_record_only_and_bounded(self): + step = _step("impl-review.yml", ADAPT) + assert step["id"] == "adapt" + assert step["if"] == "steps.lang.outputs.language == 'python'" + assert step["continue-on-error"] is True + assert step["timeout-minutes"] == 5 + assert "exit 1" not in _code_only(step["run"]) + assert step["env"]["ADAPT_REVIEW_FREE"] == "${{ vars.ADAPT_REVIEW_FREE || 'false' }}" + assert step["env"]["CONTEXT_OUTCOME"] == "${{ steps.regen_ctx.outcome }}" + + def test_every_regen_tool_runs_on_the_pinned_python(self): + """PEP 758 `except A, B:` parses on 3.14 only: one unconditional setup before the first tool call.""" + steps = _steps("impl-review.yml") + names = [s.get("name") for s in steps] + setup = _step("impl-review.yml", SETUP_PYTHON) + assert "if" not in setup + assert setup["uses"].startswith("actions/setup-python@") + assert setup["with"] == {"python-version-file": ".python-version"} + assert names.index("Checkout repository") < names.index(SETUP_PYTHON) + users = [i for i, s in enumerate(steps) if "regen-tools/" in str(s.get("run", ""))] + assert users and names.index(SETUP_PYTHON) < min(users) + # No other interpreter: every tool call is the job's python3. + assert [s for s in steps if "setup-python" in str(s.get("uses", ""))] == [setup] + for step in steps: + for line in _code_only(str(step.get("run", ""))).splitlines(): + if re.search(r"regen-tools/\w+\.py", line) and "cp " not in line: + assert re.search(r"python3 \S*regen-tools/", line) or "TOOL" in line, (step.get("name"), line) + + def test_the_regen_tools_run_from_the_workflow_ref(self): + script = _step("impl-review.yml", ADAPT)["run"] + for tool in ("adaptability_check.py", "adapt_record.py", "regen_gate.py"): + assert f'"$RUNNER_TEMP/regen-tools/{tool}"' in script, tool + assert "automation/scripts/" not in script + + def test_the_review_never_sees_the_adapt_artifacts(self): + download = _step("impl-review.yml", "Download plot images from staging")["run"] + assert 'SRC="gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}/plot-*"' in download + assert "/${LIBRARY}/*" not in download + adapt = _step("impl-review.yml", ADAPT)["run"] + assert 'OUT="$RUNNER_TEMP/adapt"' in adapt + assert 'SRC="gs://anyplot-images/staging/${SPEC_ID}/${LANGUAGE}/${LIBRARY}/adapt/*"' in adapt + assert 'gsutil -m cp -r "$SRC" "$OUT/"' in adapt + prompt = _step("impl-review.yml", "Run AI Quality Review")["with"]["prompt"] + assert "adapt" not in prompt.lower() + + def test_impl_merge_promotes_only_the_renders(self): + script = _step("impl-merge.yml", "Promote GCS images to production")["run"] + code = _code_only(script) + assert 'cp "${STAGING}/plot-*" "${PRODUCTION}/"' in code + assert '"${STAGING}/*"' not in code + assert 'gsutil -m rm -r "${STAGING}/"' in code # the cleanup still removes adapt/ + + +class TestReviewFreeWiring: + def test_the_review_is_skipped_only_on_the_switch(self): + assert _step("impl-review.yml", "Run AI Quality Review")["if"] == "steps.adapt.outputs.review_skipped != 'true'" + + @pytest.mark.parametrize( + "name", + [ + "Regen gate", + "Stage regen pair", + EARLY, + "Install Python dependencies for metadata update", + METADATA, + RESCUE, + VERDICT, + ], + ) + def test_the_review_free_path_runs_these(self, name): + assert REVIEW_FREE in _step("impl-review.yml", name)["if"], name + + @pytest.mark.parametrize( + "name", + [ + "Resolve review model", + "Extract quality score", + "Validate review output", + "Add quality score label", + "Check review feedback format (never gating)", + "Write back the re-score (regen keep)", + ], + ) + def test_and_never_these(self, name): + assert "review_skipped" not in _step("impl-review.yml", name).get("if", ""), name + + def test_a_review_free_failure_before_the_verdict_is_re_dispatched(self): + """Without it, a failed carried label or early verdict left the PR with no verdict and no retry.""" + step = _step("impl-review.yml", RESCUE) + assert step["if"] == ( + "failure() && steps.early_verdict.conclusion != 'success' && " + "((steps.review.conclusion == 'success' && steps.score.outputs.has_output == 'true') " + f"|| {REVIEW_FREE})" + ) + assert step["env"]["SCORE"] == _step("impl-review.yml", METADATA)["env"]["SCORE"] + assert step["env"]["REVIEW_SKIPPED"] == "${{ steps.adapt.outputs.review_skipped }}" + + def test_the_gate_and_the_metadata_need_a_regeneration_and_a_merge(self): + assert _step("impl-review.yml", "Regen gate")["if"].startswith("steps.regen.outputs.is_regen == 'true' && (") + cond = _step("impl-review.yml", METADATA)["if"] + assert f"({REVIEW_FREE} && steps.gate.outputs.verdict == 'merge')" in cond + + def test_the_gate_gets_the_adapt_inputs(self): + step = _step("impl-review.yml", "Regen gate") + script = step["run"] + assert step["env"]["REVIEW_SKIPPED"] == "${{ steps.adapt.outputs.review_skipped }}" + for line in ( + "[ -s /tmp/anyplot-adapt-new.json ] && ADAPT+=(--adapt-new /tmp/anyplot-adapt-new.json)", + "[ -s /tmp/anyplot-adapt-prev.json ] && ADAPT+=(--adapt-prev /tmp/anyplot-adapt-prev.json)", + '[ -s "$RUNNER_TEMP/adapt/pair.json" ] && ADAPT+=(--adapt-pair "$RUNNER_TEMP/adapt/pair.json")', + '[ "$REVIEW_SKIPPED" = "true" ] && ADAPT+=(--review-skipped)', + '[ "$ADAPT_ROUTE" = "true" ] && ADAPT+=(--adapt-route-held)', + '"${FLAGS[@]}" "${ADAPT[@]}"; then', + ): + assert line in script, line + # The pre-review route, as the adapt step wrote it before the review ran. + assert step["env"]["ADAPT_ROUTE"] == "${{ steps.adapt.outputs.route }}" + + def test_the_pair_artifact_carries_the_adapt_folder(self): + script = _step("impl-review.yml", "Stage regen pair")["run"] + assert 'cp -r "$RUNNER_TEMP/adapt/." "$DIR/adapt/"' in script + assert 'rm -rf "$DIR/adapt/prev-src"' in script + assert 'cp /tmp/anyplot-adapt-new.json /tmp/anyplot-adapt-prev.json "$DIR/adapt/"' in script + + def test_the_carried_label_comes_after_the_gate_and_before_any_verdict(self): + names = [s.get("name") for s in _steps("impl-review.yml")] + assert names.index("Regen gate") < names.index("Upload regen pair") < names.index(CARRIED_LABEL) + assert names.index(CARRIED_LABEL) + 1 == names.index(EARLY) + step = _step("impl-review.yml", CARRIED_LABEL) + assert step["if"] == f"{REVIEW_FREE} && steps.gate.outputs.verdict == 'merge'" + assert step["env"]["SCORE"] == "${{ steps.regen_ctx.outputs.prev_stored }}" + assert "continue-on-error" not in step + + def test_the_metadata_step_carries_on_the_review_free_path(self): + env = _step("impl-review.yml", METADATA)["env"] + assert env["REVIEW_CARRIED"] == f"${{{{ {REVIEW_FREE} }}}}" + assert env["CARRY_FROM"] == "/tmp/anyplot-prev-metadata.yaml" + assert ( + env["SCORE"] + == f"${{{{ {REVIEW_FREE} && steps.regen_ctx.outputs.prev_stored || steps.score.outputs.score }}}}" + ) + assert ( + 'git show "origin/main:${META_FILE}" > /tmp/anyplot-prev-metadata.yaml' + in _step("impl-review.yml", "Regen context")["run"] + ) + + +class TestReviewFreeRescue: + """The rescue step's script on the review-free path: the stored score, never a review score.""" + + ENV = { + "PR_NUM": "7", + "SPEC_ID": "spec-a", + "LIBRARY": "plotly", + "REPOSITORY": "owner/repo", + "RUN_ID": "1", + "MODEL": "sonnet", + "REVIEW_MODEL_ALIAS": "opus", + "WORKFLOW_REF": "main", + } + + def _rescue(self, tmp_path: Path, **env: str): + return _run(_step("impl-review.yml", RESCUE)["run"], tmp_path, tmp_path, **{**self.ENV, **env}) + + def _comments(self, calls: list[list[str]]) -> list[str]: + return [Path(c[c.index("--body-file") + 1]).read_text(encoding="utf-8") for c in _calls(calls, "pr", "comment")] + + def test_the_review_free_path_re_dispatches_once_with_the_stored_score(self, tmp_path): + result, calls = self._rescue(tmp_path, REVIEW_SKIPPED="true", SCORE="94", GH_LABELS="regen quality:94") + assert result.returncode == 1, result.stdout + result.stderr # the run stays red + (body,) = self._comments(calls) + assert "No review ran (the adapt route skipped it; stored score 94/100), but a later step failed" in body + assert "impl-review is dispatched once more, and decides again whether the review runs." in body + assert "The review scored" not in body + assert len(_calls(calls, "workflow", "run")) == 1 + + def test_a_spent_budget_says_no_review_ran(self, tmp_path): + result, calls = self._rescue(tmp_path, REVIEW_SKIPPED="true", SCORE="", GH_MARKERS="1") + assert result.returncode == 1 + (body,) = self._comments(calls) + assert body.splitlines()[3].startswith("No review ran (the adapt route skipped it; stored score n/a/100), but") + assert not _calls(calls, "workflow", "run") + + def test_the_reviewed_path_keeps_its_wording(self, tmp_path): + _, calls = self._rescue(tmp_path, REVIEW_SKIPPED="", SCORE="86") + (body,) = self._comments(calls) + assert "The review scored 86/100, but a later step failed" in body + assert "A fresh review is dispatched once." in body + + +class TestCarriedQualityLabel: + ENV = {"PR_NUM": "7", "SCORE": "94"} + + def _label(self, tmp_path: Path, **env: str): + return _run(_step("impl-review.yml", CARRIED_LABEL)["run"], tmp_path, tmp_path, **{**self.ENV, **env}) + + def test_the_stored_score_labels_the_pr(self, tmp_path): + result, calls = self._label(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + create = [ + "label", + "create", + "quality:94", + "--force", + "--color", + "0e8a16", + "--description", + "Quality score 94/100", + ] + assert create in calls + assert _calls(calls, "pr", "edit") == [["pr", "edit", "7", "--add-label", "quality:94"]] + + def test_an_older_quality_label_goes_first(self, tmp_path): + # The fake gh prints $GH_LABELS for the `--jq` read, which selects the other quality:* labels. + result, calls = self._label(tmp_path, GH_LABELS="quality:88") + assert result.returncode == 0, result.stdout + result.stderr + assert _calls(calls, "pr", "edit") == [ + ["pr", "edit", "7", "--remove-label", "quality:88"], + ["pr", "edit", "7", "--add-label", "quality:94"], + ] + + @pytest.mark.parametrize("score", ["", "n/a", "0", "101"]) + def test_no_score_stops_before_any_verdict(self, tmp_path, score): + result, calls = self._label(tmp_path, SCORE=score) + assert result.returncode == 1 + assert not calls + + def test_a_lost_label_fails_the_step(self, tmp_path): + result, calls = self._label(tmp_path, GH_FAIL_ADD_LABEL="quality:94") + assert result.returncode == 1 + assert calls.count(["pr", "edit", "7", "--add-label", "quality:94"]) == 4 + + +class TestAdaptCountLabel: + """regen:adapt between regen:improved and ai-approved, best effort.""" + + ENV = {"PR_NUM": "7", "SCORE": "", "ATTEMPT_COUNT": "0", "IS_REGEN": "true", "GATE_VERDICT": "merge"} + + def _early(self, tmp_path: Path, **env: str): + defaults = {"GITHUB_OUTPUT": str(tmp_path / "github_output"), "RUNNER_TEMP": str(tmp_path / "runner")} + return _run(_step("impl-review.yml", EARLY)["run"], tmp_path, tmp_path, **{**self.ENV, **defaults, **env}) + + def test_a_counted_merge_is_labelled_in_order(self, tmp_path): + result, calls = self._early(tmp_path, ADAPT_COUNTED="true") + assert result.returncode == 0, result.stdout + result.stderr + assert [c[-1] for c in _calls(calls, "pr", "edit")] == ["regen:improved", "regen:adapt", "ai-approved"] + assert any(c[:4] == ["label", "create", "regen:adapt", "--force"] for c in calls) + + @pytest.mark.parametrize("counted", ["false", ""]) + def test_an_uncounted_merge_is_not(self, tmp_path, counted): + _, calls = self._early(tmp_path, ADAPT_COUNTED=counted) + assert [c[-1] for c in _calls(calls, "pr", "edit")] == ["regen:improved", "ai-approved"] + + def test_a_lost_count_label_never_holds_up_the_merge(self, tmp_path): + result, calls = self._early(tmp_path, ADAPT_COUNTED="true", GH_FAIL_ADD_LABEL="regen:adapt") + assert result.returncode == 0, result.stdout + result.stderr + assert calls.count(["pr", "edit", "7", "--add-label", "regen:adapt"]) == 3 + assert ["pr", "edit", "7", "--add-label", "ai-approved"] in calls + assert "::error::" not in result.stdout + + def test_a_kept_regeneration_gets_no_count(self, tmp_path): + _, calls = self._early(tmp_path, ADAPT_COUNTED="true", GATE_VERDICT="keep") + assert [c[-1] for c in _calls(calls, "pr", "edit")] == ["regen:kept"] + + def test_the_label_comes_from_the_gate(self): + assert _step("impl-review.yml", EARLY)["env"]["ADAPT_COUNTED"] == "${{ steps.gate.outputs.adapt_counted }}" + assert _step("impl-review.yml", VERDICT)["env"]["ADAPT_COUNTED"] == "${{ steps.gate.outputs.adapt_counted }}" + overruled = "${{ steps.gate.outputs.adapt_overruled }}" + assert _step("impl-review.yml", EARLY)["env"]["ADAPT_OVERRULED"] == overruled + assert _step("impl-review.yml", VERDICT)["env"]["ADAPT_OVERRULED"] == overruled + + @pytest.mark.parametrize( + ("verdict", "labels"), + [ + ("keep", ["regen:adapt-overruled", "regen:kept"]), + ("merge", ["regen:improved", "regen:adapt-overruled", "ai-approved"]), + ], + ) + def test_an_overruled_route_is_labelled_before_the_verdict(self, tmp_path, verdict, labels): + result, calls = self._early(tmp_path, ADAPT_OVERRULED="true", GATE_VERDICT=verdict) + assert result.returncode == 0, result.stdout + result.stderr + assert [c[-1] for c in _calls(calls, "pr", "edit")] == labels + assert any(c[:4] == ["label", "create", "regen:adapt-overruled", "--force"] for c in calls) + assert "regen:adapt-overruled" in next(line for line in result.stdout.splitlines() if "Regen gate:" in line) + + @pytest.mark.parametrize("overruled", ["false", ""]) + def test_a_route_that_was_not_overruled_gets_no_label(self, tmp_path, overruled): + _, calls = self._early(tmp_path, ADAPT_OVERRULED=overruled, GATE_VERDICT="keep") + assert [c[-1] for c in _calls(calls, "pr", "edit")] == ["regen:kept"] + + @pytest.mark.parametrize("verdict", ["keep", "merge"]) + def test_a_lost_overruled_label_never_holds_up_the_verdict(self, tmp_path, verdict): + result, calls = self._early( + tmp_path, ADAPT_OVERRULED="true", GATE_VERDICT=verdict, GH_FAIL_ADD_LABEL="regen:adapt-overruled" + ) + assert result.returncode == 0, result.stdout + result.stderr + assert calls.count(["pr", "edit", "7", "--add-label", "regen:adapt-overruled"]) == 3 + final = "regen:kept" if verdict == "keep" else "ai-approved" + assert ["pr", "edit", "7", "--add-label", final] in calls + assert "::warning::regen:adapt-overruled not added to PR #7" in result.stdout + assert "::error::" not in result.stdout + + +def _writer() -> str: + run = _step("impl-review.yml", METADATA)["run"] + return run.split("cat > /tmp/update_metadata.py << 'EOF'\n", 1)[1].split("\nEOF\n", 1)[0] + + +PREV_REVIEW = { + "strengths": ["clean"], + "weaknesses": ["Suggestion: larger legend"], + "image_description": "Light render: ...\n\nDark render: ...", + "criteria_checklist": {"visual_quality": {"score": 30, "max": 30, "items": []}}, + "verdict": "APPROVED", + "model": "claude-opus-5-5", + "criteria_version": "qc-e1373b1495.aqr-15492a059e.sg-7006008fc6.lib-e97ebfa5c3", + "rendered_at": "2026-10-01T01:58:44Z", +} +PREV_META = { + "library": LIB, + "created": "2026-05-28T00:00:00Z", + "updated": "2026-10-01T02:00:00Z", + "library_version": "3.10.9", + "language_version": "3.13.14", + "generated_by": "claude-sonnet", + "workflow_run": 100, + "quality_score": 94, + "review": PREV_REVIEW, + "impl_tags": {"techniques": ["annotations"], "dependencies": []}, +} +# What impl-generate wrote on the PR branch: measured versions, no score, an empty review. +BRANCH_META = { + "library": LIB, + "language": "python", + "specification_id": SPEC_ID, + "created": "2026-05-28T00:00:00Z", + "updated": "2026-10-24T09:00:00Z", + "generated_by": "claude-sonnet", + "workflow_run": 200, + "issue": 42, + "language_version": "3.14.9", + "library_version": "3.11.1", + "preview_url_light": "https://storage.googleapis.com/anyplot-images/plots/bar-grouped/python/matplotlib/plot-light.png", + "preview_url_dark": "https://storage.googleapis.com/anyplot-images/plots/bar-grouped/python/matplotlib/plot-dark.png", + "preview_html_light": None, + "preview_html_dark": None, + "quality_score": None, + "review": {"strengths": [], "weaknesses": []}, +} + + +class TestCarriedMetadata: + """Contract section 7: what the merged metadata holds after a review-free merge.""" + + def _write(self, tmp_path: Path, score: str, env: dict[str, str], prev: dict[str, Any] = PREV_META): + (tmp_path / "update_metadata.py").write_text(_writer(), encoding="utf-8") + meta = tmp_path / "meta.yaml" + meta.write_text(yaml.safe_dump(BRANCH_META, sort_keys=False), encoding="utf-8") + carry = tmp_path / "prev-metadata.yaml" + carry.write_text(yaml.safe_dump(prev, sort_keys=False), encoding="utf-8") + # A stray review file never reaches a carried review. + (tmp_path / "review_strengths.json").write_text('["planted"]', encoding="utf-8") + base = { + k: v + for k, v in os.environ.items() + if k not in {"REVIEW_MODEL", "CRITERIA_VERSION", "RENDERED_AT", "REVIEW_APPROVED", "REVIEW_CARRIED"} + } + result = subprocess.run( + [sys.executable, "update_metadata.py", str(meta), score, "2026-10-24T09:30:11Z"], + cwd=tmp_path, + env={**base, "CARRY_FROM": str(carry), **env}, + capture_output=True, + text=True, + ) + data = yaml.safe_load(meta.read_text(encoding="utf-8")) + return result, data, meta.read_text(encoding="utf-8") + + def test_the_stored_review_carries_over_unchanged(self, tmp_path): + env = { + "REVIEW_CARRIED": "true", + # This run's values never overwrite or pop the carried provenance. + "REVIEW_APPROVED": "true", + "REVIEW_MODEL": "", + "CRITERIA_VERSION": "", + "RENDERED_AT": "2026-10-24T09:01:00Z", + } + result, data, _ = self._write(tmp_path, "94", env) + assert result.returncode == 0, result.stderr + assert data["quality_score"] == 94 + assert data["review"] == {**PREV_REVIEW, "carried": True} + assert data["impl_tags"] == PREV_META["impl_tags"] + assert data["updated"] == "2026-10-24T09:30:11Z" + for key in ( + "library_version", + "language_version", + "generated_by", + "workflow_run", + "created", + "preview_url_light", + "preview_url_dark", + "preview_html_light", + "preview_html_dark", + "issue", + ): + assert data[key] == BRANCH_META[key], key + + def test_the_merged_file_reads_like_any_stored_review(self, tmp_path): + """impl-merge merges the file as it is; sync_to_postgres and the next regen read it.""" + _, data, text = self._write(tmp_path, "94", {"REVIEW_CARRIED": "true"}) + assert "rendered_at: '2026-10-01T01:58:44Z'" in text + assert isinstance(data["quality_score"], int) and data["quality_score"] > 0 + assert data["review"]["criteria_checklist"] and data["review"]["weaknesses"] + + @pytest.mark.parametrize(("score", "prev_score"), [("93", 94), ("94", None), ("94", "94"), ("0", 0), ("94", True)]) + def test_a_score_that_is_not_the_stored_one_fails(self, tmp_path, score, prev_score): + result, data, _ = self._write( + tmp_path, score, {"REVIEW_CARRIED": "true"}, prev={**PREV_META, "quality_score": prev_score} + ) + assert result.returncode != 0 + assert "carried review:" in result.stderr + assert data == BRANCH_META # nothing written + + def test_a_predecessor_without_tags_or_review(self, tmp_path): + prev = {k: v for k, v in PREV_META.items() if k not in ("impl_tags", "review")} + result, data, _ = self._write(tmp_path, "94", {"REVIEW_CARRIED": "true"}, prev=prev) + assert result.returncode == 0, result.stderr + assert data["review"] == {"carried": True} + assert "impl_tags" not in data + + @pytest.mark.parametrize("carried", ["false", "", "1"]) + def test_a_review_that_ran_is_never_carried(self, tmp_path, carried): + (tmp_path / "update_metadata.py").write_text(_writer(), encoding="utf-8") + meta = tmp_path / "meta.yaml" + meta.write_text(yaml.safe_dump({**BRANCH_META, "review": {**PREV_REVIEW, "carried": True}}), encoding="utf-8") + (tmp_path / "review_strengths.json").write_text('["new"]', encoding="utf-8") + result = subprocess.run( + [sys.executable, "update_metadata.py", str(meta), "91", "2026-10-24T09:30:11Z"], + cwd=tmp_path, + env={**os.environ, "REVIEW_CARRIED": carried, "REVIEW_MODEL": "claude-opus-5-5"}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + data = yaml.safe_load(meta.read_text(encoding="utf-8")) + assert data["quality_score"] == 91 + assert data["review"]["strengths"] == ["new"] + assert "carried" not in data["review"] + + def test_the_header_and_commit_follow_the_mode(self): + script = _step("impl-review.yml", METADATA)["run"] + # The header code writes "Quality: ${SCORE}/100" from the measured versions, as on every path. + assert ( + "export IMPL_FILE LANGUAGE SPEC_ID TITLE LIBRARY LIBRARY_VERSION LANGUAGE_VERSION SCORE DATE_INFO" in script + ) + assert 'git commit -m "chore(${LIBRARY}): carry the stored review over for ${SPEC_ID}"' in script + + +class TestVerdictStepReviewFree: + ENV = { + "PR_NUM": "7", + "SPEC_ID": "spec-a", + "LIBRARY": "plotly", + "LANGUAGE": "python", + "EXT": ".py", + "SCORE": "", + "ATTEMPT": "1", + "ATTEMPT_COUNT": "0", + "ISSUE_NUMBER": "", + "REPOSITORY": "owner/repo", + "RUN_ID": "1", + "MODEL": "sonnet", + "IS_REGEN": "true", + "GATE_REASON": "adapt route", + "PREV_STORED": "94", + "MODEL_ID": "n/a", + "REVIEW_SKIPPED": "true", + } + + def _verdict(self, tmp_path: Path, **env: str): + runner = tmp_path / "runner" / "regen-tools" + runner.mkdir(parents=True, exist_ok=True) + shutil.copyfile(TOOLS / "regen_gate.py", runner / "regen_gate.py") + # The step's fixed /tmp/anyplot-* files, moved under this test's directory (parallel runs). + script = _step("impl-review.yml", VERDICT)["run"].replace("/tmp/anyplot-", f"{tmp_path}/anyplot-") + return _run(script, tmp_path, tmp_path, **{**self.ENV, "RUNNER_TEMP": str(tmp_path / "runner"), **env}) + + def _comment(self, calls: list[list[str]]) -> str: + (comment,) = _calls(calls, "pr", "comment") + return Path(comment[comment.index("--body-file") + 1]).read_text(encoding="utf-8") + + def test_a_review_free_merge_says_no_review_ran(self, tmp_path): + result, calls = self._verdict(tmp_path, GATE_VERDICT="merge", GATE_CODE="adapt_identity") + assert result.returncode == 0, result.stdout + result.stderr + body = self._comment(calls) + assert "No review ran: the adapt route held." in body + assert "The stored score (94) and its review carry over unchanged" in body + assert "(score )" not in body + assert ["workflow", "run", "impl-merge.yml", "-f", "pr_number=7"] in calls + + def test_a_reviewed_counted_merge_names_the_count(self, tmp_path): + result, calls = self._verdict( + tmp_path, GATE_VERDICT="merge", REVIEW_SKIPPED="", SCORE="92", ADAPT_COUNTED="true" + ) + assert result.returncode == 0, result.stdout + result.stderr + body = self._comment(calls) + assert "compared this render (score 92)" in body + assert "The adapt route held as well, so this PR is labelled `regen:adapt`" in body + assert "regen:adapt-overruled" not in body + + @pytest.mark.parametrize("verdict", ["merge", "keep"]) + def test_an_overruled_route_is_named_in_the_comment(self, tmp_path, verdict): + result, calls = self._verdict( + tmp_path, GATE_VERDICT=verdict, REVIEW_SKIPPED="", SCORE="92", ADAPT_OVERRULED="true", WB_STATUS="opened" + ) + assert result.returncode == 0, result.stdout + result.stderr + body = self._comment(calls) + assert ( + "The adapt route held before the review, but the review overruled it (see **Adapt route** below), " + "so this PR is labelled `regen:adapt-overruled`." in body + ) + + def test_a_review_free_keep_has_no_rescore(self, tmp_path): + result, calls = self._verdict(tmp_path, GATE_VERDICT="keep", GATE_CODE="no_score", WB_STATUS="") + assert result.returncode == 0, result.stdout + result.stderr + body = self._comment(calls) + assert "compared this render (no review: the adapt route skipped it)" in body + assert "**Stored review:** unchanged (the gate has no re-score of the live implementation: no_score)." in body + assert ["pr", "close", "7", "--delete-branch"] in calls + + +def test_every_regen_tool_the_workflow_runs_is_copied(): + """A tool missing from "Checkout PR code"'s copy only fails on a runner.""" + text = WORKFLOW.read_text(encoding="utf-8") + copy = _step("impl-review.yml", "Checkout PR code")["run"] + copied = set(re.findall(r"automation/scripts/(\w+\.py)", copy[: copy.index("git fetch origin")])) + used = set(re.findall(r"\$\{?RUNNER_TEMP\}?/regen-tools/(\w+\.py)", text)) + for alias in re.findall(r'(\w+)="\$\{?RUNNER_TEMP\}?/regen-tools/?"', text): + used |= set(re.findall(rf"\$\{{?{alias}\}}?/(\w+\.py)", text)) + assert {"adaptability_check.py", "adapt_record.py", "regen_gate.py", "regen_writeback.py"} <= used + assert used <= copied, f"used but not copied: {sorted(used - copied)}"