ci: route the base-std docs tree, skip deleted sources, report unrouted files

.github/workflows/base-std-docs-sync.yml

@@ -131,6 +131,10 @@ jobs:
     permissions:
       contents: write
       pull-requests: write
+      # Only used by "Open issue for unrouted source files" below: when a
+      # dispatch routes nothing but changed files the route table does not
+      # know, the routing report is filed as an issue instead of vanishing.
+      issues: write
     steps:
       - name: Harden the runner
         # Audit mode logs every outbound connection without blocking. After a
@@ -772,6 +776,65 @@ jobs:
 
           echo "::notice title=Payload provenance verified::kind=${KIND:-code-change} sha=${SHA:0:7} tag=${TAG:-none} pr_number=${PR_NUMBER:-none} artifact_run_id=${ARTIFACT_RUN_ID:-none} (all checks passed against ${SOURCE_REPO})"
 
+      - name: Derive trusted removed paths
+        # A code-change dispatch lists changed_paths but not what happened to
+        # each file. Before base-std#213 that did not matter; that commit
+        # deleted six documentation files the route table mapped, and the
+        # sync edited their target pages from an all-minus diff, adding
+        # "source file removed" banners to reference pages generated from an
+        # unchanged interface (base/docs#1928). The commit API is the trusted
+        # source of per-file status, so derive `removed_paths` here — never
+        # from client_payload — and let the script skip them for routing.
+        # A rename counts its previous name as removed.
+        #
+        # Best-effort: an API failure leaves removed_paths empty and logs a
+        # warning rather than failing a sync whose content is otherwise fine.
+        # Runs for every non-release dispatch so the payload's own
+        # removed_paths, if any, is always overwritten.
+        if: env.PAYLOAD_KIND != 'release'
+        env:
+          SOURCE_REPO: ${{ env.PAYLOAD_SOURCE_REPO }}
+          SHA: ${{ env.PAYLOAD_SHA }}
+          SOURCE_TOKEN: ${{ secrets.DOCS_REPO_TOKEN }}
+          PAYLOAD_PATH: ${{ steps.payload_file.outputs.path }}
+          MAX_REMOVED_PATHS: 200
+          MAX_REMOVED_PATH_BYTES: 512
+        run: |
+          set -euo pipefail
+
+          if [[ -z "${SHA:-}" ]]; then
+            jq '.removed_paths = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
+            mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"
+            echo "No sha in payload; removed_paths cleared."
+            exit 0
+          fi
+
+          commit="$RUNNER_TEMP/code-change-commit.json"
+          code=$(curl -sS -o "$commit" -w '%{http_code}' \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer $SOURCE_TOKEN" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "https://api.github.com/repos/${SOURCE_REPO}/commits/${SHA}") || code="000"
+          if [[ "$code" != "200" ]]; then
+            echo "::warning title=Removed paths unavailable::reading commit ${SHA:0:7} on ${SOURCE_REPO} failed (HTTP ${code}); deletions will route like edits this run"
+            jq '.removed_paths = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
+            mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"
+            exit 0
+          fi
+          jq -e '(.files // []) | all(.[]; (.filename | type == "string"))' "$commit" >/dev/null \
+            || { echo "::warning title=Removed paths unavailable::commit ${SHA:0:7} returned malformed file metadata"; jq '.removed_paths = []' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"; mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"; exit 0; }
+          jq --argjson cap "$MAX_REMOVED_PATHS" --argjson bytes "$MAX_REMOVED_PATH_BYTES" '
+            [ .files[]?
+              | select(.status == "removed" or .status == "renamed")
+              | (if .status == "renamed" then .previous_filename else .filename end)
+              | select(type == "string" and length > 0 and length <= $bytes)
+            ] | unique | .[0:$cap]
+          ' "$commit" > "$RUNNER_TEMP/trusted-removed-paths.json"
+          jq --slurpfile removed "$RUNNER_TEMP/trusted-removed-paths.json" \
+            '.removed_paths = $removed[0]' "$PAYLOAD_PATH" > "$PAYLOAD_PATH.tmp"
+          mv "$PAYLOAD_PATH.tmp" "$PAYLOAD_PATH"
+          echo "::notice title=Trusted removed paths derived::sha=${SHA:0:7} removed_paths=$(jq length "$RUNNER_TEMP/trusted-removed-paths.json")"
+
       - name: Derive trusted release routing inputs
         # client_payload.changed_paths is attacker-controlled JSON. For a
         # release, replace it with the paths returned by GitHub for the
@@ -1009,6 +1072,10 @@ jobs:
           RELEASE_PAGE_CONCURRENCY: ${{ vars.RELEASE_PAGE_CONCURRENCY }}
           CLAUDE_MODEL: ${{ vars.CLAUDE_MODEL }}
           CLAUDE_MAX_TOKENS: ${{ vars.CLAUDE_MAX_TOKENS }}
+          # Repo variable. "propose" (default) lists a guideline-derived
+          # placement for unrouted source files in the PR/issue body;
+          # "apply" also edits the proposed pages; "off" skips the call.
+          GUIDELINE_ROUTING: ${{ vars.GUIDELINE_ROUTING }}
         run: |
           node scripts/sync-from-base-std/index.mjs --payload "$PAYLOAD_PATH"
 
@@ -1338,3 +1405,89 @@ jobs:
             echo "::warning title=Docs PR step::Step succeeded but no PR URL was returned"
           fi
           echo "PR: ${pr_url:-(no url returned)}"
+
+      - name: Open issue for unrouted source files
+        # When a dispatch produces a PR, the routing report (unrouted and
+        # removed source files, guideline-derived placement proposals) is
+        # part of the PR body. When it produces no PR — nothing routed, or
+        # every routed page came back unchanged — the report would otherwise
+        # be visible only in the run log. File it as an issue instead, once
+        # per source sha, so a maintainer can add the missing route-table
+        # rule. Same REST + jq pattern as the PR step; no gh CLI.
+        if: steps.commit.outputs.no_changes == 'true' && steps.sync.outputs.unrouted_count != '' && steps.sync.outputs.unrouted_count != '0' && steps.sync.outputs.review_md_path != ''
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          REVIEW_MD_PATH: ${{ steps.sync.outputs.review_md_path }}
+          UNROUTED_COUNT: ${{ steps.sync.outputs.unrouted_count }}
+          SOURCE_REPO: ${{ env.PAYLOAD_SOURCE_REPO }}
+          SHA: ${{ env.PAYLOAD_SHA }}
+          PR_NUMBER: ${{ env.PAYLOAD_PR_NUMBER }}
+          PR_TITLE: ${{ env.PAYLOAD_PR_TITLE }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          if [[ ! -f "$REVIEW_MD_PATH" ]]; then
+            echo "::warning title=Unrouted report missing::sync reported ${UNROUTED_COUNT} unrouted file(s) but wrote no report"
+            exit 0
+          fi
+          short_sha="${SHA:0:7}"
+          title="Unrouted base-std docs: ${SOURCE_REPO}@${short_sha:-unknown}"
+
+          # One issue per source sha: a re-dispatch of the same commit updates
+          # the existing open issue instead of opening a twin.
+          existing=$(curl -sS \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer $GITHUB_TOKEN" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "https://api.github.com/repos/${REPO}/issues?state=open&creator=app%2Fgithub-actions&per_page=100" \
+            | jq -r --arg t "$title" '[.[] | select(.pull_request == null) | select(.title == $t)][0].number // ""')
+
+          body_file="$RUNNER_TEMP/unrouted_issue_body.md"
+          {
+            if [[ -n "${PR_NUMBER:-}" ]]; then
+              echo "> **Source PR**: [${SOURCE_REPO}#${PR_NUMBER}](https://github.com/${SOURCE_REPO}/pull/${PR_NUMBER})${PR_TITLE:+ — _${PR_TITLE}_}"
+              echo ">"
+            fi
+            if [[ -n "${SHA:-}" ]]; then
+              echo "> **Merge commit**: [\`${short_sha}\`](https://github.com/${SOURCE_REPO}/commit/${SHA})"
+            fi
+            echo
+            echo "The docs sync ran for this commit and opened no PR: ${UNROUTED_COUNT} changed file(s) match no rule in \`scripts/sync-from-base-std/route-table.json\`. Add a rule (or an \`ignored\` rule) for each one, then re-dispatch the commit."
+            echo
+            cat "$REVIEW_MD_PATH"
+            echo
+            echo "_Opened by \`Apply Base Std Update\` workflow ([run](${RUN_URL}))._"
+          } > "$body_file"
+
+          payload_file="$RUNNER_TEMP/issue.json"
+          if [[ -n "$existing" ]]; then
+            echo "Updating existing issue #$existing"
+            jq -n --arg title "$title" --rawfile body "$body_file" '{title: $title, body: $body}' > "$payload_file"
+            status=$(curl -sS -o "$RUNNER_TEMP/issue_resp.json" -w "%{http_code}" \
+              -X PATCH \
+              -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer $GITHUB_TOKEN" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/${REPO}/issues/${existing}" \
+              --data-binary @"$payload_file")
+            expected="200"
+          else
+            echo "Creating issue: $title"
+            jq -n --arg title "$title" --rawfile body "$body_file" '{title: $title, body: $body}' > "$payload_file"
+            status=$(curl -sS -o "$RUNNER_TEMP/issue_resp.json" -w "%{http_code}" \
+              -X POST \
+              -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer $GITHUB_TOKEN" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/${REPO}/issues" \
+              --data-binary @"$payload_file")
+            expected="201"
+          fi
+          if [[ "$status" != "$expected" ]]; then
+            echo "::warning title=Unrouted issue not filed::issues API returned HTTP ${status} (expected ${expected}); the routing report is in this run's log"
+            cat "$RUNNER_TEMP/issue_resp.json" >&2 || true
+            exit 0
+          fi
+          issue_url=$(jq -r '.html_url // empty' "$RUNNER_TEMP/issue_resp.json")
+          echo "::notice title=Unrouted source files::${UNROUTED_COUNT} file(s) need a route-table rule — ${issue_url}"

scripts/sync-from-base-std/README.md

@@ -17,6 +17,61 @@ The route table supports both exact `pages` and `page_globs`. Globs are expanded
 only against existing Markdown files beneath `docs/`; they cannot create new
 paths. This version intentionally does not create, rename, or delete API pages.
 
+## Routing outcomes
+
+Every changed source path in a `code-change` dispatch ends up in exactly one
+bucket, logged as `[routing]` and reported in the PR body (or an issue):
+
+| Bucket | Meaning | What happens |
+|---|---|---|
+| routed | matched a rule with `kind` interface, product-doc, changelog-* | its pages are edited |
+| ignored | matched only a `kind: "ignored"` rule | nothing; deliberately unsynced (upstream README, authoring templates) |
+| unrouted | matched no rule | listed under **Unrouted source files** with a guideline-derived placement proposal |
+| removed | deleted in the source commit (`removed_paths`) | never routed; listed under **Removed source files** |
+
+`removed_paths` is derived by the workflow from the commit API for the verified
+sha, never from the dispatcher's payload. A deleted documentation file carries
+nothing to sync: its docs pages are generated from surviving sources, and a
+deprecation shows up in the diff of the file that declares it. Routing a
+deletion used to hand the model an all-minus diff and produced "the source file
+has been removed" banners on reference pages (base/docs#1928); the validator now
+rejects callouts that describe repository housekeeping (`validateCallouts` in
+`safety.mjs`).
+
+### Placement proposals from the IA guidelines
+
+Unrouted Markdown sources go through one Haiku call (`proposePlacement`) that
+reads the same `docs/ia-guidelines.md` and `docs/content-guidelines.md` every
+page-editing prompt already carries, plus the title and description of every
+existing page under `docs/specifications/` and `docs/build-on-base/`, and
+returns the existing page each file's content belongs on with the guideline rule
+that decides it. Proposals are filtered back against the candidate list, so a
+hallucinated path is dropped; nothing here creates a page. The result lands in
+the PR or issue body so a maintainer can turn it into a route-table rule.
+
+`GUIDELINE_ROUTING` (repo variable, default `propose`) controls it: `apply`
+also edits the proposed pages in the same run, tagged `guideline:<source>` in
+the routing log; `off` skips the call. Prompt-size caps:
+`PLACEMENT_MAX_CANDIDATES` (250), `PLACEMENT_EXCERPT_LINES` (60),
+`PLACEMENT_MAX_SOURCES` (25).
+
+When a dispatch opens no PR (nothing routed, or every page came back unchanged)
+but has unrouted files, the workflow files the routing report as an issue titled
+`Unrouted base-std docs: <source_repo>@<sha>`, one per source sha.
+
+### base-std `docs/` tree
+
+Since base-std#213 the upstream docs are audience-layered (`overview.md`,
+`architecture.md`, `concepts/`, `guides/`, `reference/`). The route table maps
+them where `docs/ia-guidelines.md` and `docs/content-guidelines.md` put that
+kind of content: chain-generic precompile mechanics to Base Protocol →
+Execution, the B20 component map and key concepts to the specification
+overview, execution and versioning guarantees to the invariants page, how-to
+guides to the existing Build on Base task pages, and reference tables to the
+B20 supporting pages. Pages above the regeneration budget
+(`MAX_REGENERABLE_CHARS`) are skipped with a logged reason and need a human
+edit.
+
 ## Local checks
 
 From the copied `docs-repo` root:
@@ -34,6 +89,11 @@ LLM_GATEWAY_API_KEY=... \
   --payload scripts/sync-from-base-std/fixtures/code-change-ib20.json
 ```
 
+`fixtures/code-change-docs-restructure.json` is the real file list from
+base-std@be6d045 with its `removed_paths`; run it the same way to exercise the
+docs-tree routes and the placement pass (set `RUNNER_TEMP` to see the routing
+report written as `sync-review.md`).
+
 Configuration knobs are optional positive numbers:
 
 - `CODE_CHANGE_PAGE_CONCURRENCY` (default `4`)