feat(rebuild): generated-content fast path — Workstream C wiring (flag-gated, draft)

.github/workflows/rebuild-content.yml

@@ -83,6 +83,11 @@ on:
         required: false
         type: boolean
         default: false
+      content-cache:
+        description: 'EXPERIMENTAL (Workstream C / slug-targeted-delta-rebuild): on a slug-targeted run, reuse the generated-content cache to skip regenerating unchanged tutorials (the bulk of the Fetch step). Fail-open; default OFF. Enable to A/B the fast path on DEV before flipping it on by default.'
+        required: false
+        type: boolean
+        default: false
 
 env:
   NODE_VERSION: '22'
@@ -360,6 +365,27 @@ jobs:
           echo "::add-mask::${VCAP}"
           echo "VCAP_SERVICES=${VCAP}" >> "$GITHUB_ENV"
 
+      # EXPERIMENTAL (Workstream C): restore the previously-generated content
+      # tree so a slug-targeted run can reuse it and skip regenerating unchanged
+      # tutorials. Keyed on the parser/generator SOURCE hash — a parser change
+      # misses the key, nothing is restored, and fetch-tutorials full-regenerates
+      # (the per-slug existsSync guard falls through). The run_id suffix + prefix
+      # restore-keys let each run save its updated tree/sidecar while restoring
+      # the most recent matching one (same pattern as the tutorial cache). The
+      # runtime feed-fingerprint gate (in fetch-tutorials) covers catalog/tag
+      # changes. Only runs when the flag is on; fail-open otherwise.
+      - name: Restore generated-content cache
+        if: ${{ inputs.content-cache == true && steps.mode.outputs.effective_mode != 'catalog-only' }}
+        uses: actions/cache@v4
+        with:
+          path: |
+            hugo/content/tutorials
+            hugo/data/image_dimensions.json
+            .content-cache
+          key: content-tree-v1-${{ hashFiles('scripts/parsers/**', 'scripts/fetch-tutorials.ts', 'scripts/lib/content-cache.ts', 'scripts/lib/expand-ai-authored.ts') }}-${{ github.run_id }}
+          restore-keys: |
+            content-tree-v1-${{ hashFiles('scripts/parsers/**', 'scripts/fetch-tutorials.ts', 'scripts/lib/content-cache.ts', 'scripts/lib/expand-ai-authored.ts') }}-
+
       - name: Fetch tutorials
         if: ${{ steps.mode.outputs.effective_mode != 'catalog-only' }}
         # [#357 followup] When force-cap-refetch is true, pass --force-cap so
@@ -387,6 +413,10 @@ jobs:
           # telemetry line so the CI invariant regex keeps matching.
           AI_AUTHOR_BUILD_CAP: ${{ inputs.ai-author-build-cap }}
           CHAT_DEPLOYMENT_ID: ${{ secrets.CHAT_DEPLOYMENT_ID }}
+          # EXPERIMENTAL (Workstream C): enable the generated-content fast path.
+          # fetch-tutorials reuses cached non-target content when this is 'true'
+          # AND the run is slug-targeted AND the feed fingerprint matches.
+          CONTENT_CACHE_FAST_PATH: ${{ inputs.content-cache == true }}
 
       # [#601] Generate per-advocate profile-page markdown into
       # hugo/content/developer-advocates/ from /api/advocates. Runs on ALL

.gitignore

@@ -23,6 +23,7 @@ approuter/static/*
 !approuter/static/.well-known/*.template
 .tutorial-cache/
 .tutorial-cache-qa/
+.content-cache/
 hugo/data/image_dimensions.json
 hugo/data/homepage_shelves.json
 hugo/data/verb_definitions.json

scripts/fetch-tutorials.ts

@@ -7,6 +7,14 @@ import { flushDimensionsCache, populateImageDimensions, exportDimensionsForHugo
 import { composeTutorial } from './parsers/compose.js'
 import { discoverAllTutorials, fetchGitHubMetaBatch, fetchGitHubMeta, fetchRulesVr, fetchWithRetry, uploadDiscoveryToHana, saveDiscoveryBaseline, EXCLUDED_REPOS, type DiscoveredTutorial } from './parsers/github.js'
 import { fetchBuildCatalog, fetchCoCompletions, loadCapCache, saveCapCache, type BrowseFeaturedEntry } from './parsers/cap.js'
+import {
+  computeFeedFingerprint,
+  readSidecar,
+  writeSidecar,
+  decideFastPath,
+  navEntriesBySlug,
+  SIDECAR_VERSION,
+} from './lib/content-cache.js'
 import { parseRulesVrEnriched, collectAiGradedSpecs } from './parsers/rules.js'
 import { expandAiAuthoredQuestions, populateAiAuthoredSiblingMaps, type ExpandStats } from './lib/expand-ai-authored.js'
 import { loadAiQuizCache, saveAiQuizCache } from './lib/ai-quiz-cache.js'
@@ -834,6 +842,49 @@ async function main() {
   // An empty map is returned on failure; all tags fall back to the heuristic.
   const tagRegistry = await fetchTagLabelRegistry()
 
+  // ── Content-cache fast path (Workstream C, flag-gated: CONTENT_CACHE_FAST_PATH) ──
+  // On a slug-targeted run, reuse the previously-generated content for non-target
+  // slugs instead of recomposing all ~1400 (the bulk of Phase 3's cost). TWO gates,
+  // both must hold or we full-regenerate:
+  //   1. The CI actions/cache KEY (parser-source hash) governs whether the generated
+  //      tree + sidecar were even restored — a parser change misses the cache.
+  //   2. A runtime feed fingerprint over the CAP catalog + tag-labels — the
+  //      deterministic drivers of non-target frontmatter (prev/next/mission/
+  //      displayTags). Co-completions are excluded: they are empty on warm-CAP-cache
+  //      runs (fetched only on a cold cache in Phase 4) and their recommendations are
+  //      client-hydrated, so they can't make a cached page's static output wrong.
+  // Fail-open everywhere: no sidecar / fingerprint mismatch / missing file → full regen.
+  const CONTENT_CACHE_FAST_PATH = process.env.CONTENT_CACHE_FAST_PATH === 'true'
+  // Sidecar lives in a dedicated dir cached by the SAME parser-source-hashed
+  // actions/cache key as hugo/content/tutorials, so a parser change busts both
+  // together (the generated .md files vanish → the per-slug existsSync guard
+  // falls through to recompose). Kept OUT of .tutorial-cache (whose key has no
+  // parser hash) and OUT of hugo/data (Hugo would load it as site.Data).
+  const contentSidecarPath = join(__dirname, '..', '.content-cache', 'content-cache-sidecar.json')
+  const decisionFingerprint = computeFeedFingerprint({ catalog: loadCapCache(), tagLabels: tagRegistry })
+  const restoredSidecar = CONTENT_CACHE_FAST_PATH ? readSidecar(contentSidecarPath) : null
+  const fastPath = decideFastPath({
+    flagEnabled: CONTENT_CACHE_FAST_PATH,
+    isSlugTargeted: !!tutorialSlugFilter,
+    sidecar: restoredSidecar,
+    currentFingerprint: decisionFingerprint,
+  })
+  const reuseNavBySlug = restoredSidecar ? navEntriesBySlug(restoredSidecar) : new Map<string, Record<string, unknown>>()
+  const reuseAuthorRowsBySlug = new Map<string, AuthorTutorialRow[]>()
+  if (restoredSidecar?.authorRows) {
+    for (const row of restoredSidecar.authorRows as unknown as AuthorTutorialRow[]) {
+      const s = (row?.slug ?? '').toLowerCase()
+      if (!s) continue
+      const arr = reuseAuthorRowsBySlug.get(s) ?? []
+      arr.push(row)
+      reuseAuthorRowsBySlug.set(s, arr)
+    }
+  }
+  let reusedCount = 0
+  if (CONTENT_CACHE_FAST_PATH) {
+    console.log(`[content-cache] fast path ${fastPath.eligible ? 'ENABLED' : 'disabled'} — ${fastPath.reason}`)
+  }
+
   mkdirSync(OUTPUT_DIR, { recursive: true })
 
   const navEntries: TutorialNavEntry[] = []
@@ -849,6 +900,24 @@ async function main() {
     const tutStart = performance.now()
     const label = `[${idx + 1}/${allTutorials.length}] ${t.repo}/${t.slug}`
     try {
+      // Fast-path reuse: for a non-target slug on an eligible run, reuse the
+      // cached generated page + sidecar nav/author rows and skip compose /
+      // fetchRulesVr / AI-quiz / writeHugoPage entirely. Fail-open: if the
+      // generated file or the sidecar entry is missing, fall through to a
+      // normal (re)generation for this slug.
+      if (fastPath.eligible && tutorialSlugFilter && !tutorialSlugFilter.has(t.slug)) {
+        const cachedNav = reuseNavBySlug.get(t.slug.toLowerCase())
+        const generatedFile = join(OUTPUT_DIR, `${t.slug}.md`)
+        if (cachedNav && existsSync(generatedFile)) {
+          navEntries.push(cachedNav as unknown as TutorialNavEntry)
+          for (const row of reuseAuthorRowsBySlug.get(t.slug.toLowerCase()) ?? []) authorRows.push(row)
+          reusedCount++
+          cacheHits++
+          console.log(`${label} [reused]`)
+          timings.push({ slug: t.slug, repo: t.repo, durationMs: performance.now() - tutStart })
+          return
+        }
+      }
       let rawMd: string
       let lastUpdated = ''
       let createdAt = ''
@@ -1381,6 +1450,32 @@ async function main() {
   const navPath = join(navJsonDir, '_nav.json')
   writeFileSync(navPath, JSON.stringify(navData, null, 2), 'utf-8')
 
+  // ── Content-cache sidecar (Workstream C) ──
+  // Persist the full post-Phase-4 navEntries + author rows + the feed fingerprint
+  // so the NEXT slug-targeted run can reuse non-target content (see the fast-path
+  // block above). Written on every content-producing run (full or slug-targeted)
+  // when the flag is on, so the sidecar always reflects the latest complete set.
+  // Guarded on a resolvable catalog; fail-open (never blocks the build).
+  if (CONTENT_CACHE_FAST_PATH) {
+    try {
+      const writeCatalog = loadCapCache()
+      if (writeCatalog) {
+        mkdirSync(dirname(contentSidecarPath), { recursive: true })
+        writeSidecar(contentSidecarPath, {
+          version: SIDECAR_VERSION,
+          feedFingerprint: computeFeedFingerprint({ catalog: writeCatalog, tagLabels: tagRegistry }),
+          navEntries: navEntries as unknown as Record<string, unknown>[],
+          authorRows: authorRows as unknown as Record<string, unknown>[],
+        })
+        console.log(`[content-cache] wrote sidecar: ${navEntries.length} nav entries, ${authorRows.length} author rows (${reusedCount} slug(s) reused this run)`)
+      } else {
+        console.log('[content-cache] sidecar not written (no CAP catalog available to fingerprint)')
+      }
+    } catch (err) {
+      console.warn(`[content-cache] sidecar write failed: ${err instanceof Error ? err.message : err}`)
+    }
+  }
+
   if (target === 'vitepress') {
     // Also write to public/ so VitePress copies it to dist as a static asset
     const publicNavDir = join(__dirname, '..', 'site', 'public', 'tutorials')

scripts/lib/content-cache.ts

@@ -38,12 +38,18 @@ export interface ContentCacheSidecar {
   // Per-slug nav entries (the same objects written to _nav.json) so non-target
   // slugs can contribute to nav / browse.json without being recomposed.
   navEntries: Record<string, unknown>[]
+  // Per-slug author rows (one per tutorial) so reused slugs still contribute to
+  // author pages / "more from this author" without recomposition.
+  authorRows?: Record<string, unknown>[]
 }
 
 export interface FeedPayloads {
   catalog: unknown
-  coCompletions: unknown
   tagLabels: unknown
+  // Optional: co-completions drive recommendations, which are empty on warm-CAP-
+  // cache runs and client-hydrated at render time, so they are excluded from the
+  // fingerprint by callers on the fast path. Kept optional for completeness/tests.
+  coCompletions?: unknown
 }
 
 // Stable JSON stringify (sorted keys) so semantically-identical feeds always
@@ -62,7 +68,7 @@ function stableStringify(value: unknown): string {
 export function computeFeedFingerprint(feeds: FeedPayloads): string {
   const h = createHash('sha256')
   h.update('catalog\0'); h.update(stableStringify(feeds.catalog))
-  h.update('\0coCompletions\0'); h.update(stableStringify(feeds.coCompletions))
+  h.update('\0coCompletions\0'); h.update(stableStringify(feeds.coCompletions ?? null))
   h.update('\0tagLabels\0'); h.update(stableStringify(feeds.tagLabels))
   return h.digest('hex')
 }