diff --git a/.github/workflows/rebuild-content.yml b/.github/workflows/rebuild-content.yml index fe4d1d116..d0ca6e55c 100644 --- a/.github/workflows/rebuild-content.yml +++ b/.github/workflows/rebuild-content.yml @@ -83,6 +83,11 @@ on: required: false type: boolean default: false + content-cache: + description: 'EXPERIMENTAL (Workstream C / slug-targeted-delta-rebuild): on a slug-targeted run, reuse the generated-content cache to skip regenerating unchanged tutorials (the bulk of the Fetch step). Fail-open; default OFF. Enable to A/B the fast path on DEV before flipping it on by default.' + required: false + type: boolean + default: false env: NODE_VERSION: '22' @@ -360,6 +365,27 @@ jobs: echo "::add-mask::${VCAP}" echo "VCAP_SERVICES=${VCAP}" >> "$GITHUB_ENV" + # EXPERIMENTAL (Workstream C): restore the previously-generated content + # tree so a slug-targeted run can reuse it and skip regenerating unchanged + # tutorials. Keyed on the parser/generator SOURCE hash — a parser change + # misses the key, nothing is restored, and fetch-tutorials full-regenerates + # (the per-slug existsSync guard falls through). The run_id suffix + prefix + # restore-keys let each run save its updated tree/sidecar while restoring + # the most recent matching one (same pattern as the tutorial cache). The + # runtime feed-fingerprint gate (in fetch-tutorials) covers catalog/tag + # changes. Only runs when the flag is on; fail-open otherwise. + - name: Restore generated-content cache + if: ${{ inputs.content-cache == true && steps.mode.outputs.effective_mode != 'catalog-only' }} + uses: actions/cache@v4 + with: + path: | + hugo/content/tutorials + hugo/data/image_dimensions.json + .content-cache + key: content-tree-v1-${{ hashFiles('scripts/parsers/**', 'scripts/fetch-tutorials.ts', 'scripts/lib/content-cache.ts', 'scripts/lib/expand-ai-authored.ts') }}-${{ github.run_id }} + restore-keys: | + content-tree-v1-${{ hashFiles('scripts/parsers/**', 'scripts/fetch-tutorials.ts', 'scripts/lib/content-cache.ts', 'scripts/lib/expand-ai-authored.ts') }}- + - name: Fetch tutorials if: ${{ steps.mode.outputs.effective_mode != 'catalog-only' }} # [#357 followup] When force-cap-refetch is true, pass --force-cap so @@ -387,6 +413,10 @@ jobs: # telemetry line so the CI invariant regex keeps matching. AI_AUTHOR_BUILD_CAP: ${{ inputs.ai-author-build-cap }} CHAT_DEPLOYMENT_ID: ${{ secrets.CHAT_DEPLOYMENT_ID }} + # EXPERIMENTAL (Workstream C): enable the generated-content fast path. + # fetch-tutorials reuses cached non-target content when this is 'true' + # AND the run is slug-targeted AND the feed fingerprint matches. + CONTENT_CACHE_FAST_PATH: ${{ inputs.content-cache == true }} # [#601] Generate per-advocate profile-page markdown into # hugo/content/developer-advocates/ from /api/advocates. Runs on ALL diff --git a/.gitignore b/.gitignore index 51a2a4cd8..435aba87d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ approuter/static/* !approuter/static/.well-known/*.template .tutorial-cache/ .tutorial-cache-qa/ +.content-cache/ hugo/data/image_dimensions.json hugo/data/homepage_shelves.json hugo/data/verb_definitions.json diff --git a/scripts/fetch-tutorials.ts b/scripts/fetch-tutorials.ts index 82e01cdf8..322c47b3c 100644 --- a/scripts/fetch-tutorials.ts +++ b/scripts/fetch-tutorials.ts @@ -7,6 +7,14 @@ import { flushDimensionsCache, populateImageDimensions, exportDimensionsForHugo import { composeTutorial } from './parsers/compose.js' import { discoverAllTutorials, fetchGitHubMetaBatch, fetchGitHubMeta, fetchRulesVr, fetchWithRetry, uploadDiscoveryToHana, saveDiscoveryBaseline, EXCLUDED_REPOS, type DiscoveredTutorial } from './parsers/github.js' import { fetchBuildCatalog, fetchCoCompletions, loadCapCache, saveCapCache, type BrowseFeaturedEntry } from './parsers/cap.js' +import { + computeFeedFingerprint, + readSidecar, + writeSidecar, + decideFastPath, + navEntriesBySlug, + SIDECAR_VERSION, +} from './lib/content-cache.js' import { parseRulesVrEnriched, collectAiGradedSpecs } from './parsers/rules.js' import { expandAiAuthoredQuestions, populateAiAuthoredSiblingMaps, type ExpandStats } from './lib/expand-ai-authored.js' import { loadAiQuizCache, saveAiQuizCache } from './lib/ai-quiz-cache.js' @@ -834,6 +842,49 @@ async function main() { // An empty map is returned on failure; all tags fall back to the heuristic. const tagRegistry = await fetchTagLabelRegistry() + // ── Content-cache fast path (Workstream C, flag-gated: CONTENT_CACHE_FAST_PATH) ── + // On a slug-targeted run, reuse the previously-generated content for non-target + // slugs instead of recomposing all ~1400 (the bulk of Phase 3's cost). TWO gates, + // both must hold or we full-regenerate: + // 1. The CI actions/cache KEY (parser-source hash) governs whether the generated + // tree + sidecar were even restored — a parser change misses the cache. + // 2. A runtime feed fingerprint over the CAP catalog + tag-labels — the + // deterministic drivers of non-target frontmatter (prev/next/mission/ + // displayTags). Co-completions are excluded: they are empty on warm-CAP-cache + // runs (fetched only on a cold cache in Phase 4) and their recommendations are + // client-hydrated, so they can't make a cached page's static output wrong. + // Fail-open everywhere: no sidecar / fingerprint mismatch / missing file → full regen. + const CONTENT_CACHE_FAST_PATH = process.env.CONTENT_CACHE_FAST_PATH === 'true' + // Sidecar lives in a dedicated dir cached by the SAME parser-source-hashed + // actions/cache key as hugo/content/tutorials, so a parser change busts both + // together (the generated .md files vanish → the per-slug existsSync guard + // falls through to recompose). Kept OUT of .tutorial-cache (whose key has no + // parser hash) and OUT of hugo/data (Hugo would load it as site.Data). + const contentSidecarPath = join(__dirname, '..', '.content-cache', 'content-cache-sidecar.json') + const decisionFingerprint = computeFeedFingerprint({ catalog: loadCapCache(), tagLabels: tagRegistry }) + const restoredSidecar = CONTENT_CACHE_FAST_PATH ? readSidecar(contentSidecarPath) : null + const fastPath = decideFastPath({ + flagEnabled: CONTENT_CACHE_FAST_PATH, + isSlugTargeted: !!tutorialSlugFilter, + sidecar: restoredSidecar, + currentFingerprint: decisionFingerprint, + }) + const reuseNavBySlug = restoredSidecar ? navEntriesBySlug(restoredSidecar) : new Map>() + const reuseAuthorRowsBySlug = new Map() + if (restoredSidecar?.authorRows) { + for (const row of restoredSidecar.authorRows as unknown as AuthorTutorialRow[]) { + const s = (row?.slug ?? '').toLowerCase() + if (!s) continue + const arr = reuseAuthorRowsBySlug.get(s) ?? [] + arr.push(row) + reuseAuthorRowsBySlug.set(s, arr) + } + } + let reusedCount = 0 + if (CONTENT_CACHE_FAST_PATH) { + console.log(`[content-cache] fast path ${fastPath.eligible ? 'ENABLED' : 'disabled'} — ${fastPath.reason}`) + } + mkdirSync(OUTPUT_DIR, { recursive: true }) const navEntries: TutorialNavEntry[] = [] @@ -849,6 +900,24 @@ async function main() { const tutStart = performance.now() const label = `[${idx + 1}/${allTutorials.length}] ${t.repo}/${t.slug}` try { + // Fast-path reuse: for a non-target slug on an eligible run, reuse the + // cached generated page + sidecar nav/author rows and skip compose / + // fetchRulesVr / AI-quiz / writeHugoPage entirely. Fail-open: if the + // generated file or the sidecar entry is missing, fall through to a + // normal (re)generation for this slug. + if (fastPath.eligible && tutorialSlugFilter && !tutorialSlugFilter.has(t.slug)) { + const cachedNav = reuseNavBySlug.get(t.slug.toLowerCase()) + const generatedFile = join(OUTPUT_DIR, `${t.slug}.md`) + if (cachedNav && existsSync(generatedFile)) { + navEntries.push(cachedNav as unknown as TutorialNavEntry) + for (const row of reuseAuthorRowsBySlug.get(t.slug.toLowerCase()) ?? []) authorRows.push(row) + reusedCount++ + cacheHits++ + console.log(`${label} [reused]`) + timings.push({ slug: t.slug, repo: t.repo, durationMs: performance.now() - tutStart }) + return + } + } let rawMd: string let lastUpdated = '' let createdAt = '' @@ -1381,6 +1450,32 @@ async function main() { const navPath = join(navJsonDir, '_nav.json') writeFileSync(navPath, JSON.stringify(navData, null, 2), 'utf-8') + // ── Content-cache sidecar (Workstream C) ── + // Persist the full post-Phase-4 navEntries + author rows + the feed fingerprint + // so the NEXT slug-targeted run can reuse non-target content (see the fast-path + // block above). Written on every content-producing run (full or slug-targeted) + // when the flag is on, so the sidecar always reflects the latest complete set. + // Guarded on a resolvable catalog; fail-open (never blocks the build). + if (CONTENT_CACHE_FAST_PATH) { + try { + const writeCatalog = loadCapCache() + if (writeCatalog) { + mkdirSync(dirname(contentSidecarPath), { recursive: true }) + writeSidecar(contentSidecarPath, { + version: SIDECAR_VERSION, + feedFingerprint: computeFeedFingerprint({ catalog: writeCatalog, tagLabels: tagRegistry }), + navEntries: navEntries as unknown as Record[], + authorRows: authorRows as unknown as Record[], + }) + console.log(`[content-cache] wrote sidecar: ${navEntries.length} nav entries, ${authorRows.length} author rows (${reusedCount} slug(s) reused this run)`) + } else { + console.log('[content-cache] sidecar not written (no CAP catalog available to fingerprint)') + } + } catch (err) { + console.warn(`[content-cache] sidecar write failed: ${err instanceof Error ? err.message : err}`) + } + } + if (target === 'vitepress') { // Also write to public/ so VitePress copies it to dist as a static asset const publicNavDir = join(__dirname, '..', 'site', 'public', 'tutorials') diff --git a/scripts/lib/content-cache.ts b/scripts/lib/content-cache.ts index 4e834b2a9..b532f9f43 100644 --- a/scripts/lib/content-cache.ts +++ b/scripts/lib/content-cache.ts @@ -38,12 +38,18 @@ export interface ContentCacheSidecar { // Per-slug nav entries (the same objects written to _nav.json) so non-target // slugs can contribute to nav / browse.json without being recomposed. navEntries: Record[] + // Per-slug author rows (one per tutorial) so reused slugs still contribute to + // author pages / "more from this author" without recomposition. + authorRows?: Record[] } export interface FeedPayloads { catalog: unknown - coCompletions: unknown tagLabels: unknown + // Optional: co-completions drive recommendations, which are empty on warm-CAP- + // cache runs and client-hydrated at render time, so they are excluded from the + // fingerprint by callers on the fast path. Kept optional for completeness/tests. + coCompletions?: unknown } // Stable JSON stringify (sorted keys) so semantically-identical feeds always @@ -62,7 +68,7 @@ function stableStringify(value: unknown): string { export function computeFeedFingerprint(feeds: FeedPayloads): string { const h = createHash('sha256') h.update('catalog\0'); h.update(stableStringify(feeds.catalog)) - h.update('\0coCompletions\0'); h.update(stableStringify(feeds.coCompletions)) + h.update('\0coCompletions\0'); h.update(stableStringify(feeds.coCompletions ?? null)) h.update('\0tagLabels\0'); h.update(stableStringify(feeds.tagLabels)) return h.digest('hex') }