diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index 4616d8168..6f33fe78f 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -10,7 +10,7 @@ ID: tutorials-ims # Bump this on each release you deploy — it's the version shown by `cf mtas` # and in the mtar filename (tutorials-ims_.mtar). Deploy is manual: # `cd .deploy && mbt build && cf deploy mta_archives/tutorials-ims_.mtar -e ../deploy/.mtaext -f`. -version: 1.18.3 +version: 1.19.0 # Top-level parameters (overridable per-env via deploy/.mtaext). parameters: @@ -230,6 +230,14 @@ modules: - mkdir -p static/explore-ui - cp -r ../app/explore/dist/. static/explore-ui/ - cp -r ../app/scanner/webapp static/scanner-ui + # Display App (event-monitor dashboard) — Vue 3 + Vite, base '/display-app/'. + # Served from static/display-app via the /display-app/ approuter route. This + # file (NOT the root mta.yaml) is what `mbt build` uses, so the build+copy + # must live here or /display-app/ 404s. + - npm --prefix ../app/display-app install + - npm --prefix ../app/display-app run build + - mkdir -p static/display-app + - cp -r ../app/display-app/dist/. static/display-app/ # Data Inspector UI (#999) — bundled inside @cap-js/data-inspector's # node_modules. Rewrite the UI5 bootstrap CDN in index.html from # sapui5.hana.ondemand.com (plugin default) to ui5.sap.com — the diff --git a/.github/workflows/rebuild-content.yml b/.github/workflows/rebuild-content.yml index d8a42837c..d0ca6e55c 100644 --- a/.github/workflows/rebuild-content.yml +++ b/.github/workflows/rebuild-content.yml @@ -83,6 +83,11 @@ on: required: false type: boolean default: false + content-cache: + description: 'EXPERIMENTAL (Workstream C / slug-targeted-delta-rebuild): on a slug-targeted run, reuse the generated-content cache to skip regenerating unchanged tutorials (the bulk of the Fetch step). Fail-open; default OFF. Enable to A/B the fast path on DEV before flipping it on by default.' + required: false + type: boolean + default: false env: NODE_VERSION: '22' @@ -273,7 +278,31 @@ jobs: prod) echo "srv_url=${{ secrets.CAP_SRV_URL_PROD }}" >> "$GITHUB_OUTPUT" ;; esac + # Cache the *materialized* node_modules (root + hugo-apps), not just the + # npm download cache. `setup-node` `cache: npm` already warms ~/.npm, but + # `npm ci` still spends ~48s extracting/linking 1680 packages + running + # native builds (better-sqlite3) into node_modules on every run. An + # exact-key cache (keyed on both lockfiles + OS + Node major) lets us skip + # the install entirely when the lockfiles are unchanged. + # + # EXACT match only — NO restore-keys. A node_modules restored for a + # different lockfile would be silently wrong because we skip `npm ci` + # (which would otherwise reconcile). On any lockfile change the cache + # misses, the install below runs clean, and the new tree is re-cached. + # Safe to skip install on hit: neither package.json has install/prepare/ + # postinstall lifecycle scripts, so nothing outside node_modules is + # produced at install time. + - name: Cache node_modules + id: node-modules-cache + uses: actions/cache@v4 + with: + path: | + node_modules + hugo-apps/node_modules + key: node-modules-${{ runner.os }}-node${{ env.NODE_VERSION }}-${{ hashFiles('package-lock.json', 'hugo-apps/package-lock.json') }} + - name: Install dependencies + if: steps.node-modules-cache.outputs.cache-hit != 'true' env: NODE_AUTH_TOKEN: ${{ secrets.PACKAGES_READ_TOKEN || secrets.GITHUB_TOKEN }} run: | @@ -336,6 +365,27 @@ jobs: echo "::add-mask::${VCAP}" echo "VCAP_SERVICES=${VCAP}" >> "$GITHUB_ENV" + # EXPERIMENTAL (Workstream C): restore the previously-generated content + # tree so a slug-targeted run can reuse it and skip regenerating unchanged + # tutorials. Keyed on the parser/generator SOURCE hash — a parser change + # misses the key, nothing is restored, and fetch-tutorials full-regenerates + # (the per-slug existsSync guard falls through). The run_id suffix + prefix + # restore-keys let each run save its updated tree/sidecar while restoring + # the most recent matching one (same pattern as the tutorial cache). The + # runtime feed-fingerprint gate (in fetch-tutorials) covers catalog/tag + # changes. Only runs when the flag is on; fail-open otherwise. + - name: Restore generated-content cache + if: ${{ inputs.content-cache == true && steps.mode.outputs.effective_mode != 'catalog-only' }} + uses: actions/cache@v4 + with: + path: | + hugo/content/tutorials + hugo/data/image_dimensions.json + .content-cache + key: content-tree-v1-${{ hashFiles('scripts/parsers/**', 'scripts/fetch-tutorials.ts', 'scripts/lib/content-cache.ts', 'scripts/lib/expand-ai-authored.ts') }}-${{ github.run_id }} + restore-keys: | + content-tree-v1-${{ hashFiles('scripts/parsers/**', 'scripts/fetch-tutorials.ts', 'scripts/lib/content-cache.ts', 'scripts/lib/expand-ai-authored.ts') }}- + - name: Fetch tutorials if: ${{ steps.mode.outputs.effective_mode != 'catalog-only' }} # [#357 followup] When force-cap-refetch is true, pass --force-cap so @@ -363,6 +413,10 @@ jobs: # telemetry line so the CI invariant regex keeps matching. AI_AUTHOR_BUILD_CAP: ${{ inputs.ai-author-build-cap }} CHAT_DEPLOYMENT_ID: ${{ secrets.CHAT_DEPLOYMENT_ID }} + # EXPERIMENTAL (Workstream C): enable the generated-content fast path. + # fetch-tutorials reuses cached non-target content when this is 'true' + # AND the run is slug-targeted AND the feed fingerprint matches. + CONTENT_CACHE_FAST_PATH: ${{ inputs.content-cache == true }} # [#601] Generate per-advocate profile-page markdown into # hugo/content/developer-advocates/ from /api/advocates. Runs on ALL diff --git a/.gitignore b/.gitignore index 51a2a4cd8..435aba87d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ approuter/static/* !approuter/static/.well-known/*.template .tutorial-cache/ .tutorial-cache-qa/ +.content-cache/ hugo/data/image_dimensions.json hugo/data/homepage_shelves.json hugo/data/verb_definitions.json diff --git a/CLAUDE.md b/CLAUDE.md index 4f3cb024b..659deac10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,6 +130,8 @@ Subsystem one-liners: - **`CONTENT_API_KEY` env var required** for `POST /content/publish` and `/content/rollback`. Missing → 401. Set locally when testing publish. +- **Content served from mutable `ContentCurrent` (Option B, #2017 / Workstream D), not the old `ContentFiles` snapshot-per-version** — three env flags gate the migration, all default OFF, flip in order: `CONTENT_DELTA_WRITE_ENABLED` (publish dual-writes changed slugs → `ContentCurrent` + append-only `ContentHistory`, fail-safe) → **seed** via a full force rebuild (`-f mode=full -f force-publish=true` dual-writes all slugs; no separate migration) → `CONTENT_DELTA_READ_ENABLED` (serve/readers hit `ContentCurrent`, per-slug fallback to `ContentFiles`) → `CONTENT_DELTA_SKIP_CARRYFORWARD` (publish skips `carryForwardUnchanged` → **O(changed) publish**; rollback then **replays `ContentHistory`** into `ContentCurrent`, not clear+fallback). Measured DEV: publish commit ~62s→973ms (PROD carry-forward was ~95s @ 11k files). Serve source header `X-Content-Source: db-current` (ContentCurrent) vs `db` (legacy). Revert = flip `SKIP_CARRYFORWARD` off + `cf restart`. Flags in `srv/lib/feature-flags/registry.js`; readers via `resolveContentBlob` (`srv/lib/content-store.js`). LOB reads stay raw `db.run()`. Cutover cleanup (delete `carryForwardUnchanged` + `ContentFiles`) deferred to a release after PROD soak. + - **GitHub Actions secret is `DISPATCH_TOKEN`, not `GITHUB_DISPATCH_TOKEN`** — GH reserves the `GITHUB_` prefix. The runtime env var is `GITHUB_DISPATCH_TOKEN`, read by `srv/lib/rebuild-trigger.js`. - **`rebuild-content.yml` auto-infers `mode=slug-targeted`** when a `slug` input is set — don't pass `-f mode=slug-targeted`. Wall-clock: catalog-only ~5min, slug-targeted ~2min, full ~10min. diff --git a/db-qa/schema.cds b/db-qa/schema.cds index d09056120..906b527d1 100644 --- a/db-qa/schema.cds +++ b/db-qa/schema.cds @@ -7,6 +7,11 @@ entity ContentFiles : shared.ContentFilesAspect {} entity ContentManifest : shared.ContentManifestAspect {} +// Option B (slug-targeted-delta-rebuild) — QA-channel parity with prod. +entity ContentCurrent : shared.ContentCurrentAspect {} + +entity ContentHistory : shared.ContentHistoryAspect {} + // Plain-text projection of published Hugo HTML, indexed for full-text search. // Replaced (not versioned) on every publish so search reflects current content. @cds.autoexpose: false diff --git a/db/_content-shape.cds b/db/_content-shape.cds index d0cf714b2..35c9bd076 100644 --- a/db/_content-shape.cds +++ b/db/_content-shape.cds @@ -64,6 +64,41 @@ aspect ContentManifestAspect : managed { firstAppendAt : Timestamp; } +// Option B (slug-targeted-delta-rebuild): the MUTABLE current-content table — +// one row per slug, NO version column. Readers hit this directly (WHERE slug=?) +// instead of joining on the active manifest version, so a publish writes ONLY +// the changed slugs (no O(corpus) carry-forward). `sourceVersion` records the +// manifest version that last wrote this slug (audit + cache-generation token). +aspect ContentCurrentAspect : managed { + key slug : String(255); + content : LargeBinary; + contentHash : Sha256; + sizeBytes : Integer; + compressedBytes : Integer; + mimeType : String(100) default 'text/html'; + sourceContent : LargeBinary; + sourceHash : Sha256; + sourceVersion : Integer; +} + +// Option B: append-only per-(version, slug) history for drift-detection +// (detectReverts) + rollback replay. Carries the BLOB (`content`/`sourceContent`) +// so rollback is a self-contained replay into ContentCurrent (design.md D2); +// GC'd by the repurposed cleanupContentVersions. `action=DELETED` tombstones a +// slug removed at that version so rollback can re-add/remove correctly. +aspect ContentHistoryAspect : managed { + key version : Integer; + key slug : String(255); + action : String(10) enum { WRITTEN; DELETED; }; + content : LargeBinary; + contentHash : Sha256; + sizeBytes : Integer; + compressedBytes : Integer; + mimeType : String(100) default 'text/html'; + sourceContent : LargeBinary; + sourceHash : Sha256; +} + aspect TutorialBodyTextAspect : managed { key slug : String(255); bodyText : LargeString; diff --git a/db/schema.cds b/db/schema.cds index c2be7e051..07ffce7df 100644 --- a/db/schema.cds +++ b/db/schema.cds @@ -598,6 +598,13 @@ entity ContentFiles : shared.ContentFilesAspect {} entity ContentManifest : shared.ContentManifestAspect {} +// Option B (slug-targeted-delta-rebuild): mutable current-content table + +// append-only history. Coexist with ContentFiles/ContentManifest during the +// flag-gated dual-write migration; ContentFiles is retired one release after cutover. +entity ContentCurrent : shared.ContentCurrentAspect {} + +entity ContentHistory : shared.ContentHistoryAspect {} + // Plain-text projection of published Hugo HTML, indexed for full-text search. // Replaced (not versioned) on every publish so search reflects current content. @cds.autoexpose: false diff --git a/docs/authors/tutorial-repo-dispatch.yml b/docs/authors/tutorial-repo-dispatch.yml index 1939d4497..85efeaf25 100644 --- a/docs/authors/tutorial-repo-dispatch.yml +++ b/docs/authors/tutorial-repo-dispatch.yml @@ -27,7 +27,13 @@ name: Notify Tutorials Platform on: push: - branches: [main] + # Source repos are a MIXTURE of default branches: most default to `main`, + # but some (e.g. sap-tutorials/Tutorials) still default to `master`. List + # BOTH so this template auto-publishes regardless of a repo's default — + # a `main`-only trigger silently never fires on a `master`-default repo + # (root cause of the cp-* PROD content-drift incidents). GitHub ignores a + # listed branch that doesn't exist, so [master, main] is safe everywhere. + branches: [master, main] paths: - '**.md' - '**.png' diff --git a/hugo-apps/src/devtoberfest-sessions-grid/App.vue b/hugo-apps/src/devtoberfest-sessions-grid/App.vue index 9d2159b02..6b6ba809a 100644 --- a/hugo-apps/src/devtoberfest-sessions-grid/App.vue +++ b/hugo-apps/src/devtoberfest-sessions-grid/App.vue @@ -1,5 +1,5 @@