diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index acf75f4fc..fb8ffe6a3 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -171,7 +171,7 @@ modules: - cp -r ../../hugo/assets ./hugo/assets - cp -r ../../hugo/data ./hugo/data - cp -r ../../hugo/i18n ./hugo/i18n - - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/island-manifest.json srv/lib/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" + - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/island-manifest.json srv/lib/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" - bash -c "node -e \"const p=require('./package.json'); p.dependencies=Object.assign(p.dependencies||{},{cheerio:'^1.2.0','@sap-ai-sdk/foundation-models':'^2.10.0'}); require('fs').writeFileSync('./package.json', JSON.stringify(p,null,2));\"" properties: EXPOSE_CAP_UI: false diff --git a/db/tutorial-assets.cds b/db/tutorial-assets.cds new file mode 100644 index 000000000..d030f5a54 --- /dev/null +++ b/db/tutorial-assets.cds @@ -0,0 +1,16 @@ +using { Attachments } from '@cap-js/attachments'; +using { com.sap.developers.ims.Tutorials } from './schema'; + +namespace com.sap.developers.ims; + +entity TutorialAssets { + key ID : UUID; + sourceUrl : String(1024); // raw.githubusercontent.com URL; one-row-per-sourceUrl maintained by attachment-store put() (delete-then-insert), NOT a DB constraint + tutorial : Association to Tutorials on tutorial.slug = slug; + slug : String(255); // lowercase canonical + channel : String(8); // 'prod' | 'qa' + contentHash : String(64); // sha-256 of stored bytes + mimeType : String(128); + filename : String(255); // for Content-Disposition + content : Composition of many Attachments; +} diff --git a/docs/superpowers/plans/2026-08-21-tutorial-attachment-object-store.md b/docs/superpowers/plans/2026-08-21-tutorial-attachment-object-store.md new file mode 100644 index 000000000..99d12847f --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-tutorial-attachment-object-store.md @@ -0,0 +1,1347 @@ +# Tutorial Attachment Object-Store Pipeline — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Serve tutorial repo attachment files (`.txt/.zip/.pdf/.csv/.json/...`) through the object store like images, so relative attachment links stop 404-ing (the #1931 follow-up). + +**Architecture:** Mirror the image asset pipeline minus resize/WebP. A parser rewrites relative allowlisted links to raw-GitHub URLs; a new Hugo `render-link.html` hook wraps them to `/content/attachment-source?u=` (plus a download sibling). A new `TutorialAssets` CDS entity (own `@cap-js/attachments` composition on the existing shared S3 binding) holds bytes. A push ingest endpoint + backfill script populate the store; an anonymous serve endpoint streams with inline/download disposition (`.html` neutered). No approuter changes — the existing `^/content/(.*)$ → srv-api` route carries the endpoint. + +**Tech Stack:** SAP CAP (Node.js, `@sap/cds`), `@cap-js/attachments` v4 + `@aws-sdk/client-s3`, Hugo/Goldmark render hooks, TypeScript parsers (`scripts/parsers/`), Vitest (`unit` project, in-memory SQLite via `cds.test`), Express handlers. + +**Spec:** `docs/superpowers/specs/2026-08-21-tutorial-attachment-object-store-design.md` + +## Global Constraints + +- **srv-qa cp-list audit:** any new `srv/lib/*` file reachable from `content-store.js`/`content-publish-session.js`/`server.js` at runtime MUST be added to the `srv-qa` `cp` list in `.deploy/mta.yaml` (line ~174) or QA boot crashes at deploy. Guarded by `scripts/check-srv-qa-cp-list.ts` (runs in `postbuild:apps`). +- **CDS model validation:** run `npx cds deploy --to sqlite::memory:` before committing any `db/**/*.cds` change. +- **Schema build:** after adding an entity, run `npm run build:cds` (`cds build --production`); never hand-author `.hdbmigrationtable` ALTERs. +- **No raw SQL:** use `cds.ql`/CQL. (Store copies the image-store CQL pattern.) +- **Never SELECT a HANA BLOB alongside non-BLOB metadata in one CDS QL query** — the store fetches metadata and content in separate steps (image-store pattern already does this). +- **Tutorial slugs are lowercase canonical** — never compare slugs without `.toLowerCase()`; the store keys on `sourceUrl`, not slug, so this is informational. +- **PR over direct merge:** land via `gh pr create` from the feature branch; never direct-merge to `main`. +- **Vitest:** run unit tests with `npx vitest run --project unit ` from repo root. +- **Malware scanning stays mocked** (`status:'Clean'` hardcoded) — image parity, documented gap. Do not wire a real scanner. +- **RAW_BASE_URL** = `https://raw.githubusercontent.com` (exported from `scripts/parsers/types.ts`). +- **Attachment host allowlist** = `raw.githubusercontent.com` only (matches images). + +--- + +## File Structure + +**Parser / build (TypeScript):** +- `scripts/parsers/attachment-links.ts` — allowlist + `resolveAttachmentLinks()` (relative → raw URL, fence-aware). +- `scripts/parsers/compose.ts` — MODIFY: call `resolveAttachmentLinks` after `resolveImageURLs`. +- `hugo/layouts/_default/_markup/render-link.html` — NEW Hugo hook: passthrough + attachment wrap + download sibling. + +**Data model:** +- `db/tutorial-assets.cds` — `TutorialAssets` entity. + +**Runtime (srv, JS/ESM + CJS):** +- `srv/lib/attachment-store.cjs` — head/put/getStream/remove (keyed by sourceUrl; persists filename). +- `srv/lib/attachment-mime.cjs` — `extToMime()`, `dispositionFor()`. +- `srv/lib/attachment-warm-utils.js` — `extractAttachmentUrls()`, `warmAttachments()` (reuses `channelFor` from image-warm-utils). +- `srv/lib/attachment-ingest.cjs` — `ingestAttachment()`. +- `srv/lib/attachment-source-handler.js` — GET serve handler + `warmAttachmentsLive()`. +- `srv/lib/attachment-ingest-handler.js` — POST push handler. +- `srv/server.js` — MODIFY: register the two routes. +- `srv/lib/content-publish-session.js` — MODIFY: fire attachment warm beside image warm. + +**Scripts / deploy:** +- `scripts/backfill-attachments.ts` — enumerate + push. +- `package.json` — MODIFY: add `backfill-attachments` script. +- `.deploy/mta.yaml` — MODIFY: add 4 new `srv/lib/attachment-*` files to srv-qa `cp` list. + +--- + +## Task 1: Attachment-link resolver (parser) + +**Files:** +- Create: `scripts/parsers/attachment-links.ts` +- Test: `test/parsers/attachment-links.test.ts` + +**Interfaces:** +- Produces: `ATTACHMENT_EXTENSIONS: Set` (lowercase, no dot); `isAttachmentPath(path: string): boolean`; `resolveAttachmentLinks(content: string, opts: { repo: string, branch: string, slug: string, rewrite?: boolean }): string`. +- Consumes: `RAW_BASE_URL` from `./types.js`; `createFenceTracker` from `./fence-tracker.js`. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/parsers/attachment-links.test.ts +import { describe, it, expect } from 'vitest' +import { resolveAttachmentLinks, isAttachmentPath } from '../../scripts/parsers/attachment-links.js' + +const opts = { repo: 'abap-core-development', branch: 'main', slug: 'rap100' } +const base = 'https://raw.githubusercontent.com/sap-tutorials/abap-core-development/main/tutorials/rap100' + +describe('resolveAttachmentLinks', () => { + it('rewrites a relative allowlisted link to a raw-GitHub URL', () => { + const out = resolveAttachmentLinks('[doc](EX2_DDLX.txt)', opts) + expect(out).toBe(`[doc](${base}/EX2_DDLX.txt)`) + }) + it('rewrites ./-prefixed links and strips the ./', () => { + expect(resolveAttachmentLinks('[d](./a.csv)', opts)).toBe(`[d](${base}/a.csv)`) + }) + it('leaves images (![]) untouched', () => { + expect(resolveAttachmentLinks('![alt](img.png)', opts)).toBe('![alt](img.png)') + }) + it('leaves absolute, anchor, mailto, root-relative, and ../ links untouched', () => { + for (const s of ['[a](https://x.com/f.txt)', '[a](#sec)', '[a](mailto:x@y.z)', '[a](/other/f.txt)', '[a](../sib/f.txt)']) { + expect(resolveAttachmentLinks(s, opts)).toBe(s) + } + }) + it('leaves non-allowlisted extensions untouched', () => { + expect(resolveAttachmentLinks('[a](page.aspx)', opts)).toBe('[a](page.aspx)') + }) + it('does not touch link-like text inside a fenced code block', () => { + const src = '```md\n[x](y.txt)\n```' + expect(resolveAttachmentLinks(src, opts)).toBe(src) + }) + it('is idempotent (already-raw URLs are left as-is)', () => { + const once = resolveAttachmentLinks('[d](EX2.txt)', opts) + expect(resolveAttachmentLinks(once, opts)).toBe(once) + }) + it('respects rewrite:false', () => { + expect(resolveAttachmentLinks('[d](a.txt)', { ...opts, rewrite: false })).toBe('[d](a.txt)') + }) + it('isAttachmentPath matches allowlist case-insensitively', () => { + expect(isAttachmentPath('X.TXT')).toBe(true) + expect(isAttachmentPath('x.png')).toBe(false) + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/parsers/attachment-links.test.ts` +Expected: FAIL — cannot resolve module `attachment-links.js`. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// scripts/parsers/attachment-links.ts +import { RAW_BASE_URL } from './types.js' +import { createFenceTracker } from './fence-tracker.js' + +// Repo attachment file extensions served through the object store (lowercase, no dot). +// KEEP IN SYNC with the Hugo render-link hook's disposition and srv/lib/attachment-mime.cjs. +export const ATTACHMENT_EXTENSIONS = new Set([ + 'txt', 'csv', 'json', 'md', 'sql', 'abap', 'properties', + 'yaml', 'yml', 'xml', 'html', 'zip', 'pdf', 'war', 'jar', 'zargo', 'har', +]) + +export function isAttachmentPath(path: string): boolean { + const m = /\.([a-z0-9]+)$/i.exec(path.trim()) + return m ? ATTACHMENT_EXTENSIONS.has(m[1].toLowerCase()) : false +} + +export interface AttachmentResolveOpts { + repo: string + branch: string + slug: string + rewrite?: boolean +} + +// Matches a markdown link `[text](dest)` NOT preceded by `!` (which would be an image). +// Destination captured up to whitespace or `)`; an optional `"title"` is preserved. +const LINK_RE = /(^|[^!])(\[[^\]]*\]\()([^)\s]+)((?:\s+"[^"]*")?\))/g + +export function resolveAttachmentLinks(content: string, opts: AttachmentResolveOpts): string { + const { repo, branch, slug, rewrite = true } = opts + if (!rewrite) return content + const base = `${RAW_BASE_URL}/sap-tutorials/${repo}/${branch}/tutorials/${slug}` + const fence = createFenceTracker() + return content + .split('\n') + .map((line) => { + if (fence(line)) return line // inside a code fence — leave verbatim + return line.replace(LINK_RE, (m, pre, open, dest, tail) => { + if (/^(https?:\/\/|#|mailto:|\/)/i.test(dest)) return m + if (dest.includes('../')) return m + if (!isAttachmentPath(dest)) return m + const clean = dest.replace(/^\.?\//, '') + return `${pre}${open}${base}/${clean}${tail}` + }) + }) + .join('\n') +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/parsers/attachment-links.test.ts` +Expected: PASS (all 9 cases). + +- [ ] **Step 5: Commit** + +```bash +git add scripts/parsers/attachment-links.ts test/parsers/attachment-links.test.ts +git commit -m "feat(attachments): parser resolver for relative attachment links (#1931)" +``` + +--- + +## Task 2: Wire the resolver into compose + +**Files:** +- Modify: `scripts/parsers/compose.ts` (right after the `resolveImageURLs(mergedBody, …)` call, ~line 108-112) +- Test: `test/parsers/attachment-links-compose.test.ts` (or extend an existing compose test) + +**Interfaces:** +- Consumes: `resolveAttachmentLinks` from `./attachment-links.js` (Task 1). + +- [ ] **Step 1: Write the failing test** + +```ts +// test/parsers/attachment-links-compose.test.ts +import { describe, it, expect } from 'vitest' +import { composeTutorial } from '../../scripts/parsers/compose.js' + +describe('compose rewrites attachment links in the body', () => { + it('body [doc](EX2.txt) becomes a raw-GitHub URL', () => { + const md = `---\ntitle: T\n---\n\n## Intro\n\nSee [doc](EX2.txt) below.\n` + const out = composeTutorial(md, { repo: 'abap-core-development', branch: 'main', slug: 'rap100', target: 'hugo' } as any) + expect(out.body ?? JSON.stringify(out)).toContain( + 'https://raw.githubusercontent.com/sap-tutorials/abap-core-development/main/tutorials/rap100/EX2.txt' + ) + }) +}) +``` + +> NOTE: inspect `composeTutorial`'s real signature/return shape in `scripts/parsers/compose.ts` and adjust the call/assertion to match (it returns a composed object; assert against the field that carries the rendered body). If an existing compose test already exercises `resolveImageURLs`, add this case there instead of a new file. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/parsers/attachment-links-compose.test.ts` +Expected: FAIL — body still contains the bare `EX2.txt`. + +- [ ] **Step 3: Add the call in compose.ts** + +Find (~line 108): +```ts + let processedBody = resolveImageURLs(mergedBody, { + repo: opts.repo, branch: opts.branch, slug: opts.slug, + rewriteImages: opts.rewriteImages, + }) +``` +Add immediately after it: +```ts + // [#1931] Rewrite relative attachment links (.txt/.zip/.pdf/...) to raw-GitHub + // URLs so the render-link hook can route them through /content/attachment-source. + // Gated by rewriteImages (same "resolve relative repo paths" switch as images). + processedBody = resolveAttachmentLinks(processedBody, { + repo: opts.repo, branch: opts.branch, slug: opts.slug, + rewrite: opts.rewriteImages, + }) +``` +Add the import at the top with the other parser imports: +```ts +import { resolveAttachmentLinks } from './attachment-links.js' +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/parsers/attachment-links-compose.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/parsers/compose.ts test/parsers/attachment-links-compose.test.ts +git commit -m "feat(attachments): wire attachment-link resolver into compose (#1931)" +``` + +--- + +## Task 3: Hugo render-link hook + +**Files:** +- Create: `hugo/layouts/_default/_markup/render-link.html` +- Test: `test/parsers/render-link-hook.test.ts` (template-content assertions — a full Hugo build is verified in Task 16) + +**Interfaces:** +- Emits browser URL shape `/content/attachment-source?u=` and download sibling `…&dl=1`. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/parsers/render-link-hook.test.ts +import { describe, it, expect } from 'vitest' +import { readFileSync, existsSync } from 'node:fs' + +const p = 'hugo/layouts/_default/_markup/render-link.html' + +describe('render-link hook', () => { + it('exists', () => { expect(existsSync(p)).toBe(true) }) + it('wraps raw.githubusercontent destinations to the attachment endpoint', () => { + const t = readFileSync(p, 'utf8') + expect(t).toContain('raw.githubusercontent.com') + expect(t).toContain('/content/attachment-source?u=') + expect(t).toContain('dl=1') // download sibling + expect(t).toContain('urlquery') // encodes the source URL + }) + it('has a passthrough branch for non-attachment links', () => { + const t = readFileSync(p, 'utf8') + expect(t).toContain('.Destination | safeURL') // default anchor emission + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/parsers/render-link-hook.test.ts` +Expected: FAIL — file does not exist. + +- [ ] **Step 3: Write the hook** + +```go-html-template +{{- /* Render hook for links (issue #1931). + Default behavior for all links is Hugo's standard emission. Attachment + links whose destination is a raw.githubusercontent.com URL with an + allowlisted extension (rewritten by scripts/parsers/attachment-links.ts) + are routed through the CAP attachment store and get a download sibling. + KEEP the extension list in sync with attachment-links.ts / attachment-mime.cjs. */ -}} +{{- $dest := .Destination -}} +{{- $exts := slice "txt" "csv" "json" "md" "sql" "abap" "properties" "yaml" "yml" "xml" "html" "zip" "pdf" "war" "jar" "zargo" "har" -}} +{{- $ext := lower (path.Ext $dest | strings.TrimPrefix ".") -}} +{{- $isAttachment := and (hasPrefix $dest "https://raw.githubusercontent.com/") (in $exts $ext) -}} +{{- if $isAttachment -}} + {{- $enc := $dest | urlquery -}} + {{- $view := printf "/content/attachment-source?u=%s" $enc -}} + {{- $dl := printf "/content/attachment-source?u=%s&dl=1" $enc -}} + {{ .Text | safeHTML }}↓ +{{- else -}} + {{ .Text | safeHTML }} +{{- end -}} +``` + +> NOTE: verify against the running Hugo version that `path.Ext`, `strings.TrimPrefix`, and `in` are available (they are in modern Hugo). Confirm no OTHER partial already emits links in a way this would double-wrap; the option-tabs shortcode reprocesses inner markdown, so keep the emitted `` on a single line (same single-line rule as `render-image.html`, issue #1591) — the template above keeps each `` unbroken. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/parsers/render-link-hook.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add hugo/layouts/_default/_markup/render-link.html test/parsers/render-link-hook.test.ts +git commit -m "feat(attachments): Hugo render-link hook routes attachment links to store (#1931)" +``` + +--- + +## Task 4: TutorialAssets CDS entity + +**Files:** +- Create: `db/tutorial-assets.cds` +- Test: `test/unit/tutorial-assets-model.test.js` (mirror `test/unit/tutorial-images-model.test.js`) + +**Interfaces:** +- Produces entity `com.sap.developers.ims.TutorialAssets` with composition `…TutorialAssets.content`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/tutorial-assets-model.test.js +import { describe, it, expect } from 'vitest' +import cds from '@sap/cds' + +describe('TutorialAssets model', () => { + it('compiles with a filename column and an Attachments composition', async () => { + const m = await cds.load(['db/tutorial-assets.cds'], { root: '.' }) + const e = cds.linked(m).definitions['com.sap.developers.ims.TutorialAssets'] + expect(e).toBeTruthy() + expect(e.elements.sourceUrl.length).toBe(1024) + expect(e.elements.filename).toBeTruthy() + expect(e.elements.content.type).toBe('cds.Composition') + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/tutorial-assets-model.test.js` +Expected: FAIL — file `db/tutorial-assets.cds` not found. + +- [ ] **Step 3: Write the entity** + +```cds +using { Attachments } from '@cap-js/attachments'; +using { com.sap.developers.ims.Tutorials } from './schema'; + +namespace com.sap.developers.ims; + +entity TutorialAssets { + key ID : UUID; + sourceUrl : String(1024); // raw.githubusercontent.com URL; one-row-per-sourceUrl maintained by attachment-store put() (delete-then-insert), NOT a DB constraint + tutorial : Association to Tutorials on tutorial.slug = slug; + slug : String(255); // lowercase canonical + channel : String(8); // 'prod' | 'qa' + contentHash : String(64); // sha-256 of stored bytes + mimeType : String(128); + filename : String(255); // for Content-Disposition + content : Composition of many Attachments; +} +``` + +- [ ] **Step 4: Run test + validate model deploys** + +Run: `npx vitest run --project unit test/unit/tutorial-assets-model.test.js` +Expected: PASS. +Then (Global Constraint): `npx cds deploy --to sqlite::memory:` +Expected: exits 0, no compile error. + +- [ ] **Step 5: Commit** + +```bash +git add db/tutorial-assets.cds test/unit/tutorial-assets-model.test.js +git commit -m "feat(attachments): TutorialAssets entity (#1931)" +``` + +--- + +## Task 5: attachment-store.cjs + +**Files:** +- Create: `srv/lib/attachment-store.cjs` +- Test: `test/unit/attachment-store.test.js` (mirror `test/unit/image-store.test.js`) + +**Interfaces:** +- Produces: `head(sourceUrl) → {exists, ID?, contentHash?, mimeType?, filename?}`; `put(sourceUrl, {buffer, mimeType, contentHash, slug, channel, filename}) → void`; `getStream(sourceUrl) → {stream, mimeType, filename} | null`; `remove(sourceUrl) → void`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/attachment-store.test.js +import { describe, it, expect } from 'vitest' +import cds from '@sap/cds' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +cds.test('serve', '--project', '.', '--in-memory') +const store = require('../../srv/lib/attachment-store.cjs') + +describe('attachment-store round-trip', () => { + const url = 'https://raw.githubusercontent.com/o/r/main/EX2.txt' + it('put → head → getStream returns the same bytes, mime, and filename', async () => { + const buffer = Buffer.from('@Search.searchable: true', 'utf8') + await store.put(url, { buffer, mimeType: 'text/plain; charset=utf-8', contentHash: 'h1', slug: 's', channel: 'prod', filename: 'EX2.txt' }) + const h = await store.head(url) + expect(h.exists).toBe(true) + expect(h.contentHash).toBe('h1') + expect(h.filename).toBe('EX2.txt') + const got = await store.getStream(url) + const chunks = [] + for await (const c of got.stream) chunks.push(c) + expect(Buffer.concat(chunks)).toEqual(buffer) + expect(got.mimeType).toMatch(/text\/plain/) + expect(got.filename).toBe('EX2.txt') + }) + it('head returns exists:false for an unknown url', async () => { + expect((await store.head('https://raw.githubusercontent.com/o/r/main/none.txt')).exists).toBe(false) + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/attachment-store.test.js` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the store (copy of image-store.cjs, entity swapped, filename added)** + +```js +'use strict' +const cds = require('@sap/cds') +const { Readable } = require('node:stream') + +// Metadata on TutorialAssets; original bytes in its Attachments composition. +// Mirror of image-store.cjs; see that file for the withCtx/tenant rationale. +function linkedContent() { + return cds.linked(cds.model).definitions['com.sap.developers.ims.TutorialAssets.content'] +} +function withCtx(fn) { return cds.context ? fn() : cds.tx(fn) } + +async function head(sourceUrl) { + return withCtx(async () => { + const { TutorialAssets } = cds.entities('com.sap.developers.ims') + const row = await SELECT.one.from(TutorialAssets) + .columns('ID', 'contentHash', 'mimeType', 'filename').where({ sourceUrl }) + return row + ? { exists: true, ID: row.ID, contentHash: row.contentHash, mimeType: row.mimeType, filename: row.filename } + : { exists: false } + }) +} + +async function put(sourceUrl, { buffer, mimeType, contentHash, slug, channel, filename }) { + return withCtx(async () => { + const { TutorialAssets } = cds.entities('com.sap.developers.ims') + await remove(sourceUrl) // delete-then-insert avoids NonUpdatableProperties:[content] 409 + const parentID = cds.utils.uuid() + const name = filename || sourceUrl.split('/').pop() + await INSERT.into(TutorialAssets).entries({ ID: parentID, sourceUrl, slug, channel, contentHash, mimeType, filename: name }) + const AttachmentsSrv = await cds.connect.to('attachments') + await AttachmentsSrv.put(linkedContent(), { + ID: cds.utils.uuid(), up__ID: parentID, url: cds.utils.uuid(), + content: Readable.from(buffer), mimeType, filename: name, status: 'Clean', + }) + }) +} + +async function getStream(sourceUrl) { + return withCtx(async () => { + const meta = await head(sourceUrl) + if (!meta.exists) return null + const Content = linkedContent() + const att = await SELECT.one.from(Content).columns('ID').where({ up__ID: meta.ID }) + if (!att) return null + const AttachmentsSrv = await cds.connect.to('attachments') + const stream = await AttachmentsSrv.get(Content, { ID: att.ID }) + return stream ? { stream, mimeType: meta.mimeType, filename: meta.filename } : null + }) +} + +async function remove(sourceUrl) { + return withCtx(async () => { + const { TutorialAssets } = cds.entities('com.sap.developers.ims') + await DELETE.from(TutorialAssets).where({ sourceUrl }) + }) +} + +module.exports = { head, put, getStream, remove } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/attachment-store.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/attachment-store.cjs test/unit/attachment-store.test.js +git commit -m "feat(attachments): attachment-store CRUD on TutorialAssets (#1931)" +``` + +--- + +## Task 6: attachment-mime.cjs (MIME + disposition) + +**Files:** +- Create: `srv/lib/attachment-mime.cjs` +- Test: `test/unit/attachment-mime.test.js` + +**Interfaces:** +- Produces: `extToMime(filenameOrUrl: string) → string`; `dispositionFor(mimeType: string, opts: { download?: boolean, filename?: string }) → { contentType: string, disposition: string }`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/attachment-mime.test.js +import { describe, it, expect } from 'vitest' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +const { extToMime, dispositionFor } = require('../../srv/lib/attachment-mime.cjs') + +describe('extToMime', () => { + it('maps known extensions', () => { + expect(extToMime('a.txt')).toMatch(/text\/plain/) + expect(extToMime('a.json')).toBe('application/json') + expect(extToMime('a.csv')).toBe('text/csv') + expect(extToMime('a.zip')).toBe('application/zip') + expect(extToMime('a.pdf')).toBe('application/pdf') + }) + it('falls back to octet-stream for unknown', () => { + expect(extToMime('a.bin')).toBe('application/octet-stream') + }) +}) + +describe('dispositionFor', () => { + it('text types serve inline', () => { + expect(dispositionFor('text/plain; charset=utf-8', { filename: 'a.txt' }).disposition).toMatch(/^inline/) + }) + it('binaries force attachment with filename', () => { + const d = dispositionFor('application/zip', { filename: 'a.zip' }) + expect(d.disposition).toBe('attachment; filename="a.zip"') + }) + it('text/html is neutered to text/plain inline', () => { + const d = dispositionFor('text/html', { filename: 'a.html' }) + expect(d.contentType).toMatch(/text\/plain/) + expect(d.disposition).toMatch(/^inline/) + }) + it('download:true forces attachment for any type', () => { + const d = dispositionFor('text/plain; charset=utf-8', { download: true, filename: 'a.txt' }) + expect(d.disposition).toBe('attachment; filename="a.txt"') + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/attachment-mime.test.js` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the module** + +```js +'use strict' +// Extension→MIME + Content-Disposition policy for tutorial attachments (#1931). +// KEEP the extension set in sync with scripts/parsers/attachment-links.ts and the render-link hook. + +const EXT_MIME = { + txt: 'text/plain; charset=utf-8', csv: 'text/csv', json: 'application/json', + md: 'text/markdown; charset=utf-8', sql: 'text/plain; charset=utf-8', + abap: 'text/plain; charset=utf-8', properties: 'text/plain; charset=utf-8', + yaml: 'text/plain; charset=utf-8', yml: 'text/plain; charset=utf-8', + xml: 'text/plain; charset=utf-8', html: 'text/html', + zip: 'application/zip', pdf: 'application/pdf', + war: 'application/java-archive', jar: 'application/java-archive', + zargo: 'application/octet-stream', har: 'application/json', +} + +function extToMime(filenameOrUrl) { + const m = /\.([a-z0-9]+)(?:[?#].*)?$/i.exec(String(filenameOrUrl)) + const ext = m ? m[1].toLowerCase() : '' + return EXT_MIME[ext] || 'application/octet-stream' +} + +// Inline-viewable MIME classes (rest download). +const INLINE = new Set(['text/plain', 'text/csv', 'text/markdown', 'application/json', 'application/xml']) + +function baseType(mime) { return String(mime).split(';')[0].trim().toLowerCase() } + +function dispositionFor(mimeType, { download = false, filename = 'file' } = {}) { + const safeName = String(filename).replace(/"/g, '') + // text/html is neutered: serve as text/plain, inline, never executed. + if (baseType(mimeType) === 'text/html' && !download) { + return { contentType: 'text/plain; charset=utf-8', disposition: `inline; filename="${safeName}"` } + } + if (download) return { contentType: mimeType, disposition: `attachment; filename="${safeName}"` } + const inline = INLINE.has(baseType(mimeType)) + return { contentType: mimeType, disposition: `${inline ? 'inline' : 'attachment'}; filename="${safeName}"` } +} + +module.exports = { extToMime, dispositionFor } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/attachment-mime.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/attachment-mime.cjs test/unit/attachment-mime.test.js +git commit -m "feat(attachments): ext→MIME + Content-Disposition policy (#1931)" +``` + +--- + +## Task 7: attachment-warm-utils.js + +**Files:** +- Create: `srv/lib/attachment-warm-utils.js` +- Test: `test/unit/attachment-warm-utils.test.js` + +**Interfaces:** +- Consumes: `channelFor` from `./image-warm-utils.js` (re-export). +- Produces: `extractAttachmentUrls(html: string) → string[]`; `warmAttachments(urls, { slug, ingestFn }) → Promise`; re-export `channelFor`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/attachment-warm-utils.test.js +import { describe, it, expect, vi } from 'vitest' +import { extractAttachmentUrls, warmAttachments } from '../../srv/lib/attachment-warm-utils.js' + +describe('extractAttachmentUrls', () => { + it('extracts and decodes u= from attachment-source hrefs (view + dl)', () => { + const raw = 'https://raw.githubusercontent.com/o/r/main/tutorials/s/EX2.txt' + const enc = encodeURIComponent(raw) + const html = `d↓` + expect(extractAttachmentUrls(html)).toEqual([raw]) // deduped + }) + it('returns [] when there are no attachment links', () => { + expect(extractAttachmentUrls('

no links

')).toEqual([]) + }) +}) + +describe('warmAttachments', () => { + it('calls ingestFn per url and never throws on failure', async () => { + const ingestFn = vi.fn().mockRejectedValue(new Error('boom')) + await expect(warmAttachments(['a', 'b'], { slug: 's', ingestFn })).resolves.toBeUndefined() + expect(ingestFn).toHaveBeenCalledTimes(2) + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/attachment-warm-utils.test.js` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the module** + +```js +// srv/lib/attachment-warm-utils.js +// Pure warm-orchestration utilities for tutorial attachments (mirror of image-warm-utils.js). +export { channelFor } from './image-warm-utils.js' +import { channelFor } from './image-warm-utils.js' + +/** + * Extract + decode unique source URLs from `/content/attachment-source?u=` hrefs. + * Matches both `?u=` and `&u=`; captures up to the next & / quote / whitespace / >. + * @param {string} html + * @returns {string[]} + */ +export function extractAttachmentUrls(html) { + const results = new Set() + const re = /\/content\/attachment-source[^"'\s>]*[?&]u=([^&"'\s>]+)/g + let m + while ((m = re.exec(html)) !== null) { + try { results.add(decodeURIComponent(m[1])) } catch { /* skip malformed */ } + } + return [...results] +} + +/** + * Warm the attachment store for `urls`. Per-URL try/catch; always resolves. + * @param {string[]} urls + * @param {{ slug: string, ingestFn: (url: string, opts: {slug: string, channel: string}) => Promise<{action: string, status?: number}> }} opts + * @returns {Promise} + */ +export async function warmAttachments(urls, { slug, ingestFn }) { + for (const url of urls) { + const channel = channelFor(url) + try { + const result = await ingestFn(url, { slug, channel }) + if (result?.action === 'failed') { + console.warn(`[attachment-warm] slug=${slug} u=${url}: ingest failed (status=${result.status})`) + } + } catch (err) { + console.warn(`[attachment-warm] slug=${slug} u=${url}: ingest threw: ${err?.message}`) + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/attachment-warm-utils.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/attachment-warm-utils.js test/unit/attachment-warm-utils.test.js +git commit -m "feat(attachments): warm-utils (extract + orchestrate) (#1931)" +``` + +--- + +## Task 8: attachment-ingest.cjs + +**Files:** +- Create: `srv/lib/attachment-ingest.cjs` +- Test: `test/unit/attachment-ingest.test.js` (mirror `test/unit/image-ingest.test.js`) + +**Interfaces:** +- Consumes: `fetchImageResponse` from `./img-cdn-fetch.cjs` (reused — generic HTTP fetch with anon-first/token-on-404), `extToMime` from `./attachment-mime.cjs`, store from Task 5. +- Produces: `ingestAttachment(sourceUrl, { slug, channel, deps }) → { action: 'stored'|'unchanged'|'failed', status?, contentHash?, mimeType? }` where `deps = { fetchImageResponse, safeFetch, resolveSecret, store, hash? }`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/attachment-ingest.test.js +import { describe, it, expect, vi } from 'vitest' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +const { ingestAttachment } = require('../../srv/lib/attachment-ingest.cjs') + +function res(body, { ok = true, status = 200, ct = 'text/plain' } = {}) { + return { ok, status, headers: new Map([['content-type', ct], ['content-length', String(body.length)]]), + arrayBuffer: async () => Buffer.from(body) } +} + +describe('ingestAttachment', () => { + const url = 'https://raw.githubusercontent.com/o/r/main/EX2.txt' + it('stores on a fresh URL', async () => { + const store = { head: vi.fn().mockResolvedValue({ exists: false }), put: vi.fn().mockResolvedValue() } + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('hello')), safeFetch: {}, resolveSecret: {}, store } + const out = await ingestAttachment(url, { slug: 's', channel: 'prod', deps }) + expect(out.action).toBe('stored') + expect(store.put).toHaveBeenCalledOnce() + }) + it('is unchanged when hash matches', async () => { + const buf = Buffer.from('hello') + const crypto = require('node:crypto') + const h = crypto.createHash('sha256').update(buf).digest('hex') + const store = { head: vi.fn().mockResolvedValue({ exists: true, contentHash: h }), put: vi.fn() } + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('hello')), store } + const out = await ingestAttachment(url, { slug: 's', channel: 'prod', deps }) + expect(out.action).toBe('unchanged') + expect(store.put).not.toHaveBeenCalled() + }) + it('fails on a non-ok fetch', async () => { + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('', { ok: false, status: 404 })), store: {} } + const out = await ingestAttachment(url, { slug: 's', channel: 'prod', deps }) + expect(out).toEqual({ action: 'failed', status: 404 }) + }) + it('uses extToMime when the response content-type is generic', async () => { + const store = { head: vi.fn().mockResolvedValue({ exists: false }), put: vi.fn().mockResolvedValue() } + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('{}', { ct: 'application/octet-stream' })), store } + await ingestAttachment('https://raw.githubusercontent.com/o/r/main/a.json', { slug: 's', channel: 'prod', deps }) + expect(store.put.mock.calls[0][1].mimeType).toBe('application/json') + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/attachment-ingest.test.js` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the module (copy of image-ingest.cjs, MIME from extToMime)** + +```js +'use strict' +const crypto = require('node:crypto') +const { extToMime } = require('./attachment-mime.cjs') +const ATTACHMENT_HOSTS = new Set(['raw.githubusercontent.com']) +const MAX_BYTES = Number(process.env.ATTACHMENT_MAX_BYTES) || 25 * 1024 * 1024 + +async function ingestAttachment(sourceUrl, { slug, channel, deps }) { + const { fetchImageResponse, safeFetch, resolveSecret, store, + hash = (buf) => crypto.createHash('sha256').update(buf).digest('hex') } = deps + let host + try { host = new URL(sourceUrl).hostname } catch { return { action: 'failed', status: 400 } } + + const res = await fetchImageResponse(sourceUrl, { + safeFetch, resolveSecret, host, allowedHosts: ATTACHMENT_HOSTS, timeoutMs: 12000, maxRetries: 2, + }) + if (!res.ok) return { action: 'failed', status: res.status } + + const contentLength = Number(res.headers.get('content-length')) + if (!Number.isNaN(contentLength) && contentLength > MAX_BYTES) return { action: 'failed', status: 413 } + + const buffer = Buffer.from(await res.arrayBuffer()) + if (buffer.length > MAX_BYTES) return { action: 'failed', status: 413 } + + const contentHash = hash(buffer) + const existing = await store.head(sourceUrl) + if (existing.exists && existing.contentHash === contentHash) return { action: 'unchanged', contentHash } + + // GitHub serves most text attachments as text/plain; trust a specific content-type, + // otherwise derive from the extension so .json/.csv/.pdf get correct types. + const ct = res.headers.get('content-type') || '' + const mimeType = (ct && ct !== 'application/octet-stream') ? ct : extToMime(sourceUrl) + const filename = sourceUrl.split('/').pop() + await store.put(sourceUrl, { buffer, mimeType, contentHash, slug, channel, filename }) + return { action: 'stored', contentHash, mimeType } +} + +module.exports = { ingestAttachment } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/attachment-ingest.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/attachment-ingest.cjs test/unit/attachment-ingest.test.js +git commit -m "feat(attachments): ingestAttachment with ext-aware MIME (#1931)" +``` + +--- + +## Task 9: attachment-source-handler.js (serve + warm-live) + +**Files:** +- Create: `srv/lib/attachment-source-handler.js` +- Test: `test/unit/attachment-source-endpoint.test.js` (mirror `test/unit/image-source-endpoint.test.js`) + +**Interfaces:** +- Consumes: store (Task 5), `ingestAttachment` (Task 8), `dispositionFor` (Task 6), `channelFor`/`warmAttachments` (Task 7), `fetchImageResponse` (`./img-cdn-fetch.cjs`), `safeFetch`, `resolveSecret`. +- Produces: `attachmentSourceHandler(req, res)`; `warmAttachmentsLive(urls, { slug })`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/attachment-source-endpoint.test.js +import { describe, it, expect } from 'vitest' +import cds from '@sap/cds' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +const project = cds.test('serve', '--project', '.', '--in-memory') +const store = require('../../srv/lib/attachment-store.cjs') +const base = '/content/attachment-source' + +describe('GET /content/attachment-source', () => { + it('streams a stored .txt inline', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/EX2.txt' + await store.put(url, { buffer: Buffer.from('code'), mimeType: 'text/plain; charset=utf-8', contentHash: 'h', slug: 's', channel: 'prod', filename: 'EX2.txt' }) + const res = await project.get(`${base}?u=${encodeURIComponent(url)}`, { responseType: 'arraybuffer' }) + expect(res.status).toBe(200) + expect(res.headers['content-type']).toMatch(/text\/plain/) + expect(res.headers['content-disposition']).toMatch(/^inline/) + expect(res.headers['x-content-type-options']).toBe('nosniff') + }) + it('dl=1 forces attachment disposition', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/D2.txt' + await store.put(url, { buffer: Buffer.from('x'), mimeType: 'text/plain; charset=utf-8', contentHash: 'h2', slug: 's', channel: 'prod', filename: 'D2.txt' }) + const res = await project.get(`${base}?u=${encodeURIComponent(url)}&dl=1`, { responseType: 'arraybuffer' }) + expect(res.headers['content-disposition']).toMatch(/^attachment/) + }) + it('400 on missing u', async () => { + await expect(project.get(base)).rejects.toMatchObject({ response: { status: 400 } }) + }) + it('404 on a miss that cannot self-heal (github-blocked in test)', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/missing.txt' + await expect(project.get(`${base}?u=${encodeURIComponent(url)}`)).rejects.toMatchObject({ response: { status: 404 } }) + }) +}) +``` + +> NOTE: the 404 self-heal case relies on the srv being unable to fetch GitHub in the test env (network-blocked / anon-404), mirroring `img-store-github-blocked.test.js`. If the test env has network, stub `fetchImageResponse` or assert on `img-store-github-blocked.test.js`'s approach. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/attachment-source-endpoint.test.js` +Expected: FAIL — handler/route not present. + +- [ ] **Step 3: Write the handler** (Task 11 registers the route; this test will stay red until Task 11 — that is expected. Implement the handler now; re-run after Task 11.) + +```js +// srv/lib/attachment-source-handler.js +// Express handler for GET /content/attachment-source?u=&dl=. Anonymous. +// Streams the stored attachment; self-heals on miss (single-flight). Mirror of image-source-handler.js. +import cds from '@sap/cds' +import { createRequire } from 'node:module' +import { safeFetch } from './safe-fetch.js' +import { resolveSecret } from './secret-resolver.js' +import { channelFor, warmAttachments } from './attachment-warm-utils.js' + +const require = createRequire(import.meta.url) +const attachmentStore = require('./attachment-store.cjs') +const { ingestAttachment } = require('./attachment-ingest.cjs') +const { dispositionFor } = require('./attachment-mime.cjs') +const { fetchImageResponse } = require('./img-cdn-fetch.cjs') + +const LOG = cds.log('attachment-source') +const _inflight = new Map() + +export function warmAttachmentsLive(urls, { slug }) { + const ingestFn = (url, { slug: s, channel }) => + ingestAttachment(url, { slug: s, channel, deps: { fetchImageResponse, safeFetch, resolveSecret, store: attachmentStore } }) + return warmAttachments(urls, { slug, ingestFn }) +} + +export async function attachmentSourceHandler(req, res) { + const u = req.query.u + if (!u) return res.status(400).json({ error: 'Missing u parameter' }) + const download = req.query.dl === '1' || req.query.dl === 'true' + + let got = await attachmentStore.getStream(u) + if (!got) { + let p = _inflight.get(u) + if (!p) { + const channel = channelFor(u) + p = ingestAttachment(u, { slug: '', channel, + deps: { fetchImageResponse, safeFetch, resolveSecret, store: attachmentStore } }) + .finally(() => _inflight.delete(u)) + _inflight.set(u, p) + } + let result = { action: 'failed' } + try { result = await p } catch (err) { LOG.warn('[attachment-source] self-heal threw:', err.message) } + if (result.action === 'failed') return res.status(404).json({ error: 'Attachment unavailable' }) + got = await attachmentStore.getStream(u) + if (!got) return res.status(404).json({ error: 'Attachment unavailable' }) + } + + const filename = got.filename || String(u).split('/').pop() || 'file' + const { contentType, disposition } = dispositionFor(got.mimeType || 'application/octet-stream', { download, filename }) + res.setHeader('Content-Type', contentType) + res.setHeader('Content-Disposition', disposition) + res.setHeader('X-Content-Type-Options', 'nosniff') + res.setHeader('Cache-Control', 'public, max-age=3600') + res.setHeader('X-Content-Source', 'attachment-store') + + got.stream.on('error', (err) => { + LOG.warn('[attachment-source] stream error:', err.message) + if (!res.headersSent) res.status(500).json({ error: 'Stream error' }) + }) + got.stream.pipe(res) +} +``` + +- [ ] **Step 4: Commit (test remains red until Task 11 registers the route)** + +```bash +git add srv/lib/attachment-source-handler.js test/unit/attachment-source-endpoint.test.js +git commit -m "feat(attachments): serve handler with inline/download disposition (#1931)" +``` + +--- + +## Task 10: attachment-ingest-handler.js (POST push) + +**Files:** +- Create: `srv/lib/attachment-ingest-handler.js` +- Test: `test/unit/attachment-ingest-endpoint.test.js` (mirror `test/unit/image-ingest-endpoint.test.js`) + +**Interfaces:** +- Consumes: store (Task 5), `channelFor` (Task 7), `extToMime` (Task 6). +- Produces: `attachmentIngestHandler(req, res)` — POST `/content/attachment?u=&slug=&channel=&force=`, body = raw bytes. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/attachment-ingest-endpoint.test.js +import { describe, it, expect } from 'vitest' +import cds from '@sap/cds' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +const project = cds.test('serve', '--project', '.', '--in-memory') +const store = require('../../srv/lib/attachment-store.cjs') +const base = '/content/attachment' +const KEY = process.env.CONTENT_API_KEY || 'test-key' // set in vitest env for this suite + +describe('POST /content/attachment', () => { + it('stores pushed bytes then serves them', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/push.txt' + const res = await project.post(`${base}?u=${encodeURIComponent(url)}&slug=s`, Buffer.from('pushed'), { + headers: { authorization: `Bearer ${KEY}`, 'content-type': 'text/plain' }, + }) + expect(res.status).toBe(200) + expect(res.data.action).toBe('stored') + const got = await store.getStream(url) + const chunks = []; for await (const c of got.stream) chunks.push(c) + expect(Buffer.concat(chunks).toString()).toBe('pushed') + }) + it('401 without the api key', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/noauth.txt' + await expect(project.post(`${base}?u=${encodeURIComponent(url)}`, Buffer.from('x'), + { headers: { 'content-type': 'text/plain' } })).rejects.toMatchObject({ response: { status: 401 } }) + }) +}) +``` + +> NOTE: `CONTENT_API_KEY` must be present for `contentAuthMiddleware`. Mirror how `image-ingest-endpoint.test.js` sets it (env var in the suite or `vitest.config.ts`). Copy that exact setup. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/attachment-ingest-endpoint.test.js` +Expected: FAIL — route not registered. + +- [ ] **Step 3: Write the handler** (copy of image-ingest-handler.js, ext-aware MIME) + +```js +// srv/lib/attachment-ingest-handler.js +// POST /content/attachment?u=&slug=&channel=&force= — persist client-supplied attachment bytes. +// Bytes-in (srv CF egress is GitHub-anon-404'd, same as images). Auth: CONTENT_API_KEY. +import cds from '@sap/cds' +import crypto from 'node:crypto' +import { createRequire } from 'node:module' +import { channelFor } from './attachment-warm-utils.js' + +const require = createRequire(import.meta.url) +const attachmentStore = require('./attachment-store.cjs') +const { extToMime } = require('./attachment-mime.cjs') +const LOG = cds.log('attachment-ingest') +const MAX_BYTES = Number(process.env.ATTACHMENT_MAX_BYTES) || 25 * 1024 * 1024 + +export async function attachmentIngestHandler(req, res) { + const u = req.query.u + if (!u) return res.status(400).json({ error: 'Missing u parameter' }) + const buffer = req.body + if (!Buffer.isBuffer(buffer) || buffer.length === 0) return res.status(400).json({ error: 'Empty body' }) + if (buffer.length > MAX_BYTES) return res.status(400).json({ error: 'Attachment too large' }) + + const slug = typeof req.query.slug === 'string' ? req.query.slug : '' + const channel = typeof req.query.channel === 'string' && req.query.channel ? req.query.channel : channelFor(u) + const reqCt = req.get('content-type') || '' + const mimeType = (reqCt && reqCt !== 'application/octet-stream') ? reqCt : extToMime(u) + const filename = String(u).split('/').pop() + const contentHash = crypto.createHash('sha256').update(buffer).digest('hex') + const force = req.query.force === '1' || req.query.force === 'true' + + try { + if (!force) { + const existing = await attachmentStore.head(u) + if (existing.exists && existing.contentHash === contentHash) return res.status(200).json({ action: 'unchanged', contentHash }) + } + await attachmentStore.put(u, { buffer, mimeType, contentHash, slug, channel, filename }) + return res.status(200).json({ action: 'stored', contentHash }) + } catch (err) { + LOG.error('[attachment-ingest] store put failed for', u, '-', err.message) + return res.status(500).json({ error: 'store write failed' }) + } +} +``` + +- [ ] **Step 4: Commit (test red until Task 11)** + +```bash +git add srv/lib/attachment-ingest-handler.js test/unit/attachment-ingest-endpoint.test.js +git commit -m "feat(attachments): POST push ingest handler (#1931)" +``` + +--- + +## Task 11: Register routes in server.js + +**Files:** +- Modify: `srv/server.js` (imports near line 31-32; route registration near lines 523-527) + +**Interfaces:** +- Consumes: `attachmentSourceHandler` (Task 9), `attachmentIngestHandler` (Task 10), existing `contentAuthMiddleware`. + +- [ ] **Step 1: Add imports** (next to the image handler imports) + +```js +import { attachmentSourceHandler } from './lib/attachment-source-handler.js'; +import { attachmentIngestHandler } from './lib/attachment-ingest-handler.js'; +``` + +- [ ] **Step 2: Register the routes** (next to the `/content/image` routes, ~lines 523-527) + +```js + app.get('/content/attachment-source', attachmentSourceHandler); + app.post('/content/attachment', contentAuthMiddleware, express.raw({ type: '*/*', limit: '25mb' }), attachmentIngestHandler); +``` + +- [ ] **Step 3: Run the serve + ingest endpoint tests (now they can pass)** + +Run: `npx vitest run --project unit test/unit/attachment-source-endpoint.test.js test/unit/attachment-ingest-endpoint.test.js` +Expected: PASS (all cases from Tasks 9 & 10). + +- [ ] **Step 4: Commit** + +```bash +git add srv/server.js +git commit -m "feat(attachments): register /content/attachment-source + /content/attachment routes (#1931)" +``` + +--- + +## Task 12: Warm attachments on publish + +**Files:** +- Modify: `srv/lib/content-publish-session.js` (inside the `setImmediate` warm block, ~lines 248-261) +- Test: `test/unit/attachment-warm-utils.test.js` already covers extraction; add an assertion here that the block imports the attachment warm modules (guards against a copy/paste regression). + +- [ ] **Step 1: Add attachment warm beside image warm** + +Inside the existing `if (slugHtmlMap.size > 0) { setImmediate(async () => { … }) }` block, after the image-warm loop, add: +```js + const { extractAttachmentUrls } = await import('./attachment-warm-utils.js'); + const { warmAttachmentsLive } = await import('./attachment-source-handler.js'); + for (const [slug, html] of slugHtmlMap) { + const aUrls = extractAttachmentUrls(html); + if (aUrls.length > 0) await warmAttachmentsLive(aUrls, { slug }); + } +``` +> These run in the same `try` as the image warm (failures already swallowed and logged; never fail publish). In DEV the srv can't fetch GitHub so these warm calls no-op-fail — population is via Task 13 backfill. In PROD they auto-populate. + +- [ ] **Step 2: Add a guard test** + +```js +// append to test/unit/attachment-warm-utils.test.js +import { readFileSync } from 'node:fs' +it('content-publish-session wires attachment warm', () => { + const src = readFileSync('srv/lib/content-publish-session.js', 'utf8') + expect(src).toContain('extractAttachmentUrls') + expect(src).toContain('warmAttachmentsLive') +}) +``` + +- [ ] **Step 3: Run** + +Run: `npx vitest run --project unit test/unit/attachment-warm-utils.test.js` +Expected: PASS. + +- [ ] **Step 4: Commit** + +```bash +git add srv/lib/content-publish-session.js test/unit/attachment-warm-utils.test.js +git commit -m "feat(attachments): warm attachment store on publish (#1931)" +``` + +--- + +## Task 13: backfill-attachments.ts + npm script + +**Files:** +- Create: `scripts/backfill-attachments.ts` (mirror `scripts/backfill-images.ts`) +- Modify: `package.json` — add `"backfill-attachments": "tsx scripts/backfill-attachments.ts"` +- Test: `test/scripts/backfill-attachments.test.ts` (unit-test `collectAttachmentUrls` over a fixture HTML dir) + +**Interfaces:** +- Produces: `collectAttachmentUrls(publicDir: string) → Map` (sourceUrl → slug); CLI with `--limit --concurrency --dry-run --force`. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/scripts/backfill-attachments.test.ts +import { describe, it, expect } from 'vitest' +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { collectAttachmentUrls } from '../../scripts/backfill-attachments.js' + +describe('collectAttachmentUrls', () => { + it('collects attachment source URLs from built tutorial HTML', () => { + const root = mkdtempSync(join(tmpdir(), 'bf-')) + const dir = join(root, 'tutorials', 'rap100'); mkdirSync(dir, { recursive: true }) + const raw = 'https://raw.githubusercontent.com/sap-tutorials/abap-core-development/main/tutorials/rap100/EX2.txt' + writeFileSync(join(dir, 'index.html'), `d`) + const map = collectAttachmentUrls(root) + expect(map.get(raw)).toBe('rap100') + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/scripts/backfill-attachments.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write the script** + +Copy `scripts/backfill-images.ts` verbatim, then change: +- Import `extractAttachmentUrls` from `./parsers/... ` → actually from `../srv/lib/attachment-warm-utils.js`? `backfill-images.ts` imports `extractImgCdnUrls` — replicate that import path but for attachments (`extractAttachmentUrls` from `srv/lib/attachment-warm-utils.js`). Match how backfill-images imports it. +- Rename `collectImageUrls` → `collectAttachmentUrls` (uses `extractAttachmentUrls`). +- Rename `fetchImage`→`fetchAttachment`, `pushImage`→`pushAttachment`. +- Change the push endpoint from `/content/image` to `/content/attachment`. +- Keep the anon-first → Bearer-token-on-404 fetch, the concurrency pool, and the `--limit/--concurrency/--dry-run/--force` flags unchanged. +- `export function collectAttachmentUrls(publicDir: string): Map` so the test can import it. + +> Follow `scripts/backfill-images.ts` structure exactly; the only semantic differences are the extraction function, the endpoint path, and the identifiers. Do not re-derive the pool/flag logic. + +- [ ] **Step 4: Add the npm script + run test** + +Add to `package.json` `scripts`: +```json +"backfill-attachments": "tsx scripts/backfill-attachments.ts", +``` +Run: `npx vitest run --project unit test/scripts/backfill-attachments.test.ts` +Expected: PASS. +Then dry-run smoke (requires a prior `npm run build:all` producing `hugo/public`): +`CAP_BASE_URL=http://localhost:4004 CONTENT_API_KEY=x npx tsx scripts/backfill-attachments.ts --dry-run --limit 3` +Expected: lists candidate attachment URLs, pushes nothing. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/backfill-attachments.ts package.json test/scripts/backfill-attachments.test.ts +git commit -m "feat(attachments): backfill-attachments push script (#1931)" +``` + +--- + +## Task 14: Add new srv files to srv-qa cp list + +**Files:** +- Modify: `.deploy/mta.yaml` (srv-qa builder `cp` command, ~line 174) + +**Interfaces:** none (deploy packaging). + +- [ ] **Step 1: Add the four runtime files to the cp list** + +In the long `cp ... srv/lib/` command, append these to the source file list (before ` srv/lib/`): +``` +../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js +``` +> Rationale (Global Constraint): `server.js` registers the serve/ingest routes and `content-publish-session.js` (already copied) dynamically imports `attachment-warm-utils.js` + `attachment-source-handler.js`, which pull the store/ingest/mime modules. All six must be present in srv-qa or QA boot crashes. + +- [ ] **Step 2: Run the srv-qa cp-list guard** + +Run: `npx tsx scripts/check-srv-qa-cp-list.ts` +Expected: PASS (no missing transitive deps reported). + +- [ ] **Step 3: Commit** + +```bash +git add .deploy/mta.yaml +git commit -m "chore(attachments): add attachment-* srv files to srv-qa cp list (#1931)" +``` + +--- + +## Task 15: Build CDS artifacts + full test/lint sweep + +**Files:** none created; validates the schema build and full suite. + +- [ ] **Step 1: Build production CDS artifacts** + +Run: `npm run build:cds` +Expected: exits 0; emits `TutorialAssets` hdbtable + `@cap-js/attachments` composition table + a fresh `.hdbmigrationtable` under the gen output. Verify no "Duplicate definition"/resolve errors. Do NOT hand-edit any migration table. + +- [ ] **Step 2: Run the full unit suite** + +Run: `npm test` +Expected: PASS, including all new attachment tests. + +- [ ] **Step 3: Run the postbuild guards that touch srv-qa/routes** + +Run: `npx tsx scripts/check-srv-qa-cp-list.ts && npx tsx scripts/check-srv-qa-route-drift.ts && npx tsx scripts/check-public-endpoints.ts` +Expected: PASS. `check-public-endpoints.ts` must accept the new anonymous `GET /content/attachment-source` (if it enforces an allowlist of public endpoints, add `/content/attachment-source` there — same posture as `/content/image-source`). + +- [ ] **Step 4: Commit any guard/config updates** + +```bash +git add -A +git commit -m "chore(attachments): cds build artifacts + public-endpoint allowlist (#1931)" +``` + +--- + +## Task 16: End-to-end verification (local hybrid or DEV) + +**Files:** none — live verification. + +- [ ] **Step 1: Build + run against a backend with the RAP100 tutorial present** + +Fetch + build so the RAP100 page bakes the new link shape: +```bash +npm run fetch-tutorials +npm run build:all +``` +Grep the built page for the wrapped link: +```bash +grep -o '/content/attachment-source?u=[^"]*EX2_DDLX[^"]*' hugo/public/tutorials/abap-environment-rap100-enhance-data-model/index.html +``` +Expected: a `/content/attachment-source?u=...EX2_DDLX...` href (and a sibling `&dl=1`), NOT the bare `EX2_DDLX_ZRAP100_C_TRAVELTP.txt`. + +- [ ] **Step 2: Populate the store + verify serving (hybrid)** + +With a `cds bind`'d hybrid srv (real HANA + S3) or a DEV deploy: +```bash +CAP_BASE_URL= CONTENT_API_KEY= npm run backfill-attachments -- --limit 50 +curl -sI "/content/attachment-source?u=$(python3 -c "import urllib.parse;print(urllib.parse.quote('https://raw.githubusercontent.com/sap-tutorials/abap-core-development/main/tutorials/abap-environment-rap100-enhance-data-model/EX2_DDLX_ZRAP100_C_TRAVELTP.txt'))")" +``` +Expected: `200`, `Content-Type: text/plain; charset=utf-8`, `Content-Disposition: inline; filename="EX2_DDLX_ZRAP100_C_TRAVELTP.txt"`, `X-Content-Type-Options: nosniff`. Append `&dl=1` → `Content-Disposition: attachment; …`. + +- [ ] **Step 3: Open the PR** + +```bash +git push -u origin worktree-attachment-object-store +gh pr create --repo sap-tutorials/tutorials-ims --base main \ + --title "Attachment object-store pipeline: serve tutorial repo attachments like images (#1931)" \ + --body "Implements docs/superpowers/specs/2026-08-21-tutorial-attachment-object-store-design.md. Fixes dead relative attachment links (#1931 follow-up) by ingesting .txt/.zip/.pdf/.csv/.json into the object store and serving via /content/attachment-source (inline text, download binaries, .html neutered, ?dl=1 to force download)." +``` + +> Deploy note (from CLAUDE.md/memory): a full deploy must run `npm run build:all` before `mbt build`; content publish into HANA is a built-in final deploy step; the QA content rebuild must also run. Confirm deploy scope with the maintainer. The new `TutorialAssets` HDI artifacts deploy with the srv module. + +--- + +## Self-Review + +**Spec coverage:** §1 detect+bake → Tasks 1-3; §2 store → Tasks 4-5; §3 ingest → Tasks 8,10,11; §4 warm → Tasks 7,12; §5 serve → Tasks 6,9,11; §6 backfill → Task 13; §7 deploy wiring → Tasks 14-15; testing → per-task + Task 15-16; out-of-scope (real scan, subsystem B, prerequisites links) not implemented, as specified. ✅ + +**Placeholder scan:** No TBD/TODO in code steps. Two "NOTE" callouts (compose signature in Task 2, CONTENT_API_KEY setup in Task 10, github-blocked assumption in Task 9, backfill copy in Task 13) point the executor to the exact template file/test to mirror — not placeholders for logic. ✅ + +**Type consistency:** `resolveAttachmentLinks(content, {repo,branch,slug,rewrite})` (Tasks 1,2); store `put(url,{buffer,mimeType,contentHash,slug,channel,filename})` / `getStream→{stream,mimeType,filename}` (Tasks 5,8,9,10); `ingestAttachment(url,{slug,channel,deps})→{action,status?,contentHash?,mimeType?}` (Tasks 8,9); `extractAttachmentUrls(html)→string[]`, `warmAttachments(urls,{slug,ingestFn})`, `warmAttachmentsLive(urls,{slug})` (Tasks 7,9,12); `dispositionFor(mime,{download,filename})→{contentType,disposition}`, `extToMime(name)→string` (Tasks 6,8,9,10); routes `/content/attachment-source` + `/content/attachment` (Tasks 9,10,11,12,13). Consistent across tasks. ✅ diff --git a/docs/superpowers/specs/2026-08-21-tutorial-attachment-object-store-design.md b/docs/superpowers/specs/2026-08-21-tutorial-attachment-object-store-design.md new file mode 100644 index 000000000..efc057b12 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-tutorial-attachment-object-store-design.md @@ -0,0 +1,141 @@ +# Tutorial Attachment Object-Store Pipeline — Design + +**Date:** 2026-08-21 +**Status:** Approved (design), pending implementation plan +**Related:** [#1931](https://github.com/sap-tutorials/tutorials-ims/issues/1931) (dead attachment links — the trigger), [#1963](https://github.com/sap-tutorials/tutorials-ims/issues/1963) (subsystem B: source markdown scanner, parallel effort), `docs/superpowers/specs/2026-08-17-persist-tutorial-images-design.md` (the image pipeline this mirrors). + +## Problem + +Tutorial markdown links to **repo attachment files** — `[CDS metadata extension ...](EX2_DDLX_ZRAP100_C_TRAVELTP.txt)` — render as **bare relative anchors** (``). Served at `/tutorials/`, the browser resolves them against the page path → 404. The attachment exists in the source repo (`abap-core-development/tutorials//EX2_DDLX_ZRAP100_C_TRAVELTP.txt`) but is never ingested or served. + +Root cause: `resolveImageURLs()` (`scripts/parsers/images.ts`) rewrites `![alt](relative)` **images** to absolute raw-GitHub URLs, but plain **links** `[text](relative)` get no rewrite. Images additionally flow through an object-store pipeline (ingest → S3 → served via `/img-cdn` / `/content/image-source`); attachments have no equivalent. + +**Scope:** 44 relative non-http attachment links across 24 tutorials (`.txt`×18, `.json`×11, `.html`×7, `.csv`×6, `.zip`×1, `.pdf`×1) as of 2026-08-21. + +## Decision + +Attachments are **binary assets** and belong in the object store like images — not a markdown defect to fix at source. Build a parallel attachment pipeline that mirrors the image pipeline **minus resize/WebP**, serving files through a CAP endpoint the approuter already routes. + +Confirmed decisions: +1. **New `TutorialAssets` CDS entity** (own `@cap-js/attachments` composition) reusing the existing **shared Object Store (S3)** binding — clean separation from image dedup/warm/backfill. +2. **Malware scanning: mock/off, image parity** — `malwareScanner-mocked`, `status:'Clean'` hardcoded. Files are already publicly downloadable from GitHub; we mirror curated SAP source. Real scanning is a documented follow-up gap. +3. **Serve inline-by-default with an explicit download option.** Text types view inline (the #1931 "CTRL+F the source code" use case); binaries download; `.html` is neutered (served as `text/plain`, never inline-executed). Every rendered attachment link is accompanied by a download affordance (`&dl=1`). + +## Architecture + +Seven components across parse → publish → serve, mirroring the image lifecycle. No approuter code changes (attachments need no resize; the existing `^/content/(.*)$ → srv-api` route carries the serve endpoint). + +### 1. Detect + bake (parse JS + Hugo render-link hook) + +**JS resolve (relative → raw URL).** Add an attachment-link resolver alongside `resolveImageURLs` in `scripts/parsers/images.ts` (or a new `scripts/parsers/attachment-links.ts` imported by `compose.ts`). It rewrites `[text](path)` where `path` ends in an allowlisted extension: +``` +base = `${RAW_BASE_URL}/sap-tutorials/${repo}/${branch}/tutorials/${slug}` +[text](file.ext) → [text](${base}/file.ext) +``` +Same guardrails as the image resolver: skip destinations starting with `http://`, `https://`, `#`, `mailto:`, `/` (absolute), or containing `../`. Only rewrite when the destination's extension is in the allowlist. This guarantees links to other tutorials (absolute/`/`-prefixed) and in-page anchors are never touched. Idempotent: a destination already on `raw.githubusercontent.com` is left as-is. + +**Allowlist** (`ATTACHMENT_EXTENSIONS`): `.txt .csv .json .md .sql .abap .properties .yaml .yml .xml .html .zip .pdf .war .jar .zargo .har`. Seeded from the 6 observed extensions plus common code/text/archive types. `.html` is special-cased at serve time (see §5). Maintained in one shared module so the resolver, render hook, extract regex, and serve handler agree. + +**Hugo render-link hook** — new `hugo/layouts/_default/_markup/render-link.html`: +- **Faithfully passes through all non-attachment links** unchanged (`{{ .Text | safeHTML }}`), preserving Hugo's default behavior. Creating this hook overrides *all* link rendering, so the default branch must be complete and correct. +- When `.Destination` is a `raw.githubusercontent.com` URL with an allowlisted extension: emit the primary link with `href="/content/attachment-source?u="` **plus a sibling download link** `href="/content/attachment-source?u=&dl=1"` (small `↓`/"download" affordance, class-tagged for styling, `aria-label`). +- Security note in-template: destination is a Hugo-known raw-GitHub URL derived from trusted authored markdown; `u` is percent-encoded via `urlquery`. + +**Prerequisites/compose:** attachment links inside `## Prerequisites` raw-HTML tables (`prerequisites-markup.ts`) are **out of scope for v1** (body links only). Flagged for a follow-up if needed. + +### 2. Store — `srv/lib/attachment-store.cjs` + `db/tutorial-assets.cds` + +New CDS entity (mirror `db/tutorial-images.cds`): +```cds +entity TutorialAssets { + key ID : UUID; + sourceUrl : String(1024); + tutorial : Association to Tutorials on tutorial.slug = slug; + slug : String(255); + channel : String(8); // 'prod' | 'qa' + contentHash : String(64); // sha256 of stored bytes (dedup) + mimeType : String(128); + filename : String(255); // for Content-Disposition + content : Composition of many Attachments; // @cap-js/attachments +} +``` +`attachment-store.cjs` is a copy of `image-store.cjs` with the entity/namespace swapped and `filename` persisted: `head(sourceUrl)`, `put(sourceUrl, {buffer, mimeType, contentHash, slug, channel, filename})`, `getStream(sourceUrl) → {stream, mimeType, filename}`, `remove(sourceUrl)`. Keyed by `sourceUrl`; one-row-per-URL via delete-then-insert (avoids `NonUpdatableProperties:[content]` 409). Wrapped in `withCtx` for the out-of-request warm/serve paths. `status:'Clean'` hardcoded (mock parity). + +### 3. Ingest (push) — `srv/lib/attachment-ingest.cjs` + `POST /content/attachment` + +Mirror `image-ingest.cjs` + `image-ingest-handler.js`: +- Route `app.post('/content/attachment', contentAuthMiddleware, express.raw({ type: '*/*', limit: '25mb' }), attachmentIngestHandler)` — **CONTENT_API_KEY auth** (reused). Query `?u=&slug=&channel=&force=`. +- Bytes-in (srv CF egress is anon-404'd by GitHub — same reason images push). Host allowlist = `raw.githubusercontent.com`. `contentHash = sha256(body)`; `head` hash-match → `unchanged`; else `put` → `stored`. `force=1` bypasses dedup to heal orphaned rows. +- **MIME:** prefer request `Content-Type`; when absent/generic, derive from an **extension→MIME map** (`.txt→text/plain; charset=utf-8`, `.json→application/json`, `.csv→text/csv`, `.md→text/markdown`, `.sql/.abap/.properties/.yaml/.yml/.xml→text/plain`, `.html→text/html` (neutered at serve), `.zip→application/zip`, `.pdf→application/pdf`, `.war/.jar→application/java-archive`, else `application/octet-stream`). Preserve `filename` (`sourceUrl.split('/').pop()`). +- 25 MB cap (reuse image limit; attachments are small source docs). + +### 4. Warm on publish + +- `extractAttachmentUrls(html)` — regex matching `/content/attachment-source?u=` in published HTML (mirror `extractImgCdnUrls`), deduped + decoded, `channelFor(u)` reused for prod/qa. +- Fire-and-forget in `srv/lib/content-publish-session.js` beside the existing image warm (`setImmediate`, failures swallowed, never fail publish). +- **PROD** srv egress reaches GitHub → warm-live auto-populates S3 at publish. **DEV** srv egress is GitHub-flagged → warm-live fails; population relies on the backfill push (§6). Identical asymmetry to images; documented, not a bug. + +### 5. Serve — `srv/lib/attachment-source-handler.js` + `GET /content/attachment-source?u=&dl=` + +Mirror `image-source-handler.js` minus resize: +- **Anonymous**, registered `app.get('/content/attachment-source', attachmentSourceHandler)`. 400 if `u` missing. +- Fast path `store.getStream(u)`; miss → **self-heal single-flight** (`_inflight` Map) calling `attachmentIngest(u, {slug:'', channel, deps})`. In PROD this heals; in DEV it 404s until backfilled (same as images). +- **Content-Disposition logic** (the core new behavior): + - `dl=1` (or `dl=true`) → `attachment; filename=""` for **any** type. + - Else by MIME class: + - `text/plain`, `text/csv`, `text/markdown`, `application/json`, `application/xml` → `inline`. + - `application/zip`, `application/pdf`, `application/java-archive`, `application/octet-stream` → `attachment; filename=""`. + - **`text/html`** → override `Content-Type` to `text/plain; charset=utf-8`, serve `inline`, add `X-Content-Type-Options: nosniff`. Viewable + CTRL+F-able, **never executed** (no XSS from the content domain). `dl=1` still downloads it as `.html`. + - Always set `X-Content-Type-Options: nosniff`, `Cache-Control: public, max-age=3600`, `X-Content-Source: attachment-store`. +- Miss/ingest-failed → **404** `{error:'Attachment unavailable'}` (fail-open — the page still renders; only the file link 404s, same as a genuinely-missing source file). + +### 6. Backfill — `scripts/backfill-attachments.ts` + +Mirror `scripts/backfill-images.ts`: +- `collectAttachmentUrls(publicDir)` walks `hugo/public/tutorials/*/index.html`, runs `extractAttachmentUrls` → `Map`. +- `fetchAttachment(u, token)` anon-first, Bearer-token fallback **only on 404** for `raw.githubusercontent.com`. +- `pushAttachment(baseUrl, apiKey, u, slug, buffer, mimeType, force)` → `POST /content/attachment`. +- Concurrency pool (default 12), flags `--limit --concurrency --dry-run --force`. Exit 2 only if every push failed. `npm run backfill-attachments`. + +### 7. Deploy wiring + +- **srv-qa `cp` list** in `.deploy/mta.yaml`: add `attachment-store.cjs`, `attachment-ingest.cjs`, `attachment-source-handler.js`, and any shared helper (e.g. `attachment-warm-utils.js`) — the serve/warm handlers are registered in `srv/server.js`, which also boots srv-qa. Audit transitive `./` imports. +- New entity → `cds build --production` emits `TutorialAssets` hdbtable + `@cap-js/attachments` composition table + hdbmigrationtable. No `.cdsrc.json` build-task special-casing (our own `db/` entity auto-builds; the caching-plugin gotcha does not apply). +- **Object store binding** (`tutorials-objectstore`, `s3-standard`, shared) and **CONTENT_API_KEY** reused — no new resources/entitlements. +- Add `backfill-attachments` npm script. + +## Data flow + +``` +Author markdown: [doc](EX2.txt) + └─ compose/images.ts: → [doc](raw.githubusercontent.com/.../EX2.txt) + └─ Hugo render-link.html: → doc ↓ + └─ publish: extractAttachmentUrls(html) → warm (PROD auto; DEV via backfill push) + └─ POST /content/attachment (bytes) → attachment-store.put → S3 + Browser GET /content/attachment-source?u= + └─ approuter ^/content/(.*)$ → srv-api → attachmentSourceHandler + └─ store.getStream (hit) | self-heal ingest (miss) → stream + Content-Disposition +``` + +## Error handling & fail-open + +- Serve miss → 404, page unaffected. Broken source link (file absent in repo) → ingest fails → 404 (acceptable; equals a genuinely-missing file). Consistent with the image pipeline's "broken source img refs 404, fail-open serves." +- Warm/publish never throws into the publish tx. +- Ingest store error → 500 (surfaces to backfill for retry). + +## Testing + +- **Unit:** attachment-link resolver (allowlist hits/misses, guardrails, idempotency); `extractAttachmentUrls` regex; disposition map incl. `.html` neutering + `dl=1` override; ingest sha256 dedup + ext→MIME map. +- **Render hook (Hugo golden):** `.txt` link → `/content/attachment-source?u=...` + download sibling; **normal links (external, internal, anchors) render unchanged**. +- **Hybrid (real HANA + S3):** store put/head/getStream/remove round-trip; serve self-heal single-flight. +- **Smoke (post-deploy):** fetch a known attachment (`EX2_DDLX_ZRAP100_C_TRAVELTP.txt` from `abap-environment-rap100-enhance-data-model`) via `/content/attachment-source` → 200 + correct Content-Type + inline; `&dl=1` → attachment disposition. + +## Out of scope + +- Real malware scanning (documented gap; follow-up if downloadable-binary risk posture changes). +- Subsystem B source markdown scanner/fixer (#1963, parallel). +- Prerequisites-table attachment links (body links only, v1). + +## Rollback + +Delete the `render-link.html` hook (links revert to bare relative — the current dead-link state, no worse) and revert the parser/serve additions. The `TutorialAssets` entity + rows are inert if unreferenced. No data migration to unwind. diff --git a/hugo/layouts/_default/_markup/render-link.html b/hugo/layouts/_default/_markup/render-link.html new file mode 100644 index 000000000..3fa89c321 --- /dev/null +++ b/hugo/layouts/_default/_markup/render-link.html @@ -0,0 +1,18 @@ +{{- /* Render hook for links (issue #1931). + Default behavior for all links is Hugo's standard emission. Attachment + links whose destination is a raw.githubusercontent.com URL with an + allowlisted extension (rewritten by scripts/parsers/attachment-links.ts) + are routed through the CAP attachment store and get a download sibling. + KEEP the extension list in sync with attachment-links.ts / attachment-mime.cjs. */ -}} +{{- $dest := .Destination -}} +{{- $exts := slice "txt" "csv" "json" "md" "sql" "abap" "properties" "yaml" "yml" "xml" "html" "zip" "pdf" "war" "jar" "zargo" "har" -}} +{{- $ext := lower (path.Ext $dest | strings.TrimPrefix ".") -}} +{{- $isAttachment := and (hasPrefix $dest "https://raw.githubusercontent.com/") (in $exts $ext) -}} +{{- if $isAttachment -}} +{{- $enc := $dest | urlquery -}} +{{- $view := printf "/content/attachment-source?u=%s" $enc -}} +{{- $dl := printf "/content/attachment-source?u=%s&dl=1" $enc -}} +{{ .Text | safeHTML }}↓ +{{- else -}} +{{ .Text | safeHTML }} +{{- end -}} diff --git a/package.json b/package.json index b25b5376c..fae2de2e7 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "lint:tutorial-markdown": "tsx scripts/lint-tutorial-markdown.ts", "publish-content": "tsx scripts/publish-content.ts", "backfill-images": "tsx scripts/backfill-images.ts", + "backfill-attachments": "tsx scripts/backfill-attachments.ts", "preflight:ai-quiz-smoke": "tsx scripts/preflight-ai-quiz-smoke.ts", "prebuild:parsers-bundle": "esbuild scripts/parsers/index.ts --bundle --platform=node --format=esm --outfile=srv-qa/lib/parsers.bundle.mjs --external:node:* --banner:js=\"import { createRequire } from 'module'; const require = createRequire(import.meta.url);\"", "check:security-annotations": "node scripts/check-hugo-safe-html.cjs", diff --git a/scripts/backfill-attachments.ts b/scripts/backfill-attachments.ts new file mode 100644 index 000000000..ce3826974 --- /dev/null +++ b/scripts/backfill-attachments.ts @@ -0,0 +1,162 @@ +// scripts/backfill-attachments.ts +// +// Populates the tutorial-attachment store by fetching each referenced attachment +// from GitHub (this runner has clean GitHub egress) and POSTing the BYTES to the +// srv's `POST /content/attachment` endpoint. The srv never fetches GitHub itself — +// its CF egress IP is anon-404'd by GitHub's raw CDN and it has no runtime +// GitHub token. See srv/lib/attachment-ingest-handler.js. +// +// Enumeration source: the rendered tutorial HTML in hugo/public/tutorials/*/ +// index.html — the same `/content/attachment-source?u=` references the browser requests. +// +// Usage: +// CAP_BASE_URL= CONTENT_API_KEY= npm run backfill-attachments +// ... [--limit N] # only the first N unique attachments (smoke) +// ... [--concurrency N] # parallel fetch+push (default 12) +// ... [--dry-run] # enumerate + count only, no fetch/push +// +// GitHub auth (for the FETCH side only): anonymous-first, falls back to a +// Bearer token from TUTORIALS_GITHUB_TOKEN / GITHUB_TOKEN on a 404 (private +// QA -Contribution repos need it; public prod attachments ride anon). + +import { readdirSync, readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { extractAttachmentUrls, channelFor } from '../srv/lib/attachment-warm-utils.js'; + +interface Args { limit: number; concurrency: number; dryRun: boolean; force: boolean; } + +function parseArgs(argv: string[]): Args { + const get = (flag: string): string | undefined => { + const i = argv.indexOf(flag); + return i >= 0 ? argv[i + 1] : undefined; + }; + return { + limit: Number(get('--limit')) || 0, + concurrency: Number(get('--concurrency')) || 12, + dryRun: argv.includes('--dry-run'), + force: argv.includes('--force'), + }; +} + +/** Walk hugo/public/tutorials//index.html → Map. */ +export function collectAttachmentUrls(publicDir: string): Map { + const urlToSlug = new Map(); + const tutorialsDir = join(publicDir, 'tutorials'); + if (!existsSync(tutorialsDir)) { + throw new Error(`Not found: ${tutorialsDir} — run the Hugo build first (npm run build:all).`); + } + for (const slug of readdirSync(tutorialsDir)) { + const indexPath = join(tutorialsDir, slug, 'index.html'); + if (!existsSync(indexPath)) continue; + let html: string; + try { html = readFileSync(indexPath, 'utf8'); } catch { continue; } + for (const u of extractAttachmentUrls(html)) { + if (!urlToSlug.has(u)) urlToSlug.set(u, slug); // first-seen slug wins + } + } + return urlToSlug; +} + +/** True only when u's hostname is exactly raw.githubusercontent.com — prevents token leakage to lookalike hosts. */ +export function isRawGithubHost(u: string): boolean { + try { return new URL(u).hostname.toLowerCase() === 'raw.githubusercontent.com'; } catch { return false; } +} + +/** Fetch an attachment: anonymous-first, Bearer-token fallback on 404. */ +async function fetchAttachment(u: string, token: string | undefined): Promise<{ ok: boolean; status: number; buffer?: Buffer; mimeType?: string }> { + const doFetch = (authToken?: string) => { + const headers: Record = { 'User-Agent': 'tutorials-backfill' }; + if (authToken) headers['Authorization'] = `Bearer ${authToken}`; + return fetch(u, { headers, signal: AbortSignal.timeout(20000) }); + }; + let res = await doFetch(); + if (res.status === 404 && token && isRawGithubHost(u)) { + res = await doFetch(token); + } + if (!res.ok) return { ok: false, status: res.status }; + const buffer = Buffer.from(await res.arrayBuffer()); + const mimeType = res.headers.get('content-type') || 'application/octet-stream'; + return { ok: true, status: res.status, buffer, mimeType }; +} + +/** POST bytes to the srv ingest endpoint. Returns the server's action. */ +async function pushAttachment(baseUrl: string, apiKey: string, u: string, slug: string, buffer: Buffer, mimeType: string, force: boolean): Promise<{ ok: boolean; status: number; action?: string }> { + const url = `${baseUrl}/content/attachment?u=${encodeURIComponent(u)}&slug=${encodeURIComponent(slug)}${force ? '&force=1' : ''}`; + const res = await fetch(url, { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': mimeType }, + body: buffer, + signal: AbortSignal.timeout(30000), + }); + let action: string | undefined; + try { action = ((await res.json()) as { action?: string }).action; } catch { /* non-JSON */ } + return { ok: res.ok, status: res.status, action }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const baseUrl = (process.env.CAP_BASE_URL || '').replace(/\/$/, ''); + const apiKey = process.env.CONTENT_API_KEY || ''; + const token = process.env.TUTORIALS_GITHUB_TOKEN || process.env.GITHUB_TOKEN || undefined; + + if (!baseUrl) { console.error('Fatal: CAP_BASE_URL not set'); process.exit(1); } + if (!apiKey && !args.dryRun) { console.error('Fatal: CONTENT_API_KEY not set'); process.exit(1); } + + const publicDir = join(process.cwd(), 'hugo', 'public'); + console.log(`Enumerating attachment URLs under ${publicDir}/tutorials ...`); + let urlToSlug = collectAttachmentUrls(publicDir); + let entries = [...urlToSlug.entries()]; + const totalUnique = entries.length; + if (args.limit > 0) entries = entries.slice(0, args.limit); + console.log(`Found ${totalUnique} unique attachment URL(s)${args.limit ? `, backfilling first ${entries.length}` : ''}.`); + const channels = entries.reduce((acc, [u]) => { acc[channelFor(u)] = (acc[channelFor(u)] || 0) + 1; return acc; }, {} as Record); + console.log(`Channels: ${JSON.stringify(channels)}`); + + if (args.dryRun) { console.log('Dry run — no fetch/push performed.'); process.exit(0); } + + const stats = { stored: 0, unchanged: 0, fetchFailed: 0, pushFailed: 0 }; + let done = 0; + const total = entries.length; + + // Simple concurrency pool. + let cursor = 0; + async function worker() { + while (cursor < entries.length) { + const idx = cursor++; + const [u, slug] = entries[idx]; + try { + const got = await fetchAttachment(u, token); + if (!got.ok) { + stats.fetchFailed++; + } else { + const pushed = await pushAttachment(baseUrl, apiKey, u, slug, got.buffer!, got.mimeType!, args.force); + if (!pushed.ok) stats.pushFailed++; + else if (pushed.action === 'unchanged') stats.unchanged++; + else stats.stored++; + } + } catch { + stats.fetchFailed++; + } + done++; + if (done % 200 === 0 || done === total) { + console.log(` ${done}/${total} — stored=${stats.stored} unchanged=${stats.unchanged} fetch-fail=${stats.fetchFailed} push-fail=${stats.pushFailed}`); + } + } + } + await Promise.all(Array.from({ length: Math.max(1, args.concurrency) }, () => worker())); + + console.log('\nBackfill complete:'); + console.log(` stored: ${stats.stored}`); + console.log(` unchanged: ${stats.unchanged}`); + console.log(` fetch-fail: ${stats.fetchFailed} (attachment genuinely 404 on GitHub, or network)`); + console.log(` push-fail: ${stats.pushFailed} (srv ingest error)`); + // Non-zero exit only if EVERY push failed (indicates a broken endpoint/auth), + // not for individual missing-on-GitHub attachments (expected tail). + if (total > 0 && stats.stored === 0 && stats.unchanged === 0) process.exit(2); + process.exit(0); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(err => { console.error('Fatal:', err instanceof Error ? err.message : String(err)); process.exit(1); }); +} diff --git a/scripts/check-srv-qa-route-drift.ts b/scripts/check-srv-qa-route-drift.ts index f56c5a0c3..d7907329b 100644 --- a/scripts/check-srv-qa-route-drift.ts +++ b/scripts/check-srv-qa-route-drift.ts @@ -141,6 +141,20 @@ const ALLOWLIST_ONLY_ON_SRV: Record = { 'not an author-preview endpoint. The QA publish flow (publish-content.ts --channel qa) ' + 'never POSTs images, and srv-qa GET image-source self-heals via ingestImage directly ' + 'without this route. Re-evaluate if QA gains an image backfill flow.', + 'GET /content/attachment-source': + 'Attachment object-store pipeline (#1931) — streams stored tutorial attachments so the ' + + 'approuter can proxy attachment download links. DARK-LAUNCHED: no approuter route points ' + + 'at this endpoint on either channel yet (served HTML still emits relative attachment URLs), ' + + 'and the QA approuter wires no attachment-proxy route. Mirroring now would pull ' + + 'attachment-store/attachment-source-handler deps into the srv-qa cp list for an ' + + 'unreachable surface. Re-evaluate QA parity when the attachment proxy goes live and ' + + 'gains an approuter route.', + 'POST /content/attachment': + 'Attachment object-store bytes-in ingest (#1931) — CI and backfill POST raw attachment ' + + 'bytes here during publish so the srv stores originals in BTP Object Store. CI-only prod ' + + 'content maintenance, same rationale as POST /content/image: not an author-preview ' + + 'endpoint. The QA publish flow never POSTs attachments, and srv-qa has no attachment ' + + 'backfill flow. Re-evaluate if QA gains an attachment ingest path.', }; /** diff --git a/scripts/parsers/attachment-links.ts b/scripts/parsers/attachment-links.ts new file mode 100644 index 000000000..b65fad1b8 --- /dev/null +++ b/scripts/parsers/attachment-links.ts @@ -0,0 +1,46 @@ +import { RAW_BASE_URL } from './types.js' +import { createFenceTracker } from './fence-tracker.js' + +// Repo attachment file extensions served through the object store (lowercase, no dot). +// KEEP IN SYNC with the Hugo render-link hook's disposition and srv/lib/attachment-mime.cjs. +export const ATTACHMENT_EXTENSIONS = new Set([ + 'txt', 'csv', 'json', 'md', 'sql', 'abap', 'properties', + 'yaml', 'yml', 'xml', 'html', 'zip', 'pdf', 'war', 'jar', 'zargo', 'har', +]) + +export function isAttachmentPath(path: string): boolean { + const m = /\.([a-z0-9]+)$/i.exec(path.trim()) + return m ? ATTACHMENT_EXTENSIONS.has(m[1].toLowerCase()) : false +} + +export interface AttachmentResolveOpts { + repo: string + branch: string + slug: string + rewrite?: boolean +} + +// Matches a markdown link `[text](dest)` NOT preceded by `!` (which would be an image). +// Uses negative lookbehind (? { + if (fence(line)) return line // inside a code fence — leave verbatim + return line.replace(LINK_RE, (m, open, dest, tail) => { + if (/^(https?:\/\/|#|mailto:|\/)/i.test(dest)) return m + if (dest.includes('../')) return m + if (!isAttachmentPath(dest)) return m + const clean = dest.replace(/^\.?\//, '') + return `${open}${base}/${clean}${tail}` + }) + }) + .join('\n') +} diff --git a/scripts/parsers/compose.ts b/scripts/parsers/compose.ts index 17c31ac70..1c520435b 100644 --- a/scripts/parsers/compose.ts +++ b/scripts/parsers/compose.ts @@ -5,6 +5,7 @@ import { dedentListContinuationProse } from './list-continuation-prose.js' import { mergeBlockquoteNoteDividers } from './blockquote-notes.js' import { extractIntro } from './intro.js' import { resolveImageURLs } from './images.js' +import { resolveAttachmentLinks } from './attachment-links.js' import { prepPrerequisitesMarkup } from './prerequisites-markup.js' import { convertOptionBlocks } from './options.js' import { parseV1Steps } from './v1.js' @@ -109,6 +110,13 @@ export function composeTutorial(rawMd: string, opts: ComposeOpts): ComposeResult repo: opts.repo, branch: opts.branch, slug: opts.slug, rewriteImages: opts.rewriteImages, }) + // [#1931] Rewrite relative attachment links (.txt/.zip/.pdf/...) to raw-GitHub + // URLs so the render-link hook can route them through /content/attachment-source. + // Gated by rewriteImages (same "resolve relative repo paths" switch as images). + processedBody = resolveAttachmentLinks(processedBody, { + repo: opts.repo, branch: opts.branch, slug: opts.slug, + rewrite: opts.rewriteImages, + }) // [#1637] Prerequisites bypassed the body's image-URL rewriter and, because // its content is markdownified as-is, markdown images/links inside a raw diff --git a/srv/lib/attachment-ingest-handler.js b/srv/lib/attachment-ingest-handler.js new file mode 100644 index 000000000..137acaf36 --- /dev/null +++ b/srv/lib/attachment-ingest-handler.js @@ -0,0 +1,93 @@ +// srv/lib/attachment-ingest-handler.js +// +// Express handler for POST /content/attachment?u=&slug=&channel=&force= +// +// Accepts raw attachment BYTES in the request body and writes them to +// attachment-store. This is the "push" counterpart to the fetch-based +// self-heal in attachment-source-handler.js. +// +// Why bytes-in instead of the srv fetching GitHub itself: the tutorials-srv CF +// egress IP is flagged by GitHub's anonymous raw CDN (anonymous requests → +// 404), and no runtime GitHub token is provisioned on the srv. So the store is +// populated by whoever DOES have GitHub access — the publish step on a +// workstation/CI runner (scripts/backfill-images.ts) — which fetches each +// attachment and POSTs the bytes here. +// +// Registered in srv/server.js as (Task 11): +// app.post('/content/attachment', contentAuthMiddleware, +// express.raw({ type: '*/*', limit: '25mb' }), attachmentIngestHandler) +// +// Auth: CONTENT_API_KEY via contentAuthMiddleware (same as /content/publish). + +import cds from '@sap/cds' +import crypto from 'node:crypto' +import { createRequire } from 'node:module' +import { channelFor } from './attachment-warm-utils.js' + +const require = createRequire(import.meta.url) +const attachmentStore = require('./attachment-store.cjs') +const { extToMime } = require('./attachment-mime.cjs') + +const LOG = cds.log('attachment-ingest') + +// Mirror of the ingest cap so a client can't push an oversized blob past the +// 25 MB body limit express.raw already enforces (defensive, cheap). +const MAX_BYTES = Number(process.env.ATTACHMENT_MAX_BYTES) || 25 * 1024 * 1024 + +/** + * POST /content/attachment — persist client-supplied attachment bytes. + * + * 200 — { action: 'stored' | 'unchanged', contentHash } + * 400 — missing `u`, empty body, or oversized. + * 500 — store write failed. + */ +export async function attachmentIngestHandler(req, res) { + const u = req.query.u + if (!u) return res.status(400).json({ error: 'Missing u parameter' }) + + const buffer = req.body + if (!Buffer.isBuffer(buffer) || buffer.length === 0) { + return res.status(400).json({ error: 'Empty body' }) + } + if (buffer.length > MAX_BYTES) { + return res.status(400).json({ error: 'Attachment too large' }) + } + + const slug = typeof req.query.slug === 'string' ? req.query.slug : '' + const channel = typeof req.query.channel === 'string' && req.query.channel + ? req.query.channel + : channelFor(u) + + // Use the request Content-Type unless absent or generic octet-stream, + // in which case fall back to extension-based MIME. This lets the push + // client supply an accurate type while still working for unknown types. + const reqCt = req.get('content-type') || '' + const mimeType = (reqCt && reqCt !== 'application/octet-stream') + ? reqCt + : extToMime(u) + + const filename = String(u).split('/').pop() + const contentHash = crypto.createHash('sha256').update(buffer).digest('hex') + + // force=1 bypasses the hash dedup and always re-stores. Needed to heal + // "orphaned" rows — metadata + matching hash present but content missing/ + // unretrievable (e.g. from an earlier put that inserted metadata then threw + // before the object persisted). Without force those never re-store because + // the unchanged short-circuit trusts the hash match. put() removes-then- + // inserts, so a forced re-store cleans the orphan. + const force = req.query.force === '1' || req.query.force === 'true' + + try { + if (!force) { + const existing = await attachmentStore.head(u) + if (existing.exists && existing.contentHash === contentHash) { + return res.status(200).json({ action: 'unchanged', contentHash }) + } + } + await attachmentStore.put(u, { buffer, mimeType, contentHash, slug, channel, filename }) + return res.status(200).json({ action: 'stored', contentHash }) + } catch (err) { + LOG.error('[attachment-ingest] store put failed for', u, '-', err.message) + return res.status(500).json({ error: 'store write failed' }) + } +} diff --git a/srv/lib/attachment-ingest.cjs b/srv/lib/attachment-ingest.cjs new file mode 100644 index 000000000..d9b968af1 --- /dev/null +++ b/srv/lib/attachment-ingest.cjs @@ -0,0 +1,37 @@ +'use strict' +const crypto = require('node:crypto') +const { extToMime } = require('./attachment-mime.cjs') +const ATTACHMENT_HOSTS = new Set(['raw.githubusercontent.com']) +const MAX_BYTES = Number(process.env.ATTACHMENT_MAX_BYTES) || 25 * 1024 * 1024 + +async function ingestAttachment(sourceUrl, { slug, channel, deps }) { + const { fetchImageResponse, safeFetch, resolveSecret, store, + hash = (buf) => crypto.createHash('sha256').update(buf).digest('hex') } = deps + let host + try { host = new URL(sourceUrl).hostname } catch { return { action: 'failed', status: 400 } } + + const res = await fetchImageResponse(sourceUrl, { + safeFetch, resolveSecret, host, allowedHosts: ATTACHMENT_HOSTS, timeoutMs: 12000, maxRetries: 2, + }) + if (!res.ok) return { action: 'failed', status: res.status } + + const contentLength = Number(res.headers.get('content-length')) + if (!Number.isNaN(contentLength) && contentLength > MAX_BYTES) return { action: 'failed', status: 413 } + + const buffer = Buffer.from(await res.arrayBuffer()) + if (buffer.length > MAX_BYTES) return { action: 'failed', status: 413 } + + const contentHash = hash(buffer) + const existing = await store.head(sourceUrl) + if (existing.exists && existing.contentHash === contentHash) return { action: 'unchanged', contentHash } + + // GitHub serves most text attachments as text/plain; trust a specific content-type, + // otherwise derive from the extension so .json/.csv/.pdf get correct types. + const ct = res.headers.get('content-type') || '' + const mimeType = (ct && ct !== 'application/octet-stream') ? ct : extToMime(sourceUrl) + const filename = sourceUrl.split('/').pop() + await store.put(sourceUrl, { buffer, mimeType, contentHash, slug, channel, filename }) + return { action: 'stored', contentHash, mimeType } +} + +module.exports = { ingestAttachment } diff --git a/srv/lib/attachment-mime.cjs b/srv/lib/attachment-mime.cjs new file mode 100644 index 000000000..2b663c6ef --- /dev/null +++ b/srv/lib/attachment-mime.cjs @@ -0,0 +1,36 @@ +'use strict' +// Extension→MIME + Content-Disposition policy for tutorial attachments (#1931). +// KEEP the extension set in sync with scripts/parsers/attachment-links.ts and the render-link hook. + +const EXT_MIME = { + txt: 'text/plain; charset=utf-8', csv: 'text/csv', json: 'application/json', + md: 'text/markdown; charset=utf-8', sql: 'text/plain; charset=utf-8', + abap: 'text/plain; charset=utf-8', properties: 'text/plain; charset=utf-8', + yaml: 'text/plain; charset=utf-8', yml: 'text/plain; charset=utf-8', + xml: 'text/plain; charset=utf-8', html: 'text/html', + zip: 'application/zip', pdf: 'application/pdf', + war: 'application/java-archive', jar: 'application/java-archive', + zargo: 'application/octet-stream', har: 'application/json', +} + +function extToMime(filenameOrUrl) { + const m = /\.([a-z0-9]+)(?:[?#].*)?$/i.exec(String(filenameOrUrl)) + const ext = m ? m[1].toLowerCase() : '' + return EXT_MIME[ext] || 'application/octet-stream' +} + +// Inline-viewable MIME classes (rest download). +const INLINE = new Set(['text/plain', 'text/csv', 'text/markdown', 'application/json', 'application/xml']) + +function baseType(mime) { return String(mime).split(';')[0].trim().toLowerCase() } + +function dispositionFor(mimeType, { download = false, filename = 'file' } = {}) { + const safeName = String(filename).replace(/["\r\n]/g, '') + // text/html is neutered UNCONDITIONALLY: serve as text/plain, never executed. + const contentType = baseType(mimeType) === 'text/html' ? 'text/plain; charset=utf-8' : mimeType + if (download) return { contentType, disposition: `attachment; filename="${safeName}"` } + const inline = INLINE.has(baseType(contentType)) + return { contentType, disposition: `${inline ? 'inline' : 'attachment'}; filename="${safeName}"` } +} + +module.exports = { extToMime, dispositionFor } diff --git a/srv/lib/attachment-source-handler.js b/srv/lib/attachment-source-handler.js new file mode 100644 index 000000000..66e9ea7cc --- /dev/null +++ b/srv/lib/attachment-source-handler.js @@ -0,0 +1,125 @@ +// srv/lib/attachment-source-handler.js +// +// Express handler for GET /content/attachment-source?u=&dl=<0|1>. +// +// Streams the stored attachment from attachment-store on a cache hit. +// On a miss, self-heals by calling ingestAttachment once (single-flight so +// concurrent misses for the same URL coalesce). Returns 404 if ingest +// also fails. +// +// Registered in srv/server.js as: +// app.get('/content/attachment-source', attachmentSourceHandler) +// +// Anonymous — no auth required; public content like /content/tutorials/:slug. +// Sets Content-Disposition based on MIME type and optional ?dl=1 (force download). +// Analog of image-source-handler.js, plus disposition logic. + +import cds from '@sap/cds' +import { createRequire } from 'node:module' +import { safeFetch } from './safe-fetch.js' +import { resolveSecret } from './secret-resolver.js' +import { channelFor, warmAttachments } from './attachment-warm-utils.js' + +const require = createRequire(import.meta.url) +const attachmentStore = require('./attachment-store.cjs') +const { ingestAttachment } = require('./attachment-ingest.cjs') +const { dispositionFor } = require('./attachment-mime.cjs') +const { fetchImageResponse } = require('./img-cdn-fetch.cjs') + +const LOG = cds.log('attachment-source') + +/** In-flight map: URL → Promise for single-flight dedup. */ +const _inflight = new Map() + +/** + * Warm the attachment store for `urls` referenced by tutorial `slug`. + * Assembles the real ingestAttachment deps at module scope (attachmentStore, + * fetchImageResponse, safeFetch, resolveSecret) and delegates to the + * pure warmAttachments orchestrator from attachment-warm-utils.js. + * + * Non-fatal per URL: warmAttachments catches any throw or `failed` result and + * logs it; this function always resolves. + * + * @param {string[]} urls + * @param {{ slug: string }} opts + * @returns {Promise} + */ +export function warmAttachmentsLive(urls, { slug }) { + const ingestFn = (url, { slug: s, channel }) => + ingestAttachment(url, { + slug: s, + channel, + deps: { fetchImageResponse, safeFetch, resolveSecret, store: attachmentStore }, + }) + return warmAttachments(urls, { slug, ingestFn }) +} + +/** + * Express handler for GET /content/attachment-source?u=&dl=<0|1>. + * + * 200 — streams attachment bytes with Content-Type and Content-Disposition. + * 400 — missing `u` query parameter. + * 404 — attachment not in store and ingest self-heal also failed. + * + * Query params: + * u — URL-encoded source URL (required) + * dl — "1" or "true" forces attachment (download) disposition regardless of MIME type + */ +export async function attachmentSourceHandler(req, res) { + const u = req.query.u + if (!u) return res.status(400).json({ error: 'Missing u parameter' }) + + const download = req.query.dl === '1' || req.query.dl === 'true' + + // Fast path: attachment already in store. + let got = await attachmentStore.getStream(u) + + if (!got) { + // Slow path: self-heal by fetching and storing the original. + // Single-flight: coalesce concurrent misses for the same URL. + let p = _inflight.get(u) + if (!p) { + const channel = channelFor(u) + p = ingestAttachment(u, { + slug: '', + channel, + deps: { fetchImageResponse, safeFetch, resolveSecret, store: attachmentStore }, + }).finally(() => _inflight.delete(u)) + _inflight.set(u, p) + } + + let result = { action: 'failed' } + try { + result = await p + } catch (err) { + // ingestAttachment propagates throws from fetchImageResponse (e.g. SSRF_BLOCKED, + // network errors). Treat any throw as a fetch failure → 404. + LOG.warn('[attachment-source] self-heal ingest threw:', err.message) + } + + if (result.action === 'failed') { + return res.status(404).json({ error: 'Attachment unavailable' }) + } + + got = await attachmentStore.getStream(u) + if (!got) return res.status(404).json({ error: 'Attachment unavailable' }) + } + + const filename = got.filename || String(u).split('/').pop() || 'file' + const { contentType, disposition } = dispositionFor( + got.mimeType || 'application/octet-stream', + { download, filename } + ) + + res.setHeader('Content-Type', contentType) + res.setHeader('Content-Disposition', disposition) + res.setHeader('X-Content-Type-Options', 'nosniff') + res.setHeader('Cache-Control', 'public, max-age=3600') + res.setHeader('X-Content-Source', 'attachment-store') + + got.stream.on('error', (err) => { + LOG.warn('[attachment-source] stream error:', err.message) + if (!res.headersSent) res.status(500).json({ error: 'Stream error' }) + }) + got.stream.pipe(res) +} diff --git a/srv/lib/attachment-store.cjs b/srv/lib/attachment-store.cjs new file mode 100644 index 000000000..89e574b1e --- /dev/null +++ b/srv/lib/attachment-store.cjs @@ -0,0 +1,80 @@ +'use strict' +const cds = require('@sap/cds') +const { Readable } = require('node:stream') + +// Metadata on TutorialAssets; original bytes in its Attachments composition. +// Mirror of image-store.cjs — see that file for the withCtx/tenant rationale. +// +// API: +// head(sourceUrl) → Promise<{ exists, ID?, contentHash?, mimeType?, filename? }> +// put(sourceUrl, { buffer, mimeType, contentHash, slug, channel, filename }) → Promise +// getStream(sourceUrl) → Promise<{ stream: Readable, mimeType: string, filename: string } | null> +// remove(sourceUrl) → Promise + +function linkedContent() { + return cds.linked(cds.model).definitions['com.sap.developers.ims.TutorialAssets.content'] +} + +// The @cap-js/attachments S3 (and standard) provider reads `cds.context.tenant` +// unconditionally on every put/get. The warm path and serve routes both run +// OUTSIDE a CDS request context, so `cds.context` is undefined. Wrapping each +// store op in cds.tx() establishes a (tenant-less) context. +// Harmless on the SQLite basic provider used by unit tests, which never reads it. +function withCtx(fn) { + return cds.context ? fn() : cds.tx(fn) +} + +async function head(sourceUrl) { + return withCtx(async () => { + const { TutorialAssets } = cds.entities('com.sap.developers.ims') + const row = await SELECT.one.from(TutorialAssets) + .columns('ID', 'contentHash', 'mimeType', 'filename').where({ sourceUrl }) + return row + ? { exists: true, ID: row.ID, contentHash: row.contentHash, mimeType: row.mimeType, filename: row.filename } + : { exists: false } + }) +} + +async function put(sourceUrl, { buffer, mimeType, contentHash, slug, channel, filename }) { + return withCtx(async () => { + const { TutorialAssets } = cds.entities('com.sap.developers.ims') + // delete-then-insert avoids NonUpdatableProperties:[content] 409 on overwrite + await remove(sourceUrl) + const parentID = cds.utils.uuid() + const name = filename || sourceUrl.split('/').pop() + await INSERT.into(TutorialAssets).entries({ ID: parentID, sourceUrl, slug, channel, contentHash, mimeType, filename: name }) + const AttachmentsSrv = await cds.connect.to('attachments') + await AttachmentsSrv.put(linkedContent(), { + ID: cds.utils.uuid(), + up__ID: parentID, + url: cds.utils.uuid(), + content: Readable.from(buffer), // Readable works on both SQLite and Object Store (S3) + mimeType, + filename: name, + status: 'Clean', + }) + }) +} + +async function getStream(sourceUrl) { + return withCtx(async () => { + const meta = await head(sourceUrl) + if (!meta.exists) return null + const Content = linkedContent() + const att = await SELECT.one.from(Content).columns('ID').where({ up__ID: meta.ID }) + if (!att) return null + const AttachmentsSrv = await cds.connect.to('attachments') + const stream = await AttachmentsSrv.get(Content, { ID: att.ID }) // → Node Readable + return stream ? { stream, mimeType: meta.mimeType, filename: meta.filename } : null + }) +} + +async function remove(sourceUrl) { + return withCtx(async () => { + const { TutorialAssets } = cds.entities('com.sap.developers.ims') + // Composition cascades content delete + await DELETE.from(TutorialAssets).where({ sourceUrl }) + }) +} + +module.exports = { head, put, getStream, remove } diff --git a/srv/lib/attachment-warm-utils.js b/srv/lib/attachment-warm-utils.js new file mode 100644 index 000000000..20531f92b --- /dev/null +++ b/srv/lib/attachment-warm-utils.js @@ -0,0 +1,63 @@ +// srv/lib/attachment-warm-utils.js +// +// Pure attachment-warm orchestration utilities — no CDS or network deps. +// Importable from unit tests without a running CAP server. +// Analog of image-warm-utils.js +// +// Used by: +// srv/lib/attachment-source-handler.js — imports channelFor; assembles real ingestFn +// srv/lib/content-publish-session.js — fires warmAttachmentsLive after each append batch + +export { channelFor } from './image-warm-utils.js' +import { channelFor } from './image-warm-utils.js' + +/** + * Extract and decode all unique raw-source URLs from an HTML string + * containing `/content/attachment-source?u=&…` references. + * + * The regex matches `/content/attachment-source` followed by a query string + * containing `u=`. Both `?u=` (first param) and `&u=` (later param) + * are matched. The captured value is URL-decoded and deduplicated. + * + * @param {string} html — rendered tutorial HTML + * @returns {string[]} deduplicated decoded source URLs + */ +export function extractAttachmentUrls(html) { + const results = new Set() + // Match /content/attachment-source (with optional path prefix) followed by a query string + // that contains u=. Captures the encoded value up to the + // next & separator, quote, whitespace, or > character. + const re = /\/content\/attachment-source[^"'\s>]*[?&]u=([^&"'\s>]+)/g + let m + while ((m = re.exec(html)) !== null) { + try { results.add(decodeURIComponent(m[1])) } catch { /* skip malformed */ } + } + return [...results] +} + +/** + * Warm the attachment store for a set of source URLs. + * + * Non-fatal: a per-URL try/catch ensures that a failing ingest (network + * error, 429, etc.) never propagates to the caller. Failures are logged + * via console.warn; the publish path is never interrupted. + * + * @param {string[]} urls — raw source URLs to warm + * @param {{ slug: string, ingestFn: (url: string, opts: {slug: string, channel: string}) => Promise<{action: string, status?: number}> }} opts + * ingestFn is injected by the caller — production code passes the real + * ingestAttachment wrapper; unit tests pass a vi.fn() mock. + * @returns {Promise} — always resolves; never throws + */ +export async function warmAttachments(urls, { slug, ingestFn }) { + for (const url of urls) { + const channel = channelFor(url) + try { + const result = await ingestFn(url, { slug, channel }) + if (result?.action === 'failed') { + console.warn(`[attachment-warm] slug=${slug} u=${url}: ingest returned failed (status=${result.status})`) + } + } catch (err) { + console.warn(`[attachment-warm] slug=${slug} u=${url}: ingest threw: ${err?.message}`) + } + } +} diff --git a/srv/lib/content-publish-session.js b/srv/lib/content-publish-session.js index 1223ffd53..98d9d3794 100644 --- a/srv/lib/content-publish-session.js +++ b/srv/lib/content-publish-session.js @@ -254,6 +254,12 @@ export function createSessionHelpers({ namespace }) { const urls = extractImgCdnUrls(html); if (urls.length > 0) await warmImagesLive(urls, { slug }); } + const { extractAttachmentUrls } = await import('./attachment-warm-utils.js'); + const { warmAttachmentsLive } = await import('./attachment-source-handler.js'); + for (const [slug, html] of slugHtmlMap) { + const aUrls = extractAttachmentUrls(html); + if (aUrls.length > 0) await warmAttachmentsLive(aUrls, { slug }); + } } catch (err) { LOG.warn(`[image-warm] post-publish warm failed (non-fatal): ${err.message}`); } diff --git a/srv/server.js b/srv/server.js index 06b76b612..9cb3c2f33 100644 --- a/srv/server.js +++ b/srv/server.js @@ -30,6 +30,8 @@ import { basicAuthMiddleware } from './lib/tech-user-auth.js'; import { contentAuthMiddleware, publishHandler, serveHandler, pageServeHandler, authorServeHandler, advocateServeHandler, hashesHandler, sourceHashesHandler, navHandler, rollbackHandler, orphanPurgeHandler, invalidateRenderCache, beginHandler, appendHandler, commitHandler, abortHandler, pipelineLogFailureHandler } from './lib/content-store.js'; import { imageSourceHandler } from './lib/image-source-handler.js'; import { imageIngestHandler } from './lib/image-ingest-handler.js'; +import { attachmentSourceHandler } from './lib/attachment-source-handler.js'; +import { attachmentIngestHandler } from './lib/attachment-ingest-handler.js'; import { bumpCacheGeneration } from './lib/content-cache-coherence.js'; import { conceptsIndexHandler } from './lib/concept-list-page.js'; import { puzzlePageHandler, puzzleIndexHandler } from './lib/puzzle-page.js'; @@ -525,6 +527,8 @@ cds.on('bootstrap', (app) => { // POSTs raw image bytes here so the srv never has to fetch GitHub itself // (its egress IP is anon-404'd by GitHub's raw CDN). Auth = CONTENT_API_KEY. app.post('/content/image', contentAuthMiddleware, express.raw({ type: '*/*', limit: '25mb' }), imageIngestHandler); + app.get('/content/attachment-source', attachmentSourceHandler); + app.post('/content/attachment', contentAuthMiddleware, express.raw({ type: '*/*', limit: '25mb' }), attachmentIngestHandler); app.post('/content/publish', express.json({ limit: '100mb' }), contentAuthMiddleware, publishHandler); app.post('/content/publish/begin', express.json({ limit: '1mb' }), contentAuthMiddleware, beginHandler); app.post('/content/publish/append', express.json({ limit: '100mb' }), contentAuthMiddleware, appendHandler); diff --git a/test/parsers/attachment-links.test.ts b/test/parsers/attachment-links.test.ts new file mode 100644 index 000000000..47d802609 --- /dev/null +++ b/test/parsers/attachment-links.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest' +import { resolveAttachmentLinks, isAttachmentPath } from '../../scripts/parsers/attachment-links.js' + +const opts = { repo: 'abap-core-development', branch: 'main', slug: 'rap100' } +const base = 'https://raw.githubusercontent.com/sap-tutorials/abap-core-development/main/tutorials/rap100' + +describe('resolveAttachmentLinks', () => { + it('rewrites a relative allowlisted link to a raw-GitHub URL', () => { + const out = resolveAttachmentLinks('[doc](EX2_DDLX.txt)', opts) + expect(out).toBe(`[doc](${base}/EX2_DDLX.txt)`) + }) + it('rewrites ./-prefixed links and strips the ./', () => { + expect(resolveAttachmentLinks('[d](./a.csv)', opts)).toBe(`[d](${base}/a.csv)`) + }) + it('leaves images (![]) untouched, including allowlisted extensions', () => { + expect(resolveAttachmentLinks('![alt](guide.txt)', opts)).toBe('![alt](guide.txt)') + }) + it('leaves absolute, anchor, mailto, root-relative, and ../ links untouched', () => { + for (const s of ['[a](https://x.com/f.txt)', '[a](#sec)', '[a](mailto:x@y.z)', '[a](/other/f.txt)', '[a](../sib/f.txt)']) { + expect(resolveAttachmentLinks(s, opts)).toBe(s) + } + }) + it('leaves non-allowlisted extensions untouched', () => { + expect(resolveAttachmentLinks('[a](page.aspx)', opts)).toBe('[a](page.aspx)') + }) + it('does not touch link-like text inside a fenced code block', () => { + const src = '```md\n[x](y.txt)\n```' + expect(resolveAttachmentLinks(src, opts)).toBe(src) + }) + it('is idempotent (already-raw URLs are left as-is)', () => { + const once = resolveAttachmentLinks('[d](EX2.txt)', opts) + expect(resolveAttachmentLinks(once, opts)).toBe(once) + }) + it('respects rewrite:false', () => { + expect(resolveAttachmentLinks('[d](a.txt)', { ...opts, rewrite: false })).toBe('[d](a.txt)') + }) + it('isAttachmentPath matches allowlist case-insensitively', () => { + expect(isAttachmentPath('X.TXT')).toBe(true) + expect(isAttachmentPath('x.png')).toBe(false) + }) + it('rewrites both links in consecutive attachment links on one line', () => { + const input = '[a](x.txt)[b](y.txt)' + const expected = `[a](${base}/x.txt)[b](${base}/y.txt)` + expect(resolveAttachmentLinks(input, opts)).toBe(expected) + }) +}) diff --git a/test/parsers/compose.test.ts b/test/parsers/compose.test.ts index 17f9a39bb..cd4b7965d 100644 --- a/test/parsers/compose.test.ts +++ b/test/parsers/compose.test.ts @@ -53,4 +53,42 @@ parser: v2 expect(result.steps[0].content).toContain('./images/foo.png'); expect(result.steps[0].content).not.toContain('raw.githubusercontent.com'); }); + + // [#1931] Attachment-link resolver wired into compose + it('rewrites relative attachment links to raw-GitHub URLs when rewriteImages: true', () => { + const raw = `--- +title: T +description: x +parser: v2 +--- + +### Step One +See [doc](EX2.txt) below. +`; + const result = composeTutorial(raw, { + repo: 'abap-core-development', branch: 'main', slug: 'rap100', + target: 'hugo', rewriteImages: true, + }); + expect(result.body).toContain( + 'https://raw.githubusercontent.com/sap-tutorials/abap-core-development/main/tutorials/rap100/EX2.txt' + ); + }); + + it('leaves attachment links untouched when rewriteImages: false', () => { + const raw = `--- +title: T +description: x +parser: v2 +--- + +### Step One +See [doc](EX2.txt) below. +`; + const result = composeTutorial(raw, { + repo: 'abap-core-development', branch: 'main', slug: 'rap100', + target: 'hugo', rewriteImages: false, + }); + expect(result.body).toContain('[doc](EX2.txt)'); + expect(result.body).not.toContain('raw.githubusercontent.com'); + }); }); diff --git a/test/parsers/render-link-hook.test.ts b/test/parsers/render-link-hook.test.ts new file mode 100644 index 000000000..cf4670171 --- /dev/null +++ b/test/parsers/render-link-hook.test.ts @@ -0,0 +1,20 @@ +// test/parsers/render-link-hook.test.ts +import { describe, it, expect } from 'vitest' +import { readFileSync, existsSync } from 'node:fs' + +const p = 'hugo/layouts/_default/_markup/render-link.html' + +describe('render-link hook', () => { + it('exists', () => { expect(existsSync(p)).toBe(true) }) + it('wraps raw.githubusercontent destinations to the attachment endpoint', () => { + const t = readFileSync(p, 'utf8') + expect(t).toContain('raw.githubusercontent.com') + expect(t).toContain('/content/attachment-source?u=') + expect(t).toContain('dl=1') // download sibling + expect(t).toContain('urlquery') // encodes the source URL + }) + it('has a passthrough branch for non-attachment links', () => { + const t = readFileSync(p, 'utf8') + expect(t).toContain('.Destination | safeURL') // default anchor emission + }) +}) diff --git a/test/scripts/backfill-attachments.test.ts b/test/scripts/backfill-attachments.test.ts new file mode 100644 index 000000000..078286faa --- /dev/null +++ b/test/scripts/backfill-attachments.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest' +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { collectAttachmentUrls, isRawGithubHost } from '../../scripts/backfill-attachments.js' + +describe('collectAttachmentUrls', () => { + it('collects attachment source URLs from built tutorial HTML', () => { + const root = mkdtempSync(join(tmpdir(), 'bf-')) + const dir = join(root, 'tutorials', 'rap100'); mkdirSync(dir, { recursive: true }) + const raw = 'https://raw.githubusercontent.com/sap-tutorials/abap-core-development/main/tutorials/rap100/EX2.txt' + writeFileSync(join(dir, 'index.html'), `d`) + const map = collectAttachmentUrls(root) + expect(map.get(raw)).toBe('rap100') + }) +}) + +describe('isRawGithubHost', () => { + it('returns true for a legitimate raw.githubusercontent.com URL', () => { + expect(isRawGithubHost('https://raw.githubusercontent.com/o/r/main/f.txt')).toBe(true) + }) + it('returns false for a lookalike subdomain attack', () => { + expect(isRawGithubHost('https://raw.githubusercontent.com.evil.com/f.txt')).toBe(false) + }) + it('returns false when hostname contains raw.githubusercontent.com as a query param', () => { + expect(isRawGithubHost('https://evil.com/?q=raw.githubusercontent.com')).toBe(false) + }) + it('returns false for a malformed URL', () => { + expect(isRawGithubHost('not-a-url')).toBe(false) + }) +}) diff --git a/test/unit/attachment-ingest-endpoint.test.js b/test/unit/attachment-ingest-endpoint.test.js new file mode 100644 index 000000000..2b54995b8 --- /dev/null +++ b/test/unit/attachment-ingest-endpoint.test.js @@ -0,0 +1,118 @@ +// test/unit/attachment-ingest-endpoint.test.js +// +// Integration test for POST /content/attachment?u= (bytes-in ingest). +// Uses in-memory SQLite; no HANA binding required. +// +// Run: npx vitest run --project unit test/unit/attachment-ingest-endpoint.test.js +// +// Boots a full CAP server via cds.test('serve', ..., '--in-memory'). The +// endpoint accepts raw attachment bytes (Content-Type = mime) and writes them +// to attachment-store, authed with CONTENT_API_KEY (same as /content/publish). +// +// NOTE: The route POST /content/attachment is registered in Task 11 (server.js). +// These tests are RED BY DESIGN until Task 11 wires the route. + +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +import { createRequire } from 'node:module' +import { _resetForTests as resetSecretResolver } from '../../srv/lib/secret-resolver.js' + +const require = createRequire(import.meta.url) +const store = require('../../srv/lib/attachment-store.cjs') + +const project = cds.test('serve', '--project', '.', '--in-memory') + +const base = '/content/attachment' + +describe('POST /content/attachment (bytes-in ingest)', () => { + const key = 'test-key' + const authHeaders = { authorization: `Bearer ${key}`, 'content-type': 'text/plain' } + // cds.test's axios client re-encodes a Buffer body to {type:'Buffer',data:[...]} + // JSON shape; identity transformRequest sends raw bytes verbatim. + const rawCfg = { headers: authHeaders, transformRequest: [(d) => d] } + + beforeAll(() => { + // contentAuthMiddleware reads CONTENT_API_KEY via secret-resolver (cached + // in a globalThis singleton). Reset so this worker's value takes effect — + // matches image-ingest-endpoint.test.js precedent. + process.env.CONTENT_API_KEY = key + resetSecretResolver() + }) + + it('stores posted bytes and round-trips via store.getStream', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/push.txt' + const bytes = Buffer.from('pushed') + // Native fetch (not axios) so the server's express.raw receives raw bytes. + const post = await fetch(`${project.url}${base}?u=${encodeURIComponent(url)}&slug=s`, { + method: 'POST', headers: authHeaders, body: bytes, + }) + expect(post.status).toBe(200) + const body = await post.json() + expect(body.action).toBe('stored') + + const got = await store.getStream(url) + const chunks = [] + for await (const c of got.stream) chunks.push(c) + expect(Buffer.concat(chunks).toString()).toBe('pushed') + }) + + it('returns unchanged on identical re-post (hash dedup)', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/ingest-idem.txt' + const bytes = Buffer.from('hello dedup') + const first = await project.post(`${base}?u=${encodeURIComponent(url)}`, bytes, rawCfg) + expect(first.data.action).toBe('stored') + const second = await project.post(`${base}?u=${encodeURIComponent(url)}`, bytes, rawCfg) + expect(second.status).toBe(200) + expect(second.data.action).toBe('unchanged') + }) + + it('force=1 re-stores identical bytes (bypasses dedup, heals orphans)', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/ingest-force.txt' + const bytes = Buffer.from('force me') + const first = await fetch(`${project.url}${base}?u=${encodeURIComponent(url)}`, { + method: 'POST', headers: authHeaders, body: bytes, + }) + expect((await first.json()).action).toBe('stored') + const forced = await fetch(`${project.url}${base}?u=${encodeURIComponent(url)}&force=1`, { + method: 'POST', headers: authHeaders, body: bytes, + }) + expect(forced.status).toBe(200) + expect((await forced.json()).action).toBe('stored') + }) + + it('uses extToMime fallback when content-type is application/octet-stream', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/data.csv' + const bytes = Buffer.from('a,b\n1,2') + const post = await fetch(`${project.url}${base}?u=${encodeURIComponent(url)}&slug=s`, { + method: 'POST', + headers: { authorization: `Bearer ${key}`, 'content-type': 'application/octet-stream' }, + body: bytes, + }) + expect(post.status).toBe(200) + expect((await post.json()).action).toBe('stored') + const got = await store.getStream(url) + expect(got.mimeType).toBe('text/csv') + }) + + it('rejects an unauthenticated post', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/noauth.txt' + await expect( + project.post(`${base}?u=${encodeURIComponent(url)}`, Buffer.from('x'), + { headers: { 'content-type': 'text/plain' } }) + ).rejects.toMatchObject({ response: { status: 401 } }) + }) + + it('400s when the u parameter is missing', async () => { + await expect( + project.post(base, Buffer.from('x'), rawCfg) + ).rejects.toMatchObject({ response: { status: 400 } }) + }) + + it('400s on an empty body', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/ingest-empty.txt' + const res = await fetch(`${project.url}${base}?u=${encodeURIComponent(url)}`, { + method: 'POST', headers: authHeaders, body: Buffer.alloc(0), + }) + expect(res.status).toBe(400) + }) +}) diff --git a/test/unit/attachment-ingest.test.js b/test/unit/attachment-ingest.test.js new file mode 100644 index 000000000..402aaeb84 --- /dev/null +++ b/test/unit/attachment-ingest.test.js @@ -0,0 +1,41 @@ +import { describe, it, expect, vi } from 'vitest' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +const { ingestAttachment } = require('../../srv/lib/attachment-ingest.cjs') + +function res(body, { ok = true, status = 200, ct = 'text/plain' } = {}) { + return { ok, status, headers: new Map([['content-type', ct], ['content-length', String(body.length)]]), + arrayBuffer: async () => Buffer.from(body) } +} + +describe('ingestAttachment', () => { + const url = 'https://raw.githubusercontent.com/o/r/main/EX2.txt' + it('stores on a fresh URL', async () => { + const store = { head: vi.fn().mockResolvedValue({ exists: false }), put: vi.fn().mockResolvedValue() } + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('hello')), safeFetch: {}, resolveSecret: {}, store } + const out = await ingestAttachment(url, { slug: 's', channel: 'prod', deps }) + expect(out.action).toBe('stored') + expect(store.put).toHaveBeenCalledOnce() + }) + it('is unchanged when hash matches', async () => { + const buf = Buffer.from('hello') + const crypto = require('node:crypto') + const h = crypto.createHash('sha256').update(buf).digest('hex') + const store = { head: vi.fn().mockResolvedValue({ exists: true, contentHash: h }), put: vi.fn() } + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('hello')), store } + const out = await ingestAttachment(url, { slug: 's', channel: 'prod', deps }) + expect(out.action).toBe('unchanged') + expect(store.put).not.toHaveBeenCalled() + }) + it('fails on a non-ok fetch', async () => { + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('', { ok: false, status: 404 })), store: {} } + const out = await ingestAttachment(url, { slug: 's', channel: 'prod', deps }) + expect(out).toEqual({ action: 'failed', status: 404 }) + }) + it('uses extToMime when the response content-type is generic', async () => { + const store = { head: vi.fn().mockResolvedValue({ exists: false }), put: vi.fn().mockResolvedValue() } + const deps = { fetchImageResponse: vi.fn().mockResolvedValue(res('{}', { ct: 'application/octet-stream' })), store } + await ingestAttachment('https://raw.githubusercontent.com/o/r/main/a.json', { slug: 's', channel: 'prod', deps }) + expect(store.put.mock.calls[0][1].mimeType).toBe('application/json') + }) +}) diff --git a/test/unit/attachment-mime.test.js b/test/unit/attachment-mime.test.js new file mode 100644 index 000000000..00bb93185 --- /dev/null +++ b/test/unit/attachment-mime.test.js @@ -0,0 +1,47 @@ +import { describe, it, expect } from 'vitest' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +const { extToMime, dispositionFor } = require('../../srv/lib/attachment-mime.cjs') + +describe('extToMime', () => { + it('maps known extensions', () => { + expect(extToMime('a.txt')).toMatch(/text\/plain/) + expect(extToMime('a.json')).toBe('application/json') + expect(extToMime('a.csv')).toBe('text/csv') + expect(extToMime('a.zip')).toBe('application/zip') + expect(extToMime('a.pdf')).toBe('application/pdf') + }) + it('falls back to octet-stream for unknown', () => { + expect(extToMime('a.bin')).toBe('application/octet-stream') + }) +}) + +describe('dispositionFor', () => { + it('text types serve inline', () => { + expect(dispositionFor('text/plain; charset=utf-8', { filename: 'a.txt' }).disposition).toMatch(/^inline/) + }) + it('binaries force attachment with filename', () => { + const d = dispositionFor('application/zip', { filename: 'a.zip' }) + expect(d.disposition).toBe('attachment; filename="a.zip"') + }) + it('text/html is neutered to text/plain inline', () => { + const d = dispositionFor('text/html', { filename: 'a.html' }) + expect(d.contentType).toMatch(/text\/plain/) + expect(d.disposition).toMatch(/^inline/) + }) + it('download:true forces attachment for any type', () => { + const d = dispositionFor('text/plain; charset=utf-8', { download: true, filename: 'a.txt' }) + expect(d.disposition).toBe('attachment; filename="a.txt"') + }) + it('text/html stays neutered even with download:true', () => { + const d = dispositionFor('text/html', { download: true, filename: 'a.html' }) + expect(d.contentType).toBe('text/plain; charset=utf-8') + expect(d.disposition).toBe('attachment; filename="a.html"') + }) + it('filename with CR/LF is sanitized from disposition header', () => { + const d = dispositionFor('text/plain', { filename: 'a.txt\r\nX-Injection: bad' }) + expect(d.disposition).not.toContain('\r') + expect(d.disposition).not.toContain('\n') + expect(d.disposition).toBe('inline; filename="a.txtX-Injection: bad"') + }) +}) diff --git a/test/unit/attachment-source-endpoint.test.js b/test/unit/attachment-source-endpoint.test.js new file mode 100644 index 000000000..28ba3a1fb --- /dev/null +++ b/test/unit/attachment-source-endpoint.test.js @@ -0,0 +1,81 @@ +// test/unit/attachment-source-endpoint.test.js +// +// Integration test for GET /content/attachment-source?u=&dl=. +// Uses in-memory SQLite; no HANA binding required. +// +// Run: npx vitest run --project unit test/unit/attachment-source-endpoint.test.js +// +// NOTE: runs under the `unit` vitest project. It boots a full CAP server via +// cds.test('serve', '--project', '.', '--in-memory') — SQLite only, no cds bind/cf login. +// +// RED-BY-DESIGN until Task 11: the route GET /content/attachment-source is +// registered in srv/server.js at Task 11. All cases here will 404 at the +// router level (no matching route) until then. Do NOT register the route here. + +import { describe, it, expect } from 'vitest' +import cds from '@sap/cds' +import { createRequire } from 'node:module' + +const require = createRequire(import.meta.url) + +// Boot full CAP server in-memory (SQLite). cds.test registers its own +// beforeAll/afterAll hooks at this scope so all `it()` blocks run after boot. +const project = cds.test('serve', '--project', '.', '--in-memory') + +// Loaded after cds.test registers hooks; actual store access happens inside +// function bodies (invoked in it() blocks, post-boot). +const store = require('../../srv/lib/attachment-store.cjs') + +const base = '/content/attachment-source' + +describe('GET /content/attachment-source', () => { + it('streams a stored .txt inline', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/EX2.txt' + await store.put(url, { + buffer: Buffer.from('code'), + mimeType: 'text/plain; charset=utf-8', + contentHash: 'h', + slug: 's', + channel: 'prod', + filename: 'EX2.txt', + }) + const res = await project.get(`${base}?u=${encodeURIComponent(url)}`, { + responseType: 'arraybuffer', + }) + expect(res.status).toBe(200) + expect(res.headers['content-type']).toMatch(/text\/plain/) + expect(res.headers['content-disposition']).toMatch(/^inline/) + expect(res.headers['x-content-type-options']).toBe('nosniff') + }) + + it('dl=1 forces attachment disposition', async () => { + const url = 'https://raw.githubusercontent.com/o/r/main/D2.txt' + await store.put(url, { + buffer: Buffer.from('x'), + mimeType: 'text/plain; charset=utf-8', + contentHash: 'h2', + slug: 's', + channel: 'prod', + filename: 'D2.txt', + }) + const res = await project.get(`${base}?u=${encodeURIComponent(url)}&dl=1`, { + responseType: 'arraybuffer', + }) + expect(res.headers['content-disposition']).toMatch(/^attachment/) + }) + + it('400 on missing u', async () => { + await expect(project.get(base)).rejects.toMatchObject({ response: { status: 400 } }) + }) + + it('404 on a miss that cannot self-heal (disallowed host)', async () => { + // Use a host NOT in ATTACHMENT_HOSTS (raw.githubusercontent.com). + // fetchImageResponse / safeFetch will throw SSRF_BLOCKED synchronously + // (no network call) because example.invalid is not on the allowed-hosts set. + // attachmentSourceHandler catches the throw and returns 404. + const url = 'https://example.invalid/missing.txt' + await expect( + project.get(`${base}?u=${encodeURIComponent(url)}`) + ).rejects.toMatchObject({ response: { status: 404 } }) + }) +}) diff --git a/test/unit/attachment-store.test.js b/test/unit/attachment-store.test.js new file mode 100644 index 000000000..d5041e0cb --- /dev/null +++ b/test/unit/attachment-store.test.js @@ -0,0 +1,46 @@ +// test/unit/attachment-store.test.js +// +// Round-trip test for srv/lib/attachment-store.cjs — put → head → getStream. +// Uses in-memory SQLite; no HANA binding required. +// +// Run: npx vitest run --project unit test/unit/attachment-store.test.js +// +// NOTE: runs under the `unit` vitest project. It boots a full CAP server via +// cds.test('serve', '--in-memory') — SQLite only, no `cds bind`/cf-login. It +// lives under test/unit/ (NOT test/hybrid/) on purpose: the `hybrid` project's +// setup binds a live HANA and hangs without cf-login, whereas this test is +// self-contained in-memory. + +import { describe, it, expect } from 'vitest' +import cds from '@sap/cds' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) + +// Boot full CAP server in-memory (SQLite). cds.test registers its own +// beforeAll/afterAll hooks at this scope so all `it()` blocks run after boot. +cds.test('serve', '--project', '.', '--in-memory') + +// Loaded after cds.test registers hooks; actual cds.model access happens +// inside function bodies (invoked in it() blocks, post-boot). +const store = require('../../srv/lib/attachment-store.cjs') + +describe('attachment-store round-trip', () => { + const url = 'https://raw.githubusercontent.com/o/r/main/EX2.txt' + it('put → head → getStream returns the same bytes, mime, and filename', async () => { + const buffer = Buffer.from('@Search.searchable: true', 'utf8') + await store.put(url, { buffer, mimeType: 'text/plain; charset=utf-8', contentHash: 'h1', slug: 's', channel: 'prod', filename: 'EX2.txt' }) + const h = await store.head(url) + expect(h.exists).toBe(true) + expect(h.contentHash).toBe('h1') + expect(h.filename).toBe('EX2.txt') + const got = await store.getStream(url) + const chunks = [] + for await (const c of got.stream) chunks.push(c) + expect(Buffer.concat(chunks)).toEqual(buffer) + expect(got.mimeType).toMatch(/text\/plain/) + expect(got.filename).toBe('EX2.txt') + }) + it('head returns exists:false for an unknown url', async () => { + expect((await store.head('https://raw.githubusercontent.com/o/r/main/none.txt')).exists).toBe(false) + }) +}) diff --git a/test/unit/attachment-warm-utils.test.js b/test/unit/attachment-warm-utils.test.js new file mode 100644 index 000000000..3c0470336 --- /dev/null +++ b/test/unit/attachment-warm-utils.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect, vi } from 'vitest' +import { extractAttachmentUrls, warmAttachments } from '../../srv/lib/attachment-warm-utils.js' + +describe('extractAttachmentUrls', () => { + it('extracts and decodes u= from attachment-source hrefs (view + dl)', () => { + const raw = 'https://raw.githubusercontent.com/o/r/main/tutorials/s/EX2.txt' + const enc = encodeURIComponent(raw) + const html = `d↓` + expect(extractAttachmentUrls(html)).toEqual([raw]) // deduped + }) + it('returns [] when there are no attachment links', () => { + expect(extractAttachmentUrls('

no links

')).toEqual([]) + }) +}) + +describe('warmAttachments', () => { + it('calls ingestFn per url and never throws on failure', async () => { + const ingestFn = vi.fn().mockRejectedValue(new Error('boom')) + await expect(warmAttachments(['a', 'b'], { slug: 's', ingestFn })).resolves.toBeUndefined() + expect(ingestFn).toHaveBeenCalledTimes(2) + }) +}) + +import { readFileSync } from 'node:fs' +it('content-publish-session wires attachment warm', () => { + const src = readFileSync('srv/lib/content-publish-session.js', 'utf8') + expect(src).toContain('extractAttachmentUrls') + expect(src).toContain('warmAttachmentsLive') +}) diff --git a/test/unit/tutorial-assets-model.test.js b/test/unit/tutorial-assets-model.test.js new file mode 100644 index 000000000..3685188db --- /dev/null +++ b/test/unit/tutorial-assets-model.test.js @@ -0,0 +1,13 @@ +import { describe, it, expect } from 'vitest' +import cds from '@sap/cds' + +describe('TutorialAssets model', () => { + it('compiles with a filename column and an Attachments composition', async () => { + const m = await cds.load(['db/tutorial-assets.cds'], { root: '.' }) + const e = cds.linked(m).definitions['com.sap.developers.ims.TutorialAssets'] + expect(e).toBeTruthy() + expect(e.elements.sourceUrl.length).toBe(1024) + expect(e.elements.filename).toBeTruthy() + expect(e.elements.content.type).toBe('cds.Composition') + }) +})