diff --git a/src/core/json-summary.ts b/src/core/json-summary.ts index 13acf80..801b95b 100644 --- a/src/core/json-summary.ts +++ b/src/core/json-summary.ts @@ -44,6 +44,21 @@ export interface JsonSummaryDetail { model?: string; /** PROD-2542: the required fields it was missing, as "name (Type)". */ fields?: string[]; + /** + * PROD-2773 (additive): set on warnings that are not item failures, e.g. "foreignCdnReference" + * — one entry per asset URL into a CDN container owned by neither instance, with the URL, its + * container and every item/field/locale that references it. + */ + code?: string; + url?: string; + container?: string; + references?: Array<{ + contentID: number; + referenceName: string; + locale: string; + field: string; + kind: "attachment" | "text"; + }>; } /** diff --git a/src/core/push.ts b/src/core/push.ts index 19e544c..1b00072 100644 --- a/src/core/push.ts +++ b/src/core/push.ts @@ -19,6 +19,8 @@ import { Pull } from "./pull"; import { preflightReport } from "../lib/preflight/preflight-report"; import { buildJsonSummary, writeJsonSummary, JsonSummaryDetail, JsonSummaryPhase } from "./json-summary"; import type { FailureDetail } from "../types/sourceData"; +import { foreignCdnReport } from "../lib/content/foreign-cdn-report"; +import { resetForeignCdnDetectorCache } from "../lib/pushers/content-pusher/util/report-foreign-cdn-references"; /** * PROD-2310: Decide whether a set of auto-publish errors should fail the sync exit code. @@ -62,6 +64,9 @@ export class Push { // Preflight (PROD-2203): start each run with a clean report of planned actions preflightReport.reset(); + // PROD-2773: foreign CDN references are reported once per URL per run + foreignCdnReport.reset(); + resetForeignCdnDetectorCache(); // Initialize logger for push operation // Determine if this is a sync operation by checking if both source and target GUIDs exist @@ -299,6 +304,14 @@ export class Push { }); } + // PROD-2773: asset URLs into a CDN container owned by neither instance — passed through + // unchanged, so non-blocking, but an image/file field holding one may arrive empty. + if (foreignCdnReport.hasEntries()) { + const [headline, ...rest] = foreignCdnReport.renderSummaryLines(); + console.log(ansiColors.yellow(`\n ${headline}`)); + rest.forEach((line) => console.log(ansiColors.yellow(` ${line}`))); + } + // PROD-2311: post-publish mapping/version bookkeeping notices are NOT publish // failures — show them under a non-blocking header so they don't read as broken. if (mappingWarnings.length > 0) { @@ -320,6 +333,9 @@ export class Push { // Preflight (PROD-2203): print the report of actions that WOULD have been // taken, and signal conflicts via a non-zero exit code so automation can gate on it. if (state.preflight) { + if (foreignCdnReport.hasEntries()) { + preflightReport.recordWarning(`${foreignCdnReport.headline()} — see the list above`); + } preflightReport.print(); if (preflightReport.hasConflicts()) { process.exitCode = 1; @@ -334,7 +350,8 @@ export class Push { exitCode: typeof process.exitCode === "number" ? process.exitCode : success ? 0 : 1, phases: results.flatMap((r) => r.phases ?? []), failures: syncFailureDetails, - warnings: syncWarningDetails, + // PROD-2773: one additive entry per foreign CDN URL (code "foreignCdnReference") + warnings: [...syncWarningDetails, ...foreignCdnReport.toWarningDetails(sourceGuid)], operationErrors: syncErrors, autoPublishErrors, startedAt, diff --git a/src/core/tests/push.test.ts b/src/core/tests/push.test.ts index 63d4cc3..8a61459 100644 --- a/src/core/tests/push.test.ts +++ b/src/core/tests/push.test.ts @@ -1,5 +1,8 @@ import { Push, hasBlockingAutoPublishErrors, formatFailureName } from "../push"; import { resetState, setState } from "../state"; +import { Pull } from "../pull"; +import { Pushers } from "../../lib/pushers/orchestrate-pushers"; +import { foreignCdnReport } from "../../lib/content/foreign-cdn-report"; beforeEach(() => { resetState(); @@ -286,6 +289,85 @@ describe("Push.pushInstances — --jsonSummary", () => { expect(parsed.operationErrors).toEqual([{ type: "fatal", error: "Model validation failed" }]); }); + describe("foreign CDN references (PROD-2773)", () => { + const FOREIGN = "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png"; + + /** Have the mocked orchestrator record what the content pusher would have found. */ + async function runWithForeignReference(extraState: Record) { + jest.spyOn(Pull.prototype, "pullInstances").mockResolvedValue(undefined as any); + jest.spyOn(Pushers.prototype, "instanceOrchestrator").mockImplementation(async () => { + foreignCdnReport.setOwnContainers(["site-qa"], ["site-cat"]); + foreignCdnReport.record( + { url: FOREIGN, container: "upstream-qa", field: "mobileBackgroundImage", kind: "attachment" }, + { contentID: 474, referenceName: "home_homebanner_homebanner", locale: "en-us" } + ); + return [pushResult({ warningDetails: [{ name: "Home", error: "Dropped Component Hero — unresolved" }] })] as any; + }); + setState({ sourceGuid: "src-u", targetGuid: "tgt-u", ...extraState }); + return new Push().pushInstances(); + } + + it("adds one warning per URL to --jsonSummary without affecting success", async () => { + const out = path.join(tmp, "summary.json"); + const result = await runWithForeignReference({ jsonSummary: out }); + + const parsed = JSON.parse(fs.readFileSync(out, "utf8")); + expect(result.success).toBe(true); + expect(parsed.success).toBe(true); + expect(parsed.exitCode).toBe(0); + expect(parsed.schemaVersion).toBe(1); + expect(parsed.failures).toEqual([]); + expect(parsed.warnings).toHaveLength(2); + expect(parsed.warnings[0]).toEqual({ name: "Home", error: "Dropped Component Hero — unresolved" }); + expect(parsed.warnings[1]).toMatchObject({ + name: FOREIGN, + type: "content", + contentID: 474, + guid: "src-u", + locale: "en-us", + code: "foreignCdnReference", + url: FOREIGN, + container: "upstream-qa", + references: [ + { + contentID: 474, + referenceName: "home_homebanner_homebanner", + locale: "en-us", + field: "mobileBackgroundImage", + kind: "attachment", + }, + ], + }); + expect(parsed.warnings[1].error).toContain("belongs to neither the source (site-qa)"); + + const printed = (console.log as jest.Mock).mock.calls.map((c) => String(c[0])).join("\n"); + expect(printed).toContain("Foreign CDN references: 1 URL in 1 item (non-blocking):"); + expect(printed).toContain(FOREIGN); + }); + + it("lists the count in the preflight report without changing its totals or exit code", async () => { + const out = path.join(tmp, "summary.json"); + await runWithForeignReference({ jsonSummary: out, preflight: true }); + + const parsed = JSON.parse(fs.readFileSync(out, "utf8")); + expect(parsed.exitCode).toBe(0); + expect(process.exitCode).toBeUndefined(); + expect(parsed.preflight.totals).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(parsed.preflight.hasConflicts).toBe(false); + expect(parsed.warnings.some((w: any) => w.code === "foreignCdnReference")).toBe(true); + + const printed = (console.log as jest.Mock).mock.calls.map((c) => String(c[0])).join("\n"); + expect(printed).toContain("Warnings (1, non-blocking — not counted above):"); + }); + + it("starts each run with an empty report", async () => { + const out = path.join(tmp, "summary.json"); + await runWithForeignReference({}); + await runPush([pushResult()], { jsonSummary: out }); + expect(JSON.parse(fs.readFileSync(out, "utf8")).warnings).toEqual([]); + }); + }); + it("does not fail the run when the summary path cannot be written", async () => { // A reporting artifact must never turn a good sync into a failure. const blocker = path.join(tmp, "not-a-dir"); diff --git a/src/lib/content/foreign-cdn-detector.ts b/src/lib/content/foreign-cdn-detector.ts new file mode 100644 index 0000000..2be8371 --- /dev/null +++ b/src/lib/content/foreign-cdn-detector.ts @@ -0,0 +1,256 @@ +/** + * PROD-2773: detect asset URLs that point at an Agility CDN container belonging to NEITHER the + * source NOR the target instance of a sync. + * + * Content synced before the PROD-915 fix kept the URLs of the instance it was copied FROM, so a + * source can carry URLs into a third instance's container (e.g. `cdn-usa2.aglty.io/upstream-qa/…` + * in an instance whose own container is `site-qa`). The asset mapping only knows the + * source's own container, so those URLs pass through unchanged and silently: an image/file field + * holding one may be dropped by the target, and a link in rich text keeps pointing at the other + * instance. This module only DETECTS them — it never changes a value. + * + * Container ownership rule: + * - A URL is an Agility CDN URL when its host is a generic Agility CDN host (`cdn*.aglty.io`, + * `agility-cdn*.azurewebsites.net`, `cdn*.agilitycms.com`) or a custom edge host seen in the + * source/target container URLs (asset mappings, pulled assets). Non-CDN Agility hosts such as + * `api.aglty.io` or `manager.agilitycms.com` are deliberately not matched. + * - Its container is the first path segment (case-insensitive), whatever the host — the same + * container is served from the aglty.io edge, the azurewebsites origin and any custom domain. + * - It is foreign when that container is in neither the source's nor the target's known set. + * - When either side's container is unknown (no asset mappings and no pulled assets for it), any + * unclaimed container could plausibly be that side's, so nothing is reported at all. + */ +import * as mgmtApi from "@agility/management-sdk"; + +/** Generic Agility CDN hosts; anything else must be learned from the instances' own URLs. */ +const GENERIC_CDN_HOST_PATTERNS: RegExp[] = [ + /^cdn[a-z0-9-]*\.aglty\.io$/, + /^agility-?cdn[a-z0-9-]*\.azurewebsites\.net$/, + /^cdn[a-z0-9-]*\.agilitycms\.com$/, +]; + +// host, then the path up to the first character that cannot be part of a URL in text/HTML/JSON +const URL_PATTERN = /(https?:)?\/\/([a-z0-9-]+(?:\.[a-z0-9-]+)+)(?::\d+)?\/([^\s"'<>()\\[\]{}|^`]*)/gi; + +export interface CdnUrlMatch { + /** The asset URL as found (unescaped, query string and fragment removed). */ + url: string; + host: string; + /** Lower-cased first path segment. */ + container: string; +} + +export interface CdnOwnership { + sourceContainers: string[]; + targetContainers: string[]; + /** Custom (non-generic) edge hosts used by the source or target, lower-cased. */ + customHosts: string[]; +} + +export function isGenericAgilityCdnHost(host: string): boolean { + const h = host.toLowerCase(); + return GENERIC_CDN_HOST_PATTERNS.some((p) => p.test(h)); +} + +/** + * Undo the encodings a URL picks up when stored in a field: JSON-escaped slashes (`https:\/\/`), + * HTML-escaped ampersands (`&`) and percent-encoded nested URLs (`?src=https%3A%2F%2F…`). + * Only used for detection; field values are never rewritten. + */ +function normalizeText(text: string): string { + return text.replace(/\\+\//g, "/").replace(/&/gi, "&").replace(/%3A/gi, ":").replace(/%2F/gi, "/"); +} + +/** + * Every Agility CDN asset URL in a string (a plain URL, rich text, HTML or a JSON blob). + * A bare container URL (no file path) is not an asset reference and is ignored; on a custom host + * the last segment must also look like a file, since that host may serve ordinary pages too. + */ +export function extractAgilityCdnUrls(text: string, customHosts: Iterable = []): CdnUrlMatch[] { + if (!text || typeof text !== "string" || !text.includes("/")) return []; + const custom = new Set(Array.from(customHosts, (h) => h.toLowerCase())); + const matches: CdnUrlMatch[] = []; + + const normalized = normalizeText(text); + const pattern = new RegExp(URL_PATTERN.source, URL_PATTERN.flags); + let m: RegExpExecArray | null; + while ((m = pattern.exec(normalized)) !== null) { + const scheme = m[1] || ""; + const host = m[2].toLowerCase(); + const pathStart = m.index + m[0].length - m[3].length; + const path = m[3].split(/[?#]/)[0].replace(/[.,;:!*]+$/, ""); + // resume scanning inside this URL's query string, where another URL may be nested + // (e.g. an image proxy's ?src=https://cdn.aglty.io/...) + pattern.lastIndex = pathStart + path.length; + + const isGeneric = isGenericAgilityCdnHost(host); + if (!isGeneric && !custom.has(host)) continue; + + const segments = path.split("/").filter((s) => s.length > 0); + if (segments.length < 2) continue; + if (!isGeneric && !/\.[a-z0-9]{1,8}$/i.test(segments[segments.length - 1])) continue; + + matches.push({ url: `${scheme}//${m[2]}/${path}`, host, container: segments[0].toLowerCase() }); + } + return matches; +} + +/** Host and container of a container/asset URL such as `https://cdn-usa2.aglty.io/my-container`. */ +export function parseContainerUrl(url: string | null | undefined): { host: string; container: string } | null { + if (!url || typeof url !== "string") return null; + try { + const parsed = new URL(url); + const container = parsed.pathname.split("/").filter((s) => s.length > 0)[0]; + if (!container) return null; + return { host: parsed.hostname.toLowerCase(), container: decodeURIComponent(container).toLowerCase() }; + } catch { + return null; + } +} + +interface ContainerUrlRecord { + sourceContainerEdgeUrl?: string; + targetContainerEdgeUrl?: string; + sourceContainerOriginUrl?: string; + targetContainerOriginUrl?: string; + sourceUrl?: string; + targetUrl?: string; +} + +/** + * Collect each side's containers (and custom edge hosts) from the asset mapping records and the + * pulled asset lists. Either input may be empty; a side with nothing to learn from stays unknown. + */ +export function buildCdnOwnership(input: { + assetMappings?: ContainerUrlRecord[]; + sourceAssets?: Array>; + targetAssets?: Array>; +}): CdnOwnership { + const source = new Set(); + const target = new Set(); + const hosts = new Set(); + + const add = (side: Set, url: string | undefined) => { + const parsed = parseContainerUrl(url); + if (!parsed) return; + side.add(parsed.container); + if (!isGenericAgilityCdnHost(parsed.host)) hosts.add(parsed.host); + }; + + for (const m of input.assetMappings ?? []) { + if (!m) continue; + const hasContainerUrls = m.sourceContainerEdgeUrl || m.sourceContainerOriginUrl; + add(source, m.sourceContainerEdgeUrl); + add(source, m.sourceContainerOriginUrl); + add(target, m.targetContainerEdgeUrl); + add(target, m.targetContainerOriginUrl); + // legacy records (before container URLs were tracked) only have the per-asset URLs + if (!hasContainerUrls) add(source, m.sourceUrl); + if (!m.targetContainerEdgeUrl && !m.targetContainerOriginUrl) add(target, m.targetUrl); + } + + for (const [assets, side] of [ + [input.sourceAssets, source], + [input.targetAssets, target], + ] as const) { + for (const a of assets ?? []) { + if (!a) continue; + add(side, a.containerEdgeUrl); + add(side, a.containerOriginUrl); + add(side, a.edgeUrl); + add(side, a.originUrl); + } + } + + return { + sourceContainers: Array.from(source), + targetContainers: Array.from(target), + customHosts: Array.from(hosts), + }; +} + +export class ForeignCdnDetector { + private source: Set; + private target: Set; + private customHosts: Set; + + constructor(ownership: CdnOwnership) { + this.source = new Set(ownership.sourceContainers.map((c) => c.toLowerCase())); + this.target = new Set(ownership.targetContainers.map((c) => c.toLowerCase())); + this.customHosts = new Set(ownership.customHosts.map((h) => h.toLowerCase())); + } + + /** False when either side's container is unknown — see the ownership rule above. */ + isEnabled(): boolean { + return this.source.size > 0 && this.target.size > 0; + } + + get sourceContainers(): string[] { + return Array.from(this.source); + } + + get targetContainers(): string[] { + return Array.from(this.target); + } + + isForeignContainer(container: string): boolean { + const c = container.toLowerCase(); + return this.isEnabled() && !this.source.has(c) && !this.target.has(c); + } + + findForeignUrls(text: string): CdnUrlMatch[] { + if (!this.isEnabled()) return []; + return extractAgilityCdnUrls(text, this.customHosts).filter( + (m) => !this.source.has(m.container) && !this.target.has(m.container) + ); + } +} + +/** "attachment": the URL of an image/file/attachment-list field. "text": inside a string field. */ +export type ForeignCdnFieldKind = "attachment" | "text"; + +export interface ForeignCdnFieldReference extends CdnUrlMatch { + /** Field path, named the way ContentFieldMapper names nested values (`field[0]`, `field.sub`). */ + field: string; + kind: ForeignCdnFieldKind; +} + +/** + * Walk a content item's fields and return every foreign CDN reference, one per (field, URL). + * Mirrors ContentFieldMapper's shapes: an object with a string `url` is an attachment (ImageAttachment, + * FileAttachment, an AttachmentList entry); any other string is scanned as text. + */ +export function collectForeignCdnReferences(fields: any, detector: ForeignCdnDetector): ForeignCdnFieldReference[] { + if (!detector.isEnabled() || !fields || typeof fields !== "object") return []; + const refs: ForeignCdnFieldReference[] = []; + const seen = new Set(); + + const push = (field: string, kind: ForeignCdnFieldKind, matches: CdnUrlMatch[]) => { + for (const m of matches) { + const key = `${field}\u0000${m.url}`; + if (seen.has(key)) continue; + seen.add(key); + refs.push({ ...m, field, kind }); + } + }; + + const walk = (path: string, value: any) => { + if (value === null || value === undefined) return; + if (typeof value === "string") { + push(path, "text", detector.findForeignUrls(value)); + } else if (Array.isArray(value)) { + value.forEach((v, i) => walk(`${path}[${i}]`, v)); + } else if (typeof value === "object") { + for (const [key, v] of Object.entries(value)) { + if (key === "url" && typeof v === "string") { + push(path, "attachment", detector.findForeignUrls(v)); + } else { + walk(`${path}.${key}`, v); + } + } + } + }; + + for (const [name, value] of Object.entries(fields)) walk(name, value); + return refs; +} diff --git a/src/lib/content/foreign-cdn-report.ts b/src/lib/content/foreign-cdn-report.ts new file mode 100644 index 0000000..c5e9ebe --- /dev/null +++ b/src/lib/content/foreign-cdn-report.ts @@ -0,0 +1,186 @@ +/** + * PROD-2773: per-run collector for foreign CDN references (see foreign-cdn-detector.ts). + * + * The content pusher records each reference it finds; this collector dedupes them to one entry + * per distinct URL, aggregating the items/fields/locales that use it, and renders that entry for + * the console/log, the end-of-run summary, --jsonSummary and the --preflight report. + * + * Reporting only: nothing here affects item payloads, counts, `success` or the exit code. + * A process-level singleton, like preflightReport, because pushers share state globally. + */ +import type { ForeignCdnFieldKind, ForeignCdnFieldReference } from "./foreign-cdn-detector"; + +export interface ForeignCdnOccurrence { + contentID: number; + referenceName: string; + locale: string; + field: string; + kind: ForeignCdnFieldKind; +} + +export interface ForeignCdnUrlEntry { + url: string; + container: string; + occurrences: ForeignCdnOccurrence[]; +} + +/** Shape of a --jsonSummary `warnings` entry (JsonSummaryDetail plus additive fields). */ +export interface ForeignCdnWarningDetail { + name: string; + error: string; + type: "content"; + contentID?: number; + guid?: string; + locale?: string; + code: "foreignCdnReference"; + url: string; + container: string; + references: ForeignCdnOccurrence[]; +} + +export const FOREIGN_CDN_WARNING_CODE = "foreignCdnReference"; + +class ForeignCdnReport { + private entries = new Map(); + private sourceContainers: string[] = []; + private targetContainers: string[] = []; + + reset(): void { + this.entries.clear(); + this.sourceContainers = []; + this.targetContainers = []; + } + + /** The containers the run treated as its own, quoted in every message. */ + setOwnContainers(sourceContainers: string[], targetContainers: string[]): void { + this.sourceContainers = sourceContainers; + this.targetContainers = targetContainers; + } + + /** + * Record one reference. Returns "new-url" the first time a URL is seen in this run, + * "new-occurrence" for a further item/field/locale using it, and "duplicate" otherwise. + */ + record( + ref: Pick, + item: { contentID: number; referenceName: string; locale: string } + ): "new-url" | "new-occurrence" | "duplicate" { + const key = ref.url.toLowerCase(); + let entry = this.entries.get(key); + const isNewUrl = !entry; + if (!entry) { + entry = { url: ref.url, container: ref.container, occurrences: [] }; + this.entries.set(key, entry); + } + const exists = entry.occurrences.some( + (o) => o.contentID === item.contentID && o.locale === item.locale && o.field === ref.field + ); + if (exists) return "duplicate"; + entry.occurrences.push({ ...item, field: ref.field, kind: ref.kind }); + return isNewUrl ? "new-url" : "new-occurrence"; + } + + /** Distinct URLs in first-seen order, those used by an image/file field (higher risk) first. */ + getEntries(): ForeignCdnUrlEntry[] { + const rank = (e: ForeignCdnUrlEntry) => (e.occurrences.some((o) => o.kind === "attachment") ? 0 : 1); + return Array.from(this.entries.values()).sort((a, b) => rank(a) - rank(b)); + } + + /** Distinct source content items (by contentID; locales of one item count once). */ + getItemCount(): number { + const ids = new Set(); + this.entries.forEach((e) => e.occurrences.forEach((o) => ids.add(o.contentID))); + return ids.size; + } + + hasEntries(): boolean { + return this.entries.size > 0; + } + + /** "Foreign CDN references: N URLs in M items" */ + headline(): string { + const urls = this.entries.size; + const items = this.getItemCount(); + return `Foreign CDN references: ${urls} URL${urls === 1 ? "" : "s"} in ${items} item${items === 1 ? "" : "s"}`; + } + + /** What is wrong with the URL and what happens to it; shared by every surface. */ + describe(container: string, kinds: ForeignCdnFieldKind[]): string { + const own = (list: string[]) => (list.length > 0 ? list.join(", ") : "unknown"); + const consequences: string[] = []; + if (kinds.includes("attachment")) { + consequences.push("it is set in an image/file field, which the target may drop (the field arrives empty)"); + } + if (kinds.includes("text")) { + consequences.push( + "it is embedded in a text/rich-text field, so the synced content keeps linking to that container" + ); + } + return ( + `references CDN container '${container}', which belongs to neither the source (${own(this.sourceContainers)}) ` + + `nor the target (${own(this.targetContainers)}). The URL is passed through unchanged; ${consequences.join("; ")}. ` + + `Repoint it at the source instance's own asset.` + ); + } + + /** One log line for a single reference, used the first time its URL is seen. */ + formatOccurrence( + ref: Pick, + item: { contentID: number; referenceName: string; locale: string } + ): string { + const where = ref.kind === "attachment" ? "image/file field" : "text field"; + return ( + `⚠️ Foreign CDN reference: ${item.referenceName} (contentID: ${item.contentID}) [${item.locale}] ${where} '${ref.field}' ` + + `→ ${ref.url} ${this.describe(ref.container, [ref.kind])}` + ); + } + + private formatOccurrenceList(occurrences: ForeignCdnOccurrence[]): string { + return occurrences + .map( + (o) => + `${o.referenceName} #${o.contentID} [${o.locale}] ${o.field}${o.kind === "attachment" ? " (image/file)" : ""}` + ) + .join("; "); + } + + /** Lines for the end-of-run summary: headline, then each URL with the items using it. */ + renderSummaryLines(): string[] { + if (!this.hasEntries()) return []; + const lines = [`${this.headline()} (non-blocking):`]; + for (const e of this.getEntries()) { + const kinds = Array.from(new Set(e.occurrences.map((o) => o.kind))); + const risk = kinds.includes("attachment") ? "image/file field" : "text only"; + lines.push(` • ${e.url} — container '${e.container}', ${risk}`); + lines.push(` used by: ${this.formatOccurrenceList(e.occurrences)}`); + } + lines.push( + ` These URLs belong to neither the source (${this.sourceContainers.join(", ") || "unknown"}) nor the target ` + + `(${this.targetContainers.join(", ") || "unknown"}) and are passed through unchanged. Repoint them at the source instance's own assets.` + ); + return lines; + } + + /** One --jsonSummary warning per distinct URL, in getEntries() order. */ + toWarningDetails(sourceGuid?: string): ForeignCdnWarningDetail[] { + return this.getEntries().map((e) => { + const kinds = Array.from(new Set(e.occurrences.map((o) => o.kind))); + const first = e.occurrences[0]; + return { + name: e.url, + error: `${this.describe(e.container, kinds)} Used by: ${this.formatOccurrenceList(e.occurrences)}.`, + type: "content" as const, + contentID: first?.contentID, + guid: sourceGuid, + locale: first?.locale, + code: FOREIGN_CDN_WARNING_CODE, + url: e.url, + container: e.container, + references: e.occurrences, + }; + }); + } +} + +/** Process-level singleton shared by the content pusher and the end-of-run summary. */ +export const foreignCdnReport = new ForeignCdnReport(); diff --git a/src/lib/content/tests/foreign-cdn-detector.test.ts b/src/lib/content/tests/foreign-cdn-detector.test.ts new file mode 100644 index 0000000..c05603d --- /dev/null +++ b/src/lib/content/tests/foreign-cdn-detector.test.ts @@ -0,0 +1,215 @@ +import { + buildCdnOwnership, + collectForeignCdnReferences, + extractAgilityCdnUrls, + ForeignCdnDetector, + isGenericAgilityCdnHost, + parseContainerUrl, +} from "lib/content/foreign-cdn-detector"; + +// A clone pair: source container on the aglty.io edge and a custom domain, target likewise. +const SOURCE = "clone-qa-20261002"; +const TARGET = "clone-cat-20261002"; +const mapping = { + sourceUrl: `https://cdn.example.com/${SOURCE}/a/b.png`, + targetUrl: `https://cdn.example.com/${TARGET}/a/b.png`, + sourceContainerEdgeUrl: `https://cdn.example.com/${SOURCE}`, + targetContainerEdgeUrl: `https://cdn.example.com/${TARGET}`, + sourceContainerOriginUrl: `https://agility-cdn-usa2.azurewebsites.net/${SOURCE}`, + targetContainerOriginUrl: `https://agility-cdn-usa2.azurewebsites.net/${TARGET}`, +}; + +function detector(): ForeignCdnDetector { + return new ForeignCdnDetector(buildCdnOwnership({ assetMappings: [mapping] })); +} + +describe("isGenericAgilityCdnHost", () => { + it("matches the Agility CDN edge, origin and legacy hosts", () => { + expect(isGenericAgilityCdnHost("cdn.aglty.io")).toBe(true); + expect(isGenericAgilityCdnHost("cdn-usa2.aglty.io")).toBe(true); + expect(isGenericAgilityCdnHost("agility-cdn-usa2.azurewebsites.net")).toBe(true); + expect(isGenericAgilityCdnHost("cdn.agilitycms.com")).toBe(true); + }); + + it("does not match non-CDN Agility hosts or unrelated hosts", () => { + expect(isGenericAgilityCdnHost("api.aglty.io")).toBe(false); + expect(isGenericAgilityCdnHost("manager.agilitycms.com")).toBe(false); + expect(isGenericAgilityCdnHost("myapp.azurewebsites.net")).toBe(false); + expect(isGenericAgilityCdnHost("www.example.org")).toBe(false); + }); +}); + +describe("parseContainerUrl", () => { + it("returns the host and lower-cased first path segment", () => { + expect(parseContainerUrl("https://cdn-usa2.aglty.io/Upstream-QA/x.png")).toEqual({ + host: "cdn-usa2.aglty.io", + container: "upstream-qa", + }); + }); + + it("returns null for missing or unparseable input", () => { + expect(parseContainerUrl(undefined)).toBeNull(); + expect(parseContainerUrl("not a url")).toBeNull(); + expect(parseContainerUrl("https://cdn.aglty.io/")).toBeNull(); + }); +}); + +describe("buildCdnOwnership", () => { + it("learns each side's containers and custom edge hosts from asset mappings", () => { + const own = buildCdnOwnership({ assetMappings: [mapping] }); + expect(own.sourceContainers).toEqual([SOURCE]); + expect(own.targetContainers).toEqual([TARGET]); + expect(own.customHosts).toEqual(["cdn.example.com"]); + }); + + it("learns containers from pulled assets when there are no mappings", () => { + const own = buildCdnOwnership({ + sourceAssets: [ + { containerEdgeUrl: "https://cdn-usa2.aglty.io/src-c", edgeUrl: "https://cdn-usa2.aglty.io/src-c/x.png" }, + ], + targetAssets: [{ originUrl: "https://agility-cdn-usa2.azurewebsites.net/tgt-c/y.png" }], + }); + expect(own.sourceContainers).toEqual(["src-c"]); + expect(own.targetContainers).toEqual(["tgt-c"]); + expect(own.customHosts).toEqual([]); + }); + + it("falls back to per-asset URLs for legacy mapping records", () => { + const own = buildCdnOwnership({ + assetMappings: [ + { sourceUrl: "https://cdn.aglty.io/old-src/a.png", targetUrl: "https://cdn.aglty.io/old-tgt/a.png" }, + ], + }); + expect(own.sourceContainers).toEqual(["old-src"]); + expect(own.targetContainers).toEqual(["old-tgt"]); + }); +}); + +describe("extractAgilityCdnUrls", () => { + it("finds an aglty.io URL and its container", () => { + expect(extractAgilityCdnUrls("https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png")).toEqual([ + { + url: "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png", + host: "cdn-usa2.aglty.io", + container: "upstream-qa", + }, + ]); + }); + + it("finds an azurewebsites origin URL", () => { + const [m] = extractAgilityCdnUrls(''); + expect(m.container).toBe("other"); + }); + + it("ignores custom-domain URLs unless the host is known, and then requires a file name", () => { + const html = + 'x '; + expect(extractAgilityCdnUrls(html)).toEqual([]); + expect(extractAgilityCdnUrls(html, ["cdn.example.com"]).map((m) => m.url)).toEqual([ + "https://cdn.example.com/foo/x.svg", + ]); + }); + + it("unescapes JSON-escaped slashes", () => { + const json = '{"img":"https:\\/\\/cdn.aglty.io\\/other\\/a.png"}'; + expect(extractAgilityCdnUrls(json).map((m) => m.url)).toEqual(["https://cdn.aglty.io/other/a.png"]); + }); + + it("strips query strings (including & variants) and fragments", () => { + const html = + ''; + expect(extractAgilityCdnUrls(html).map((m) => m.url)).toEqual([ + "https://cdn.aglty.io/other/a.png", + "https://cdn.aglty.io/other/b.png", + ]); + }); + + it("finds percent-encoded URLs nested in a query string", () => { + const text = "https://images.example.com/resize?src=https%3A%2F%2Fcdn.aglty.io%2Fother%2Fa.png&w=10"; + expect(extractAgilityCdnUrls(text).map((m) => m.container)).toEqual(["other"]); + }); + + it("finds protocol-relative URLs and drops trailing punctuation", () => { + expect(extractAgilityCdnUrls("see //cdn.aglty.io/other/a.png.").map((m) => m.url)).toEqual([ + "//cdn.aglty.io/other/a.png", + ]); + }); + + it("ignores bare container URLs and non-CDN Agility hosts", () => { + expect(extractAgilityCdnUrls("https://cdn.aglty.io/other")).toEqual([]); + expect(extractAgilityCdnUrls("https://api.aglty.io/abc-u/fetch/en-us/item/1")).toEqual([]); + expect(extractAgilityCdnUrls("https://manager.agilitycms.com/instance/abc/x")).toEqual([]); + }); + + it("returns every occurrence, duplicates included", () => { + expect(extractAgilityCdnUrls("https://cdn.aglty.io/o/a.png https://cdn.aglty.io/o/a.png")).toHaveLength(2); + }); +}); + +describe("ForeignCdnDetector", () => { + it("does not flag the source's or target's own containers on any host", () => { + const d = detector(); + const text = [ + `https://cdn-usa2.aglty.io/${SOURCE}/x.png`, + `https://cdn.example.com/${SOURCE}/x.png`, + `https://agility-cdn-usa2.azurewebsites.net/${SOURCE}/x.png`, + `https://cdn-usa2.aglty.io/${TARGET}/x.png`, + `https://cdn.example.com/${TARGET.toUpperCase()}/x.png`, + ].join(" "); + expect(d.findForeignUrls(text)).toEqual([]); + }); + + it("flags a third instance's container on aglty.io and on the shared custom domain", () => { + const d = detector(); + const text = + "https://cdn-usa2.aglty.io/upstream-qa/a.png https://cdn.example.com/site-qa/upload/b.svg"; + expect(d.findForeignUrls(text).map((m) => m.container)).toEqual(["upstream-qa", "site-qa"]); + }); + + it("reports nothing when either side's container is unknown", () => { + const sourceOnly = new ForeignCdnDetector({ sourceContainers: [SOURCE], targetContainers: [], customHosts: [] }); + const targetOnly = new ForeignCdnDetector({ sourceContainers: [], targetContainers: [TARGET], customHosts: [] }); + expect(sourceOnly.isEnabled()).toBe(false); + expect(targetOnly.isEnabled()).toBe(false); + expect(sourceOnly.findForeignUrls("https://cdn.aglty.io/other/a.png")).toEqual([]); + expect(targetOnly.findForeignUrls("https://cdn.aglty.io/other/a.png")).toEqual([]); + }); +}); + +describe("collectForeignCdnReferences", () => { + const foreign = "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png"; + + it("classifies ImageAttachment, AttachmentList, plain URL and rich text fields", () => { + const fields = { + mobileBackgroundImage: { label: "Hero Image", url: foreign, target: null }, + gallery: [{ url: `https://cdn.aglty.io/upstream-qa/g1.png` }, { url: `https://cdn.aglty.io/${SOURCE}/g2.png` }], + imagesrc: "https://cdn.aglty.io/othersite/footer/find-a-store.png", + body: `

`, + title: "No URLs here", + }; + const refs = collectForeignCdnReferences(fields, detector()); + expect(refs.map(({ field, kind, container }) => ({ field, kind, container }))).toEqual([ + { field: "mobileBackgroundImage", kind: "attachment", container: "upstream-qa" }, + { field: "gallery[0]", kind: "attachment", container: "upstream-qa" }, + { field: "imagesrc", kind: "text", container: "othersite" }, + { field: "body", kind: "text", container: "upstream-qa" }, + ]); + }); + + it("reports a URL repeated within one field once", () => { + const refs = collectForeignCdnReferences({ body: `` }, detector()); + expect(refs).toHaveLength(1); + }); + + it("never mutates the fields it scans", () => { + const fields = { a: { url: foreign }, b: [{ url: foreign }], c: `` }; + const before = JSON.stringify(fields); + collectForeignCdnReferences(Object.freeze(fields), detector()); + expect(JSON.stringify(fields)).toBe(before); + }); + + it("returns nothing when the detector is disabled", () => { + const off = new ForeignCdnDetector({ sourceContainers: [], targetContainers: [], customHosts: [] }); + expect(collectForeignCdnReferences({ a: { url: foreign } }, off)).toEqual([]); + }); +}); diff --git a/src/lib/content/tests/foreign-cdn-report.test.ts b/src/lib/content/tests/foreign-cdn-report.test.ts new file mode 100644 index 0000000..1085a77 --- /dev/null +++ b/src/lib/content/tests/foreign-cdn-report.test.ts @@ -0,0 +1,211 @@ +import { foreignCdnReport } from "lib/content/foreign-cdn-report"; +import { ForeignCdnDetector } from "lib/content/foreign-cdn-detector"; +import { ContentFieldMapper } from "lib/content/content-field-mapper"; +import { reportForeignCdnReferences } from "lib/pushers/content-pusher/util/report-foreign-cdn-references"; + +const SOURCE = "site-qa"; +const TARGET = "site-cat"; +const FOREIGN = "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png"; +const FOREIGN_TEXT = "https://cdn-usa2.aglty.io/upstream-qa/navigation-icons/contact-us-success-image.png"; + +const detector = new ForeignCdnDetector({ sourceContainers: [SOURCE], targetContainers: [TARGET], customHosts: [] }); + +function item(contentID: number, referenceName: string, fields: any): any { + return { contentID, properties: { referenceName }, fields }; +} + +function makeLogger() { + return { warning: jest.fn(), fileOnly: jest.fn() }; +} + +beforeEach(() => { + foreignCdnReport.reset(); + jest.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +describe("foreignCdnReport", () => { + it("dedupes to one entry per URL, aggregating items, fields and locales", () => { + const logger = makeLogger(); + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger, detector }; + const banner = (id: number) => + item(id, "home_homebanner_homebanner", { mobileBackgroundImage: { label: "x", url: FOREIGN } }); + + reportForeignCdnReferences([banner(474), banner(475)], "en-us", opts); + reportForeignCdnReferences([banner(474)], "pt-br", opts); + // the same item/locale/field scanned again (e.g. a second pass) is not a new occurrence + reportForeignCdnReferences([banner(474)], "pt-br", opts); + + const entries = foreignCdnReport.getEntries(); + expect(entries).toHaveLength(1); + expect(entries[0].container).toBe("upstream-qa"); + expect(entries[0].occurrences).toEqual([ + { + contentID: 474, + referenceName: "home_homebanner_homebanner", + locale: "en-us", + field: "mobileBackgroundImage", + kind: "attachment", + }, + { + contentID: 475, + referenceName: "home_homebanner_homebanner", + locale: "en-us", + field: "mobileBackgroundImage", + kind: "attachment", + }, + { + contentID: 474, + referenceName: "home_homebanner_homebanner", + locale: "pt-br", + field: "mobileBackgroundImage", + kind: "attachment", + }, + ]); + expect(foreignCdnReport.getItemCount()).toBe(2); + expect(foreignCdnReport.headline()).toBe("Foreign CDN references: 1 URL in 2 items"); + + // console/log warning once per URL; further occurrences go to the log file only + expect(logger.warning).toHaveBeenCalledTimes(1); + expect(logger.warning.mock.calls[0][0]).toContain("references CDN container 'upstream-qa'"); + expect(logger.fileOnly).toHaveBeenCalledTimes(2); + }); + + it("distinguishes image/file fields from text fields in the message", () => { + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger: makeLogger(), detector }; + reportForeignCdnReferences( + [ + item(474, "banner", { img: { url: FOREIGN } }), + item(1271, "contactus", { body: `

` }), + ], + "en-us", + opts + ); + + const [attachment, text] = foreignCdnReport.toWarningDetails("src-u"); + expect(attachment.error).toContain( + `references CDN container 'upstream-qa', which belongs to neither the source (${SOURCE}) nor the target (${TARGET}). The URL is passed through unchanged` + ); + expect(attachment.error).toContain("image/file field, which the target may drop"); + expect(attachment.error).toContain("Repoint it at the source instance's own asset."); + expect(attachment.error).not.toContain("text/rich-text field"); + expect(text.error).toContain("embedded in a text/rich-text field"); + expect(text.error).not.toContain("image/file field"); + }); + + it("builds additive --jsonSummary warning entries, attachment-field URLs first", () => { + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger: makeLogger(), detector }; + reportForeignCdnReferences( + [ + item(1271, "contactus", { body: `` }), + item(474, "banner", { img: { url: FOREIGN } }), + ], + "en-us", + opts + ); + + const details = foreignCdnReport.toWarningDetails("src-u"); + expect(details.map((d) => d.url)).toEqual([FOREIGN, FOREIGN_TEXT]); + expect(details[0]).toMatchObject({ + name: FOREIGN, + type: "content", + contentID: 474, + guid: "src-u", + locale: "en-us", + code: "foreignCdnReference", + container: "upstream-qa", + references: [{ contentID: 474, referenceName: "banner", locale: "en-us", field: "img", kind: "attachment" }], + }); + expect(details[0].error).toContain("Used by: banner #474 [en-us] img (image/file)"); + }); + + it("renders the end-of-run summary block", () => { + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger: makeLogger(), detector }; + reportForeignCdnReferences([item(474, "banner", { img: { url: FOREIGN } })], "en-us", opts); + const lines = foreignCdnReport.renderSummaryLines(); + expect(lines[0]).toBe("Foreign CDN references: 1 URL in 1 item (non-blocking):"); + expect(lines.join("\n")).toContain(`• ${FOREIGN} — container 'upstream-qa', image/file field`); + expect(lines.join("\n")).toContain("used by: banner #474 [en-us] img (image/file)"); + }); + + it("records nothing for the instances' own containers or when a side is unknown", () => { + const own = `https://cdn-usa2.aglty.io/${SOURCE}/a.png https://cdn.aglty.io/${TARGET}/b.png`; + reportForeignCdnReferences( + [item(1, "x", { body: own, img: { url: `https://cdn.aglty.io/${SOURCE}/c.png` } })], + "en-us", + { + sourceGuid: "s", + targetGuid: "t", + detector, + } + ); + const unknownTarget = new ForeignCdnDetector({ sourceContainers: [SOURCE], targetContainers: [], customHosts: [] }); + reportForeignCdnReferences([item(2, "x", { img: { url: FOREIGN } })], "en-us", { + sourceGuid: "s", + targetGuid: "t", + detector: unknownTarget, + }); + expect(foreignCdnReport.hasEntries()).toBe(false); + expect(foreignCdnReport.renderSummaryLines()).toEqual([]); + }); +}); + +describe("foreign CDN detection alongside ContentFieldMapper", () => { + // Detection is reporting only: the payload the field mapper builds must be exactly what it + // was before, for every field shape that can carry a foreign URL. + const fields = { + mobileBackgroundImage: { + label: "Hero Image", + url: FOREIGN, + target: null, + pixelHeight: null, + pixelWidth: null, + }, + attachments: [ + { label: "a", url: FOREIGN }, + { label: "b", url: "https://cdn.aglty.io/upstream-qa/b.pdf" }, + ], + imagesrc: "https://cdn.aglty.io/othersite/footer/find-a-store.png", + richText: `

`, + }; + + const context = (): any => ({ + referenceMapper: { getContentItemMappingByContentID: jest.fn().mockReturnValue(null) }, + assetMapper: { + getAssetMappingByMediaUrl: jest.fn().mockReturnValue(null), + remapUrlByContainer: jest.fn().mockReturnValue(null), + isKnownAssetUrl: jest.fn().mockReturnValue(false), + }, + }); + + it("finds every foreign reference while the mapped payload stays identical", () => { + const original = JSON.parse(JSON.stringify(fields)); + const before = new ContentFieldMapper().mapContentFields(original, context()); + + reportForeignCdnReferences([item(474, "banner", original)], "en-us", { + sourceGuid: "s", + targetGuid: "t", + logger: makeLogger(), + detector, + }); + + const after = new ContentFieldMapper().mapContentFields(original, context()); + expect(after).toEqual(before); + expect(after.mappedFields).toEqual(fields); + expect(original).toEqual(fields); + + const refs = foreignCdnReport.getEntries().flatMap((e) => e.occurrences.map((o) => `${o.field}:${o.kind}`)); + expect(refs.sort()).toEqual( + [ + "attachments[0]:attachment", + "attachments[1]:attachment", + "imagesrc:text", + "mobileBackgroundImage:attachment", + "richText:text", + ].sort() + ); + }); +}); diff --git a/src/lib/mappers/asset-mapper.ts b/src/lib/mappers/asset-mapper.ts index 47637ca..ead4904 100644 --- a/src/lib/mappers/asset-mapper.ts +++ b/src/lib/mappers/asset-mapper.ts @@ -49,6 +49,11 @@ export class AssetMapper { return mapping; } + /** PROD-2773: every record, read-only — the foreign CDN detector reads each side's container URLs. */ + getMappings(): ReadonlyArray { + return this.mappings; + } + getAssetMappingByMediaUrl(url: string, type: "source" | "target"): AssetMapping | null { // Try exact match first const exact = this.mappings.find((m: AssetMapping) => diff --git a/src/lib/preflight/preflight-report.ts b/src/lib/preflight/preflight-report.ts index c736b48..00dba94 100644 --- a/src/lib/preflight/preflight-report.ts +++ b/src/lib/preflight/preflight-report.ts @@ -63,6 +63,9 @@ const ACTION_GLYPH: Record = { class PreflightReport { private entries: PreflightEntry[] = []; + // PROD-2773: non-blocking notices shown under their own heading. Kept apart from `entries` + // so they never change a phase count, the totals, or the conflict exit code. + private warnings: string[] = []; /** Whether preflight mode is active for the current run. */ isEnabled(): boolean { @@ -72,6 +75,7 @@ class PreflightReport { /** Clear all recorded actions (called per-command via resetState). */ reset(): void { this.entries = []; + this.warnings = []; } /** @@ -83,6 +87,16 @@ class PreflightReport { this.entries.push(entry); } + /** PROD-2773: record a non-blocking warning line. No-op when preflight mode is off. */ + recordWarning(message: string): void { + if (!this.isEnabled()) return; + this.warnings.push(message); + } + + getWarnings(): string[] { + return this.warnings; + } + /** True if any item would conflict (used to set a non-zero exit code). */ hasConflicts(): boolean { return this.entries.some((e) => e.action === "conflict"); @@ -176,6 +190,7 @@ class PreflightReport { const phases = this.getPhaseSummaries(); if (phases.length === 0) { lines.push(ansiColors.gray("\nNothing to do — no creates, updates, skips, or conflicts detected.\n")); + lines.push(...this.renderWarnings()); return lines.join("\n"); } @@ -221,11 +236,21 @@ class PreflightReport { if (this.hasConflicts()) { lines.push(ansiColors.red("\n⚠️ Conflicts detected — a real sync would require --overwrite (exit code 1).")); } + lines.push(...this.renderWarnings()); lines.push(ansiColors.cyan(bar)); return lines.join("\n"); } + /** PROD-2773: the warnings section — not counted in the totals and never affects the exit code. */ + private renderWarnings(): string[] { + if (this.warnings.length === 0) return []; + return [ + ansiColors.yellow(`\nWarnings (${this.warnings.length}, non-blocking — not counted above):`), + ...this.warnings.map((w) => ansiColors.yellow(` ⚠ ${w}`)), + ]; + } + /** Print the report to stdout. */ print(): void { console.log(this.renderTable()); diff --git a/src/lib/preflight/tests/preflight-report.test.ts b/src/lib/preflight/tests/preflight-report.test.ts index ac2318a..ab10e6b 100644 --- a/src/lib/preflight/tests/preflight-report.test.ts +++ b/src/lib/preflight/tests/preflight-report.test.ts @@ -420,3 +420,42 @@ describe("invalid action (PROD-2542)", () => { expect(preflightReport.renderTable()).not.toContain("invalid"); }); }); + +// ─── warnings (PROD-2773) ───────────────────────────────────────────────────── + +describe("recordWarning", () => { + it("is a no-op when preflight is disabled", () => { + preflightReport.recordWarning("Foreign CDN references: 1 URL in 1 item"); + expect(preflightReport.getWarnings()).toEqual([]); + }); + + it("renders a warnings section without changing counts, totals or the conflict signal", () => { + setState({ preflight: true }); + preflightReport.record({ phase: "Content", action: "update", name: "banner", locale: "en-us" }); + const before = { totals: preflightReport.getTotals(), json: preflightReport.toJSON() }; + + preflightReport.recordWarning("Foreign CDN references: 14 URLs in 11 items — see the list above"); + + expect(preflightReport.getTotals()).toEqual(before.totals); + expect(preflightReport.toJSON()).toEqual(before.json); + expect(preflightReport.hasConflicts()).toBe(false); + const table = preflightReport.renderTable(); + expect(table).toContain("Warnings (1, non-blocking — not counted above):"); + expect(table).toContain("Foreign CDN references: 14 URLs in 11 items"); + }); + + it("renders warnings even when there is nothing to do", () => { + setState({ preflight: true }); + preflightReport.recordWarning("Foreign CDN references: 1 URL in 1 item"); + const table = preflightReport.renderTable(); + expect(table).toContain("Nothing to do"); + expect(table).toContain("Foreign CDN references: 1 URL in 1 item"); + }); + + it("is cleared by reset", () => { + setState({ preflight: true }); + preflightReport.recordWarning("x"); + preflightReport.reset(); + expect(preflightReport.getWarnings()).toEqual([]); + }); +}); diff --git a/src/lib/pushers/content-pusher/content-pusher.ts b/src/lib/pushers/content-pusher/content-pusher.ts index 40f4769..1c2c076 100644 --- a/src/lib/pushers/content-pusher/content-pusher.ts +++ b/src/lib/pushers/content-pusher/content-pusher.ts @@ -9,6 +9,7 @@ import { ModelMapper } from "lib/mappers/model-mapper"; import { ContentItem, Model } from "@agility/management-sdk"; import { ContainerMapper } from "lib/mappers/container-mapper"; import { getApiClient } from "core/state"; +import { reportForeignCdnReferences } from "./util/report-foreign-cdn-references"; /** * Push content to the target instance @@ -40,6 +41,19 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content logger: logger as any, }); + // PROD-2773: warn about asset URLs into a CDN container owned by neither instance. Reporting + // only — runs before change detection (and in --preflight) and never alters an item. Items + // skipped above for missing container/model mappings are not synced, so they are not scanned. + try { + reportForeignCdnReferences([...linkedContentItems, ...normalContentItems], locale, { + sourceGuid: sourceGuidStr, + targetGuid: targetGuidStr, + logger, + }); + } catch (scanError: any) { + logger?.fileOnly?.(`Foreign CDN reference scan failed: ${scanError?.message || scanError}`); + } + let totalSuccessful = 0; let totalFailed = 0; let totalSkipped = 0; diff --git a/src/lib/pushers/content-pusher/util/report-foreign-cdn-references.ts b/src/lib/pushers/content-pusher/util/report-foreign-cdn-references.ts new file mode 100644 index 0000000..ef420b0 --- /dev/null +++ b/src/lib/pushers/content-pusher/util/report-foreign-cdn-references.ts @@ -0,0 +1,74 @@ +import { ContentItem } from "@agility/management-sdk"; +import { AssetMapper } from "lib/mappers/asset-mapper"; +import { fileOperations } from "core/fileOperations"; +import { getAssetsFromFileSystem } from "lib/getters/filesystem/get-assets"; +import { buildCdnOwnership, collectForeignCdnReferences, ForeignCdnDetector } from "lib/content/foreign-cdn-detector"; +import { foreignCdnReport } from "lib/content/foreign-cdn-report"; + +// One detector per source/target pair per process: building it reads every pulled asset file. +// Asset mappings created later in the same run only add containers already known from the pulled +// assets, so reusing it across the content phases of every locale is safe. +let cachedDetector: { key: string; detector: ForeignCdnDetector } | null = null; + +export function resetForeignCdnDetectorCache(): void { + cachedDetector = null; +} + +export function getForeignCdnDetector(sourceGuid: string, targetGuid: string): ForeignCdnDetector { + const key = `${sourceGuid}->${targetGuid}`; + if (cachedDetector?.key === key) return cachedDetector.detector; + + const read = (fn: () => T[]): T[] => { + try { + return fn() || []; + } catch { + return []; // best-effort: an unreadable side just stays unknown (no warnings) + } + }; + const assetMappings = read(() => [...new AssetMapper(sourceGuid, targetGuid).getMappings()]); + const sourceAssets = read(() => getAssetsFromFileSystem(new fileOperations(sourceGuid))); + const targetAssets = read(() => getAssetsFromFileSystem(new fileOperations(targetGuid))); + + const detector = new ForeignCdnDetector(buildCdnOwnership({ assetMappings, sourceAssets, targetAssets })); + cachedDetector = { key, detector }; + return detector; +} + +/** + * PROD-2773: scan the content items of one locale for foreign CDN references and record them in + * foreignCdnReport. Runs in both real and --preflight runs, before change detection, so an + * unchanged item still carrying a foreign URL is reported too. Reporting only — the items are not + * touched. Logs a warning the first time each URL is seen; repeat sightings go to the log file only. + */ +export function reportForeignCdnReferences( + contentItems: ContentItem[], + locale: string, + opts: { sourceGuid: string; targetGuid: string; logger?: any; detector?: ForeignCdnDetector } +): number { + const detector = opts.detector ?? getForeignCdnDetector(opts.sourceGuid, opts.targetGuid); + if (!detector.isEnabled()) return 0; + foreignCdnReport.setOwnContainers(detector.sourceContainers, detector.targetContainers); + + let found = 0; + for (const contentItem of contentItems) { + const refs = collectForeignCdnReferences(contentItem?.fields, detector); + for (const ref of refs) { + const item = { + contentID: contentItem.contentID, + referenceName: contentItem.properties?.referenceName ?? "unknown", + locale, + }; + const outcome = foreignCdnReport.record(ref, item); + if (outcome === "duplicate") continue; + found++; + const message = foreignCdnReport.formatOccurrence(ref, item); + if (outcome === "new-url") { + if (opts.logger?.warning) opts.logger.warning(message); + else console.warn(message); + } else { + opts.logger?.fileOnly?.(message); + } + } + } + return found; +} diff --git a/src/lib/pushers/content-pusher/util/tests/report-foreign-cdn-references.test.ts b/src/lib/pushers/content-pusher/util/tests/report-foreign-cdn-references.test.ts new file mode 100644 index 0000000..4044157 --- /dev/null +++ b/src/lib/pushers/content-pusher/util/tests/report-foreign-cdn-references.test.ts @@ -0,0 +1,78 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { resetState, setState } from "core/state"; +import { foreignCdnReport } from "lib/content/foreign-cdn-report"; +import { + getForeignCdnDetector, + reportForeignCdnReferences, + resetForeignCdnDetectorCache, +} from "../report-foreign-cdn-references"; + +let tmpDir: string; + +function writeJson(rel: string, data: any) { + const file = path.join(tmpDir, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); +} + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agility-foreign-cdn-")); + resetState(); + setState({ rootPath: tmpDir }); + resetForeignCdnDetectorCache(); + foreignCdnReport.reset(); + jest.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + jest.restoreAllMocks(); +}); + +describe("getForeignCdnDetector", () => { + it("learns the source container from asset mappings and the target's from its pulled assets", () => { + writeJson("mappings/src-u-tgt-u/assets/mappings.json", [ + { + sourceMediaID: 1, + targetMediaID: 2, + sourceContainerEdgeUrl: "https://cdn.example-media.com/src-container", + sourceContainerOriginUrl: "https://agility-cdn-usa2.azurewebsites.net/src-container", + }, + ]); + writeJson("tgt-u/assets/json/1.json", { + assetMedias: [{ mediaID: 2, containerEdgeUrl: "https://cdn-usa2.aglty.io/tgt-container" }], + }); + + const detector = getForeignCdnDetector("src-u", "tgt-u"); + expect(detector.isEnabled()).toBe(true); + expect(detector.sourceContainers).toEqual(["src-container"]); + expect(detector.targetContainers).toEqual(["tgt-container"]); + // the custom edge host is learned, so a third instance's container on it is recognised + expect(detector.findForeignUrls("https://cdn.example-media.com/other/a.png").map((m) => m.container)).toEqual([ + "other", + ]); + }); + + it("is disabled when the target has no assets and no mappings yet", () => { + writeJson("src-u/assets/json/1.json", { + assetMedias: [{ mediaID: 1, containerEdgeUrl: "https://cdn-usa2.aglty.io/src-container" }], + }); + const detector = getForeignCdnDetector("src-u", "tgt-u"); + expect(detector.isEnabled()).toBe(false); + + reportForeignCdnReferences( + [ + { + contentID: 1, + properties: { referenceName: "x" }, + fields: { img: { url: "https://cdn.aglty.io/other/a.png" } }, + } as any, + ], + "en-us", + { sourceGuid: "src-u", targetGuid: "tgt-u" } + ); + expect(foreignCdnReport.hasEntries()).toBe(false); + }); +});