From 785e667e8d237bd6f4229509c0aaa3a8d7f2a1cc Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 14 Oct 2026 14:14:10 -0400 Subject: [PATCH 1/3] PROD-2552: add `agility verify` to compare a synced target against its source PROD-2552: verify comparison library (mapping index, neutral form, checks, report) - mapping-index: Map-backed bidirectional indices over every mapping type, read-only - canonicalize: neutral comparison form (target IDs, linked refs, asset URL translation and host aliasing, null/empty/CRLF/case/numeric normalization); shares no code with the push transforms - checks: mapping integrity, models/containers/templates, content (fields, counts, list order), page fields and zones (section-ID or exact-name pairing, never positional), sitemap (parent, sibling order, missing) - verify-report: findings, tallies, exit codes, console table, JSON report - writeJsonSummary accepts any report object PROD-2552: add `agility verify` command Status guard (bounded 10-minute wait in preview and fetch mode, re-pull once if an update lands during the pull), full pull of both instances, per-locale checks, console + JSON report. Exit 0 pass, 1 differences, 2 could not verify. Accepts only existing flags via a dedicated verifyArgs set; never writes to an instance or to the mapping files. Also satisfies PROD-2549. PROD-2552: document the verify command in the README The required-field check (validate-required-fields.ts) is PROD-2542's (#234); this branch builds on it. Co-Authored-By: Claude Opus 5.5 --- README.md | 42 ++ src/core/auth.ts | 21 +- src/core/index.ts | 2 +- src/core/json-summary.ts | 5 +- src/core/logs.ts | 9 +- src/core/system-args.ts | 50 ++ src/core/tests/verify.test.ts | 433 +++++++++++++ src/core/verify.ts | 571 ++++++++++++++++++ src/index.ts | 20 + src/lib/verify/canonicalize.ts | 462 ++++++++++++++ src/lib/verify/check-content.ts | 266 ++++++++ src/lib/verify/check-mapping-integrity.ts | 299 +++++++++ src/lib/verify/check-page-zones.ts | 344 +++++++++++ src/lib/verify/check-sitemap.ts | 151 +++++ src/lib/verify/check-structure.ts | 138 +++++ src/lib/verify/mapping-index.ts | 213 +++++++ src/lib/verify/tests/canonicalize.test.ts | 227 +++++++ src/lib/verify/tests/check-content.test.ts | 308 ++++++++++ src/lib/verify/tests/check-page-zones.test.ts | 231 +++++++ src/lib/verify/tests/check-sitemap.test.ts | 144 +++++ src/lib/verify/tests/check-structure.test.ts | 220 +++++++ src/lib/verify/tests/fixtures.ts | 127 ++++ src/lib/verify/tests/mapping-index.test.ts | 152 +++++ src/lib/verify/tests/verify-report.test.ts | 233 +++++++ src/lib/verify/verify-context.ts | 334 ++++++++++ src/lib/verify/verify-report.ts | 394 ++++++++++++ src/tests/sync/verify-after-sync.test.ts | 171 ++++++ 27 files changed, 5561 insertions(+), 6 deletions(-) create mode 100644 src/core/tests/verify.test.ts create mode 100644 src/core/verify.ts create mode 100644 src/lib/verify/canonicalize.ts create mode 100644 src/lib/verify/check-content.ts create mode 100644 src/lib/verify/check-mapping-integrity.ts create mode 100644 src/lib/verify/check-page-zones.ts create mode 100644 src/lib/verify/check-sitemap.ts create mode 100644 src/lib/verify/check-structure.ts create mode 100644 src/lib/verify/mapping-index.ts create mode 100644 src/lib/verify/tests/canonicalize.test.ts create mode 100644 src/lib/verify/tests/check-content.test.ts create mode 100644 src/lib/verify/tests/check-page-zones.test.ts create mode 100644 src/lib/verify/tests/check-sitemap.test.ts create mode 100644 src/lib/verify/tests/check-structure.test.ts create mode 100644 src/lib/verify/tests/fixtures.ts create mode 100644 src/lib/verify/tests/mapping-index.test.ts create mode 100644 src/lib/verify/tests/verify-report.test.ts create mode 100644 src/lib/verify/verify-context.ts create mode 100644 src/lib/verify/verify-report.ts create mode 100644 src/tests/sync/verify-after-sync.test.ts diff --git a/README.md b/README.md index 622c55a..05a4ecb 100644 --- a/README.md +++ b/README.md @@ -517,6 +517,48 @@ agility reverse-sync --sourceGuid A --targetGuid B --locales en-us --preflight agility reverse-sync --sourceGuid A --targetGuid B --locales en-us ``` +### Verify + +Compare a synced target instance against its source and report every difference, independent of the sync's own logs and change tracking. + +```bash +agility verify --sourceGuid --targetGuid [--locales en-us,fr-ca] [--channel website] [--models ...] [--pages ...] [--containers ...] [--jsonSummary ] +``` + +Use the same `--sourceGuid` / `--targetGuid` as the forward `sync`. Verify never writes to either instance or to the mapping files. + +#### What it does + +1. Waits until neither instance has a content update in progress (preview and published), up to 10 minutes. +2. Runs a full pull of both instances, then checks again. If an update ran during the pull, it pulls once more. +3. Compares the two instances through the mapping files: + - mapping integrity (unmapped source items, mappings to missing targets, two sources mapped to one target) + - models, containers and templates + - content counts per model and container, list order, and field values (linked content, assets and companion fields resolved through the mappings) + - component order within each page zone, and which zone each component is in + - sitemap parent and sibling order per locale and channel + +#### Results + +- **Failures**: field mismatches, missing items, components in the wrong order or zone, sitemap order errors. +- **Warnings**: content that exists only on the target, publish-state differences. These don't fail the run. +- **Source defects**: differences explained by the source data, such as an item missing a required field or an item in a deleted container. These are listed but don't fail the run. + +The console prints a summary table per locale and the first 20 findings per category. The full report is written to `--jsonSummary`, or to `agility-files/logs/verify-{source}-{target}-{timestamp}.json`. + +| Exit code | Meaning | +| --------- | ----------------------------------------------------------------------------- | +| `0` | Target matches source | +| `1` | Differences found | +| `2` | Could not verify (auth, status check timeout, pull failure, missing mappings) | + +```bash +# Blue/green: sync into the new instance, then prove it before switching traffic +agility sync --sourceGuid A --targetGuid B --locales en-us --preflight +agility sync --sourceGuid A --targetGuid B --locales en-us +agility verify --sourceGuid A --targetGuid B --locales en-us +``` + ## File Structure The CLI organizes downloaded content in a structured format: diff --git a/src/core/auth.ts b/src/core/auth.ts index 5a7f6dc..977bc66 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -993,7 +993,7 @@ export class Auth { * Validate command-specific requirements and set up instance access * This should be called by each command after auth.init() */ - async validateCommand(commandType: "pull" | "sync" | "clean" | "interactive" | "push"): Promise { + async validateCommand(commandType: "pull" | "sync" | "clean" | "interactive" | "push" | "verify"): Promise { const missingFields: string[] = []; // Check command-specific requirements @@ -1030,6 +1030,20 @@ export class Auth { if (!state.channel) missingFields.push("channel (use --channel or AGILITY_WEBSITE in .env)"); break; + case "verify": + // Verify (PROD-2552) reads both instances and writes to neither + if (!state.sourceGuid) + missingFields.push("sourceGuid (use --sourceGuid or AGILITY_GUID in .env)"); + if (!state.targetGuid) + missingFields.push("targetGuid (use --targetGuid or AGILITY_TARGET_GUID in .env)"); + + const hasVerifyUserLocales = state.locale && state.locale.length > 0; + const hasVerifyAutoDetectedLocales = state.guidLocaleMap && state.guidLocaleMap.size > 0; + if (!hasVerifyUserLocales && !hasVerifyAutoDetectedLocales) { + missingFields.push("locale (use --locales or AGILITY_LOCALES in .env, or locales will be auto-detected)"); + } + break; + case "clean": // Clean needs minimal validation since it prompts for instance selection break; @@ -1078,6 +1092,11 @@ export class Auth { ); return false; } + } else if (commandType === "verify") { + // Verify only needs read access to both instances; the pull uses the API keys auth.init() + // already resolved for each guid. + await this.validateInstanceAccess(state.sourceGuid, "source"); + await this.validateInstanceAccess(state.targetGuid, "target"); } else if (commandType === "pull" && state.sourceGuid) { // Pull operation - validate source access and get API keys await this.validateInstanceAccess(state.sourceGuid, "instance"); diff --git a/src/core/index.ts b/src/core/index.ts index 74f318a..b633026 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -6,7 +6,7 @@ // Core authentication and state management export { Auth } from "./auth"; export { state, setState, resetState, primeFromEnv, getState, getUIMode, enableReverseSync } from "./state"; -export { systemArgs, workflowArgs, type SystemArgsType } from "./system-args"; +export { systemArgs, workflowArgs, verifyArgs, type SystemArgsType } from "./system-args"; export { normalizeProcessArgs, normalizeArgv } from "./arg-normalizer"; // Main operation services diff --git a/src/core/json-summary.ts b/src/core/json-summary.ts index d170bc3..13acf80 100644 --- a/src/core/json-summary.ts +++ b/src/core/json-summary.ts @@ -316,13 +316,14 @@ export function buildJsonSummary(input: BuildJsonSummaryInput): JsonSummary { } /** - * Write the summary to disk, creating parent directories as needed. + * Write the summary to disk, creating parent directories as needed. Also used for the + * `agility verify` report (PROD-2552), which has its own schema but the same write contract. * * Never throws. A reporting artifact must not be able to fail a sync that otherwise * succeeded — an unwritable path is worth a warning, not a non-zero exit. Returns whether * the write landed so the caller can say so. */ -export function writeJsonSummary(filePath: string, summary: JsonSummary): { written: boolean; error?: string } { +export function writeJsonSummary(filePath: string, summary: JsonSummary | object): { written: boolean; error?: string } { try { const resolved = path.resolve(filePath); const dir = path.dirname(resolved); diff --git a/src/core/logs.ts b/src/core/logs.ts index d099cce..1103cea 100644 --- a/src/core/logs.ts +++ b/src/core/logs.ts @@ -69,7 +69,7 @@ function safeStringify(obj: any, indent?: number): string { } } -export type OperationType = "pull" | "push" | "sync" | "reverse-sync"; +export type OperationType = "pull" | "push" | "sync" | "reverse-sync" | "verify"; export type EntityType = | "model" @@ -524,7 +524,12 @@ export class Logs { } // Build filename with GUID - if (this.operationType === "push" || this.operationType === "sync" || this.operationType === "reverse-sync") { + if ( + this.operationType === "push" || + this.operationType === "sync" || + this.operationType === "reverse-sync" || + this.operationType === "verify" + ) { const sourceGuid = state.sourceGuid || "unknown"; const targetGuid = state.targetGuid || "unknown"; filename = `${sourceGuid}-${targetGuid}-${this.operationType}-${timestamp}.txt`; diff --git a/src/core/system-args.ts b/src/core/system-args.ts index 288df5e..69aba1a 100644 --- a/src/core/system-args.ts +++ b/src/core/system-args.ts @@ -200,6 +200,56 @@ export const workflowArgs = { }, }; +/** + * Args accepted by the `verify` command (PROD-2552). + * + * A deliberate subset of systemArgs, reusing their parsing and aliases — NOT a spread of the whole + * set: verify never writes to an instance, so the sync-only options (overwrite, autoPublish, + * preflight, fullPull, elements, modelsWithDeps) would advertise behavior it does not have. Verify + * always does a full pull of both instances. + */ +export const verifyArgs = { + token: systemArgs.token, + dev: systemArgs.dev, + headless: systemArgs.headless, + verbose: systemArgs.verbose, + sourceGuid: { + ...systemArgs.sourceGuid, + describe: + "The source instance GUID of the sync being verified. Required; falls back to AGILITY_GUID from .env.", + }, + targetGuid: { + ...systemArgs.targetGuid, + describe: + "The target instance GUID of the sync being verified. Required; falls back to AGILITY_TARGET_GUID from .env.", + }, + locales: systemArgs.locales, + channel: { + ...systemArgs.channel, + describe: "Verify only this channel's sitemap. If not provided, every channel is verified.", + default: undefined, + }, + models: { + ...systemArgs.models, + describe: "Verify only these models (comma-separated reference names), as `sync --models` scopes a sync.", + }, + pages: { + ...systemArgs.pages, + describe: + "Verify only these pages and what they need (comma-separated paths, names or IDs), as `sync --pages` scopes a sync.", + }, + containers: { + ...systemArgs.containers, + describe: + "Verify only these containers and what they depend on (comma-separated reference names, titles or IDs), as `sync --containers` scopes a sync.", + }, + jsonSummary: { + ...systemArgs.jsonSummary, + describe: + "Write the full verify report (every finding, per-locale counts, timings) as JSON to this path. Defaults to agility-files/logs/verify---.json.", + }, +}; + /** * Type helper for command arguments that include system args */ diff --git a/src/core/tests/verify.test.ts b/src/core/tests/verify.test.ts new file mode 100644 index 0000000..84258de --- /dev/null +++ b/src/core/tests/verify.test.ts @@ -0,0 +1,433 @@ +/** + * PROD-2552: `agility verify` — the run around the checks: status guard, pull, exit codes, and + * how the command primes state. The checks themselves are covered in src/lib/verify/tests. + */ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { changedSnapshots, runVerifyCommand, STATUS_TIMEOUT_MS, Verify, VerifyDeps, VerifyOptions } from "../verify"; +import { resetState, setState, state } from "../state"; +import { EXIT_CANNOT_VERIFY, EXIT_DIFFERENCES, EXIT_PASS, VerifyReport } from "../../lib/verify/verify-report"; +import { emptySide } from "../../lib/verify/tests/fixtures"; +import { SideEntities } from "../../lib/verify/verify-context"; + +const SRC = "src-1"; +const TGT = "tgt-1"; +const LOCALE = "en-us"; + +let root: string; + +function writeJson(file: string, data: any): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); +} + +/** A cache + mapping tree on disk, so the "can we verify at all" preconditions pass. */ +function seedRoot(): void { + writeJson(path.join(root, SRC, "models", "1.json"), { id: 1, referenceName: "Post", fields: [] }); + writeJson(path.join(root, TGT, "models", "11.json"), { id: 11, referenceName: "Post", fields: [] }); + writeJson(path.join(root, "mappings", `${SRC}-${TGT}`, "models", "mappings.json"), [ + { sourceID: 1, targetID: 11, sourceReferenceName: "Post", targetReferenceName: "Post" }, + ]); + writeJson(path.join(root, "mappings", `${SRC}-${TGT}`, LOCALE, "item", "mappings.json"), [ + { sourceContentID: 5, targetContentID: 55 }, + ]); +} + +const POST = { id: 1, referenceName: "Post", fields: [{ name: "Title", type: "Text" }] }; +const POST_T = { ...POST, id: 11 }; +const item = (contentID: number, title: string) => ({ + contentID, + properties: { state: 2, referenceName: "posts", definitionName: "Post", itemOrder: 0, versionID: 1 }, + fields: { title }, +}); + +function sides(targetTitle: string): Record { + return { + [SRC]: emptySide({ models: [POST], content: [item(5, "Hello")] }), + [TGT]: emptySide({ models: [POST_T], content: [item(55, targetTitle)] }), + }; +} + +function options(overrides: Partial = {}): VerifyOptions { + return { + sourceGuid: SRC, + targetGuid: TGT, + locales: [LOCALE], + models: "", + pages: "", + containers: "", + jsonSummary: path.join(root, "report.json"), + rootPath: root, + ...overrides, + }; +} + +const idle = { inProgress: false, lastContentVersionID: 10, maxChangeDate: "2026-10-01T00:00:00Z" } as any; +const busy = { ...idle, inProgress: true }; +const okPull = async () => ({ + success: true, + results: [ + { guidProcessed: SRC, failed: [] }, + { guidProcessed: TGT, failed: [] }, + ], +}); + +/** Deps with a fake clock that sleep() advances, a no-op pull, and in-memory sides. */ +function deps(overrides: Partial = {}, targetTitle = "Hello"): VerifyDeps & { clock: { t: number } } { + const clock = { t: Date.parse("2026-10-07T00:00:00Z") }; + const data = sides(targetTitle); + return { + clock, + getStatus: async () => idle, + pull: okPull, + sleep: async (ms: number) => { + clock.t += ms; + }, + now: () => clock.t, + log: () => {}, + loadSide: async (guid: string) => data[guid], + statusTimeoutMs: STATUS_TIMEOUT_MS, + statusPollMs: 5000, + print: false, + ...overrides, + }; +} + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "verify-core-")); + resetState(); + setState({ rootPath: root, sourceGuid: SRC, targetGuid: TGT }); + seedRoot(); +}); + +afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + jest.restoreAllMocks(); +}); + +describe("exit codes", () => { + it("0 when the target matches", async () => { + const report = await new Verify(options(), deps()).run(); + expect(report.exitCode).toBe(EXIT_PASS); + expect(report.verdict).toBe("pass"); + }); + + it("1 when the target differs", async () => { + const report = await new Verify(options(), deps({}, "Changed")).run(); + expect(report.exitCode).toBe(EXIT_DIFFERENCES); + expect(report.findings).toEqual([expect.objectContaining({ category: "fieldMismatch", path: "title" })]); + }); + + it("writes the JSON report to --jsonSummary", async () => { + await new Verify(options(), deps()).run(); + const written = JSON.parse(fs.readFileSync(path.join(root, "report.json"), "utf8")); + expect(written).toMatchObject({ command: "verify", exitCode: 0, source: SRC, target: TGT }); + expect(written.run.status.before).toHaveLength(4); + }); + + it("writes to agility-files/logs/verify-*.json without --jsonSummary", async () => { + await new Verify(options({ jsonSummary: "" }), deps()).run(); + const logs = fs.readdirSync(path.join(root, "logs")); + expect(logs).toEqual([expect.stringMatching(new RegExp(`^verify-${SRC}-${TGT}-.*\\.json$`))]); + }); + + it("2 when the pair has no mapping files", async () => { + fs.rmSync(path.join(root, "mappings"), { recursive: true }); + const report = await new Verify(options(), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/No mapping files/); + }); + + it("2 when a requested locale has no mappings", async () => { + const report = await new Verify(options({ locales: [LOCALE, "fr-ca"] }), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/fr-ca/); + }); + + it("2 when the pulled cache is unreadable", async () => { + fs.writeFileSync(path.join(root, TGT, "models", "12.json"), "{ truncated"); + const report = await new Verify(options(), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/unreadable/); + }); + + it("2 when scope flags are combined", async () => { + const report = await new Verify(options({ models: "Post", pages: "/home" }), deps()).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + }); + + it("never writes to the mapping files", async () => { + const dir = path.join(root, "mappings"); + const snapshot = () => JSON.stringify(fs.readdirSync(dir, { recursive: true } as any).sort()); + const before = snapshot(); + // A type folder without its mappings.json — fileOperations.getMappingFile would seed one. + fs.mkdirSync(path.join(dir, `${SRC}-${TGT}`, "containers")); + await new Verify(options(), deps()).run(); + expect(snapshot()).toBe(JSON.stringify([...JSON.parse(before), `${SRC}-${TGT}/containers`].sort())); + }); +}); + +describe("status guard", () => { + it("checks both instances in preview and fetch mode before and after the pull", async () => { + const calls: string[] = []; + await new Verify( + options(), + deps({ + getStatus: async (guid, mode) => { + calls.push(`${guid}:${mode}`); + return idle; + }, + }) + ).run(); + const once = [`${SRC}:preview`, `${SRC}:fetch`, `${TGT}:preview`, `${TGT}:fetch`]; + expect(calls).toEqual([...once, ...once]); + }); + + it("waits for an in-progress update to finish", async () => { + let checks = 0; + const d = deps({ + getStatus: async (guid, mode) => (guid === TGT && mode === "fetch" && ++checks <= 3 ? busy : idle), + }); + const sleep = jest.spyOn(d, "sleep"); + const report = await new Verify(options(), d).run(); + expect(report.exitCode).toBe(EXIT_PASS); + expect(sleep).toHaveBeenCalledTimes(3); + }); + + it("gives up after the timeout with exit 2", async () => { + const pull = jest.fn(okPull); + const report = await new Verify(options(), deps({ getStatus: async () => busy, pull })).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/Timed out after 10 minute/); + expect(pull).not.toHaveBeenCalled(); + }); + + it("treats a status error as exit 2 (it is not swallowed)", async () => { + const report = await new Verify( + options(), + deps({ + getStatus: async () => { + throw new Error("503 Service Unavailable"); + }, + }) + ).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/503/); + }); + + it("re-pulls once when an update lands during the pull", async () => { + let version = 10; + const pull = jest.fn(async () => { + if (pull.mock.calls.length === 1) version = 11; // an update lands during the first pull only + return okPull(); + }); + const report = await new Verify( + options(), + deps({ getStatus: async () => ({ ...idle, lastContentVersionID: version }), pull }) + ).run(); + expect(pull).toHaveBeenCalledTimes(2); + expect(report.exitCode).toBe(EXIT_PASS); + expect(report.run.pull.attempts).toBe(2); + }); + + it("exits 2 when an update lands during the re-pull too", async () => { + let version = 10; + const pull = jest.fn(async () => { + version++; + return okPull(); + }); + const report = await new Verify( + options(), + deps({ getStatus: async () => ({ ...idle, lastContentVersionID: version }), pull }) + ).run(); + expect(pull).toHaveBeenCalledTimes(2); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/update ran during the pull/); + }); + + it("detects a change by maxChangeDate or a new in-progress update", () => { + const base = { + guid: "g", + mode: "fetch", + checkedAt: "", + inProgress: false, + lastContentVersionID: 1, + maxChangeDate: "a", + }; + expect(changedSnapshots([base], [{ ...base }])).toEqual([]); + expect(changedSnapshots([base], [{ ...base, maxChangeDate: "b" }])).toHaveLength(1); + expect(changedSnapshots([base], [{ ...base, inProgress: true }])).toHaveLength(1); + }); +}); + +describe("pull", () => { + it("does a full pull with every element and no scope, then restores state", async () => { + setState({ models: "Post", fullPull: false }); + let during: any; + await new Verify( + options({ models: "Post" }), + deps({ + pull: async () => { + during = { fullPull: state.fullPull, models: state.models, elements: state.elements }; + return okPull(); + }, + }) + ).run(); + expect(during.fullPull).toBe(true); + expect(during.models).toBe(""); + expect(during.elements).toContain("Content"); + expect(state.fullPull).toBe(false); + expect(state.models).toBe("Post"); + }); + + it.each([ + ["throws", async () => Promise.reject(new Error("network down"))], + ["reports failure", async () => ({ success: false, results: [] })], + [ + "has a failed operation", + async () => ({ + success: true, + results: [ + { guidProcessed: SRC, failed: [{ operation: "downloadAllModels", error: "boom" }] }, + { guidProcessed: TGT, failed: [] }, + ], + }), + ], + ["misses an instance", async () => ({ success: true, results: [{ guidProcessed: SRC, failed: [] }] })], + ])("exits 2 when the pull %s", async (_label, pull) => { + const report = await new Verify(options(), deps({ pull: pull as any })).run(); + expect(report.exitCode).toBe(EXIT_CANNOT_VERIFY); + expect(report.error).toMatch(/Pull failed/); + }); +}); + +describe("runVerifyCommand", () => { + let cwd: string; + + beforeEach(() => { + cwd = process.cwd(); + process.chdir(root); + }); + + afterEach(() => process.chdir(cwd)); + + const passingReport = { exitCode: 0 } as VerifyReport; + const okAuth = () => ({ init: async () => true, validateCommand: async () => true }); + + it("primes state from argv and hands the verify options over", async () => { + let received: VerifyOptions | undefined; + const code = await runVerifyCommand( + { + sourceGuid: SRC, + targetGuid: TGT, + locales: "en-us,fr-ca", + models: "Post", + jsonSummary: "out.json", + channel: "Mobile", + }, + { + createAuth: okAuth, + createVerify: (opts) => { + received = opts; + return { run: async () => passingReport }; + }, + log: () => {}, + } + ); + expect(code).toBe(0); + expect(received).toEqual({ + sourceGuid: SRC, + targetGuid: TGT, + locales: ["en-us", "fr-ca"], + channel: "Mobile", + models: "Post", + pages: "", + containers: "", + jsonSummary: "out.json", + rootPath: "agility-files", + }); + }); + + it("leaves the channel unset when --channel is not passed (every channel is verified)", async () => { + let received: VerifyOptions | undefined; + await runVerifyCommand( + { sourceGuid: SRC, targetGuid: TGT, locales: "en-us" }, + { + createAuth: okAuth, + createVerify: (opts) => { + received = opts; + return { run: async () => passingReport }; + }, + log: () => {}, + } + ); + expect(received?.channel).toBeUndefined(); + }); + + it("falls back to AGILITY_GUID / AGILITY_TARGET_GUID from .env", async () => { + fs.writeFileSync(path.join(root, ".env"), `AGILITY_GUID=${SRC}\nAGILITY_TARGET_GUID=${TGT}\n`); + let received: VerifyOptions | undefined; + await runVerifyCommand( + { locales: "en-us" }, + { + createAuth: okAuth, + createVerify: (opts) => { + received = opts; + return { run: async () => passingReport }; + }, + log: () => {}, + } + ); + expect(received).toMatchObject({ sourceGuid: SRC, targetGuid: TGT }); + }); + + it("returns the report's exit code", async () => { + const code = await runVerifyCommand( + { sourceGuid: SRC, targetGuid: TGT, locales: "en-us" }, + { + createAuth: okAuth, + createVerify: () => ({ run: async () => ({ exitCode: 1 }) as VerifyReport }), + log: () => {}, + } + ); + expect(code).toBe(1); + }); + + it.each([ + [ + "an authentication error", + { sourceGuid: SRC, targetGuid: TGT }, + () => ({ + init: async () => { + throw new Error("ECONNRESET"); + }, + validateCommand: async () => true, + }), + ], + ["missing guids", {}, okAuth], + ["identical guids", { sourceGuid: SRC, targetGuid: SRC }, okAuth], + [ + "failed authentication", + { sourceGuid: SRC, targetGuid: TGT }, + () => ({ init: async () => false, validateCommand: async () => true }), + ], + [ + "failed validation", + { sourceGuid: SRC, targetGuid: TGT }, + () => ({ init: async () => true, validateCommand: async () => false }), + ], + ])("exits 2 with a report on %s", async (_label, argv: any, createAuth: any) => { + const createVerify = jest.fn(); + const code = await runVerifyCommand( + { ...argv, jsonSummary: "abort.json" }, + { createAuth, createVerify, log: () => {} } + ); + expect(code).toBe(EXIT_CANNOT_VERIFY); + expect(createVerify).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(path.join(root, "abort.json"), "utf8"))).toMatchObject({ + verdict: "error", + exitCode: 2, + }); + }); +}); diff --git a/src/core/verify.ts b/src/core/verify.ts new file mode 100644 index 0000000..5154dbc --- /dev/null +++ b/src/core/verify.ts @@ -0,0 +1,571 @@ +/** + * `agility verify` (PROD-2552, PROD-2549): prove a migrated target matches its source. + * + * A sync reports on its own bookkeeping; a customer had syncs report success while + * the components on their pages were in the wrong order. Verify is the independent check for a + * blue/green cutover: it pulls both instances fresh, then compares them in a neutral form (see + * lib/verify/canonicalize.ts) that shares no code with the push transforms. + * + * Flow: + * 1. Status guard — wait (bounded, 10 minutes) until neither instance has a Fetch API update in + * progress, in both preview and fetch modes. A status error or timeout is exit 2. + * 2. Full pull of both instances. A pull failure is exit 2. Mapping files are never written. + * 3. Re-check status. If an update started or landed during the pull, pull once more; if it + * happens again, exit 2 ("an update ran during the pull, re-run"). + * 4. Per-locale checks, aggregate, console report + JSON report, exit code. + * + * Exit codes: 0 pass, 1 differences found, 2 could not verify. + */ +import * as fs from "fs"; +import ansiColors from "ansi-colors"; +import { Auth } from "./auth"; +import { Pull } from "./pull"; +import { state, setState, resetState, primeFromEnv, getPageCmsLink, getContentCmsLink } from "./state"; +import { normalizeArgv } from "./arg-normalizer"; +import { FetchApiStatus, FetchApiSyncMode, getFetchApiStatus } from "../lib/shared/get-fetch-api-status"; +import { sleep as defaultSleep } from "../lib/shared/sleep"; +import { ModelFilterOptions } from "../lib/pushers/guid-data-loader"; +import { resolvePageSyncScope } from "../lib/pages/resolve-page-sync-scope"; +import { parsePageSelectors } from "../lib/pages/page-scope"; +import { resolveContainerSyncScope } from "../lib/containers/resolve-container-sync-scope"; +import { parseContainerSelectors } from "../lib/containers/container-scope"; +import { + buildMappingIndex, + createFileMappingReader, + mappingPairDirectory, + MappingIndex, +} from "../lib/verify/mapping-index"; +import { + createVerifyContext, + findUnreadableCacheFiles, + loadSideEntities, + SideEntities, +} from "../lib/verify/verify-context"; +import { checkInstanceIntegrity, checkLocaleIntegrity } from "../lib/verify/check-mapping-integrity"; +import { checkStructure } from "../lib/verify/check-structure"; +import { checkContent } from "../lib/verify/check-content"; +import { checkPages } from "../lib/verify/check-page-zones"; +import { checkSitemap } from "../lib/verify/check-sitemap"; +import { + buildErrorReport, + buildVerifyReport, + defaultReportPath, + emptyRunInfo, + EXIT_CANNOT_VERIFY, + Finding, + FindingCollector, + printVerifyReport, + StatusSnapshot, + VerifyOptionsSummary, + VerifyReport, + VerifyRunInfo, + writeVerifyReport, +} from "../lib/verify/verify-report"; +import * as path from "path"; + +/** How long the status guard waits for an in-progress update before giving up. */ +export const STATUS_TIMEOUT_MS = 10 * 60 * 1000; +/** How often the status guard polls while an update is in progress. */ +export const STATUS_POLL_MS = 5000; + +const STATUS_MODES: FetchApiSyncMode[] = ["preview", "fetch"]; + +/** A condition under which verify cannot produce a verdict — always exit 2. */ +export class VerifyAbort extends Error {} + +export interface VerifyOptions { + sourceGuid: string; + targetGuid: string; + locales: string[]; + /** Restrict sitemap checks to this channel; undefined = every channel. */ + channel?: string; + models: string; + pages: string; + containers: string; + /** Report path; empty = agility-files/logs/verify-{src}-{tgt}-{ts}.json. */ + jsonSummary: string; + rootPath: string; +} + +export interface VerifyDeps { + getStatus: (guid: string, mode: FetchApiSyncMode) => Promise; + /** Full pull of state.sourceGuid and state.targetGuid. */ + pull: () => Promise<{ success: boolean; results: any[] }>; + sleep: (ms: number) => Promise; + now: () => number; + log: (line: string) => void; + /** Load one side of one locale from the pulled cache. */ + loadSide: (guid: string, locale: string, filter?: ModelFilterOptions) => Promise; + statusTimeoutMs: number; + statusPollMs: number; + /** Print the console report (default true). */ + print: boolean; +} + +const defaultDeps: VerifyDeps = { + getStatus: (guid, mode) => getFetchApiStatus(guid, mode, false), + pull: () => new Pull().pullInstances(true), + sleep: (ms) => defaultSleep(ms), + now: () => Date.now(), + log: (line) => console.log(line), + loadSide: (guid, locale, filter) => loadSideEntities(guid, locale, filter), + statusTimeoutMs: STATUS_TIMEOUT_MS, + statusPollMs: STATUS_POLL_MS, + print: true, +}; + +function snapshot(guid: string, mode: string, status: FetchApiStatus, at: number): StatusSnapshot { + return { + guid, + mode, + checkedAt: new Date(at).toISOString(), + inProgress: status?.inProgress === true, + lastContentVersionID: status?.lastContentVersionID ?? null, + maxChangeDate: status?.maxChangeDate ?? null, + }; +} + +/** Snapshots whose content version or change date moved, or that are now in progress. */ +export function changedSnapshots(before: StatusSnapshot[], after: StatusSnapshot[]): StatusSnapshot[] { + return after.filter((a) => { + const b = before.find((x) => x.guid === a.guid && x.mode === a.mode); + return ( + a.inProgress || + !b || + b.lastContentVersionID !== a.lastContentVersionID || + (b.maxChangeDate ?? null) !== (a.maxChangeDate ?? null) + ); + }); +} + +export class Verify { + private readonly options: VerifyOptions; + private readonly deps: VerifyDeps; + + constructor(options: VerifyOptions, deps: Partial = {}) { + this.options = options; + this.deps = { ...defaultDeps, ...deps }; + } + + get optionsSummary(): VerifyOptionsSummary { + return { + locales: this.options.locales.slice(), + channel: this.options.channel ?? "", + models: this.options.models, + pages: this.options.pages, + containers: this.options.containers, + }; + } + + /** Run the whole command. Never throws and never exits the process: the exit code is on the report. */ + async run(): Promise { + const startedAt = new Date(this.deps.now()); + const run = emptyRunInfo(startedAt); + let report: VerifyReport; + + try { + await this.guardedPull(run); + const compareStarted = this.deps.now(); + const collector = await this.compare(); + run.compareElapsedMs = this.deps.now() - compareStarted; + this.finishRun(run, startedAt); + report = buildVerifyReport({ + source: this.options.sourceGuid, + target: this.options.targetGuid, + options: this.optionsSummary, + collector, + run, + }); + } catch (error: any) { + this.finishRun(run, startedAt); + const message = + error instanceof VerifyAbort ? error.message : `Unexpected error while verifying: ${error?.message || error}`; + report = buildErrorReport({ + source: this.options.sourceGuid, + target: this.options.targetGuid, + options: this.optionsSummary, + error: message, + run, + }); + } + + this.emit(report, startedAt); + return report; + } + + private finishRun(run: VerifyRunInfo, startedAt: Date): void { + const finished = this.deps.now(); + run.finishedAt = new Date(finished).toISOString(); + run.elapsedMs = finished - startedAt.getTime(); + } + + private emit(report: VerifyReport, startedAt: Date): void { + const { sourceGuid, targetGuid } = this.options; + if (this.deps.print) { + printVerifyReport(report, { + log: this.deps.log, + linkFor: (f) => cmsLinkFor(f, sourceGuid, targetGuid), + }); + const pull = report.run.pull; + if (pull.attempts > 0) { + this.deps.log( + ansiColors.gray( + `Pull: ${pull.attempts} attempt(s), ${pull.elapsedMs.map((ms) => `${(ms / 1000).toFixed(1)}s`).join(" + ")}; ` + + `compare: ${(report.run.compareElapsedMs / 1000).toFixed(1)}s; total: ${(report.run.elapsedMs / 1000).toFixed(1)}s` + ) + ); + } + } + const reportPath = + this.options.jsonSummary || defaultReportPath(this.options.rootPath, sourceGuid, targetGuid, startedAt); + const written = writeVerifyReport(reportPath, report); + if (this.deps.print) { + this.deps.log( + written.written + ? ansiColors.gray(`Report: ${path.resolve(reportPath)}`) + : ansiColors.yellow(`⚠️ Could not write the verify report to ${reportPath}: ${written.error}`) + ); + } + } + + // ─── status guard + pull ───────────────────────────────────────────────── + + private guids(): string[] { + return [this.options.sourceGuid, this.options.targetGuid]; + } + + private async checkStatus(guid: string, mode: FetchApiSyncMode): Promise { + try { + const status = await this.deps.getStatus(guid, mode); + if (!status || typeof status !== "object") throw new Error("empty response"); + return status; + } catch (error: any) { + throw new VerifyAbort(`Could not check the Fetch API status of ${guid} (${mode}): ${error?.message || error}`); + } + } + + /** Wait until no instance has an update in progress, in either mode. Bounded by statusTimeoutMs. */ + async waitForIdle(): Promise { + const deadline = this.deps.now() + this.deps.statusTimeoutMs; + const snapshots: StatusSnapshot[] = []; + for (const guid of this.guids()) { + for (const mode of STATUS_MODES) { + let status = await this.checkStatus(guid, mode); + let announced = false; + while (status.inProgress) { + if (this.deps.now() >= deadline) { + throw new VerifyAbort( + `Timed out after ${Math.round(this.deps.statusTimeoutMs / 60000)} minute(s) waiting for an in-progress ` + + `Fetch API update on ${guid} (${mode}) to finish` + ); + } + if (!announced) { + this.deps.log( + ansiColors.gray(`⏳ ${guid} (${mode}): an update is in progress, waiting for it to finish...`) + ); + announced = true; + } + await this.deps.sleep(this.deps.statusPollMs); + status = await this.checkStatus(guid, mode); + } + snapshots.push(snapshot(guid, mode, status, this.deps.now())); + } + } + return snapshots; + } + + private async currentStatus(): Promise { + const snapshots: StatusSnapshot[] = []; + for (const guid of this.guids()) { + for (const mode of STATUS_MODES) { + snapshots.push(snapshot(guid, mode, await this.checkStatus(guid, mode), this.deps.now())); + } + } + return snapshots; + } + + async guardedPull(run: VerifyRunInfo): Promise { + for (let attempt = 1; attempt <= 2; attempt++) { + const before = await this.waitForIdle(); + run.status.before = before; + + const pullStarted = this.deps.now(); + await this.pullOnce(); + run.pull.attempts = attempt; + run.pull.elapsedMs.push(this.deps.now() - pullStarted); + + const after = await this.currentStatus(); + run.status.after = after; + + const changed = changedSnapshots(before, after); + if (changed.length === 0) return; + + const which = changed.map((s) => `${s.guid} (${s.mode})`).join(", "); + if (attempt === 2) { + throw new VerifyAbort( + `An update ran during the pull again (${which}); re-run verify once the instances are idle` + ); + } + this.deps.log(ansiColors.yellow(`⚠️ An update ran during the pull (${which}); pulling once more`)); + } + } + + /** + * One full pull of both instances. The scope flags are cleared for the pull's duration: a scoped + * verify still needs complete data (the `--models` filter would otherwise pull models only). + */ + private async pullOnce(): Promise { + const saved = { + fullPull: state.fullPull, + models: state.models, + modelsWithDeps: state.modelsWithDeps, + pages: state.pages, + containers: state.containers, + elements: state.elements, + }; + state.fullPull = true; + state.models = ""; + state.modelsWithDeps = ""; + state.pages = ""; + state.containers = ""; + state.elements = "Models,Galleries,Assets,Containers,Content,Templates,Pages,Sitemaps,UrlRedirections"; + + let result: { success: boolean; results: any[] }; + try { + this.deps.log(ansiColors.cyan(`⬇️ Full pull of ${this.guids().join(" and ")}...`)); + result = await this.deps.pull(); + } catch (error: any) { + throw new VerifyAbort(`Pull failed: ${error?.message || error}`); + } finally { + Object.assign(state, saved); + } + + const results = Array.isArray(result?.results) ? result.results : []; + const pulled = new Set(results.map((r: any) => r?.guidProcessed).filter(Boolean)); + const missing = this.guids().filter((g) => !pulled.has(g)); + const failures = results.flatMap((r: any) => + (r?.failed || []).map((f: any) => `${r.guidProcessed}: ${f.operation} — ${f.error}`) + ); + if (!result?.success || failures.length > 0 || missing.length > 0) { + const detail = [...failures, ...missing.map((g) => `${g}: no pull result`)].join("; "); + throw new VerifyAbort(`Pull failed${detail ? `: ${detail}` : ""}`); + } + } + + // ─── comparison ────────────────────────────────────────────────────────── + + private resolveFilter(): Promise | ModelFilterOptions | undefined { + const { sourceGuid, targetGuid, locales, models, pages, containers } = this.options; + const used = [models && "--models", pages && "--pages", containers && "--containers"].filter(Boolean); + if (used.length > 1) { + throw new VerifyAbort(`${used.join(", ")} cannot be combined — pass one scope at a time`); + } + if (pages) { + // Verify does not care whether a page's ancestors were synced: it reports what is there. + const pageScope = resolvePageSyncScope({ + sourceGuid, + targetGuid, + locales, + selectors: parsePageSelectors(pages), + isAncestorMapped: () => true, + }); + return { pageScope }; + } + if (containers) { + return resolveContainerSyncScope({ + sourceGuid, + targetGuid, + locales, + selectors: parseContainerSelectors(containers), + loadEntities: async (guid, locale) => (await this.deps.loadSide(guid, locale)) as any, + }).then((containerScope) => ({ containerScope })); + } + if (models) { + const names = models + .split(",") + .map((m) => m.trim()) + .filter(Boolean); + return names.length > 0 ? { models: names } : undefined; + } + return undefined; + } + + private loadMappings(): MappingIndex { + const { rootPath, sourceGuid, targetGuid, locales } = this.options; + const pairDir = mappingPairDirectory(rootPath, sourceGuid, targetGuid); + if (!fs.existsSync(pairDir)) { + throw new VerifyAbort( + `No mapping files for ${sourceGuid} → ${targetGuid} (expected ${pairDir}). Run a sync first.` + ); + } + const missingLocales = locales.filter((locale) => !fs.existsSync(path.join(pairDir, locale))); + if (missingLocales.length > 0) { + throw new VerifyAbort( + `No content/page mappings for locale(s) ${missingLocales.join(", ")} in ${pairDir}. Sync those locales first, or pass --locales.` + ); + } + try { + return buildMappingIndex(createFileMappingReader(rootPath, sourceGuid, targetGuid), locales); + } catch (error: any) { + throw new VerifyAbort(`Could not read the mapping files in ${pairDir}: ${error?.message || error}`); + } + } + + async compare(): Promise { + const { sourceGuid, targetGuid, locales, rootPath, channel } = this.options; + if (locales.length === 0) throw new VerifyAbort("No locales to verify"); + + const mappings = this.loadMappings(); + + const unreadable = [ + ...findUnreadableCacheFiles(rootPath, sourceGuid, locales), + ...findUnreadableCacheFiles(rootPath, targetGuid, locales), + ]; + if (unreadable.length > 0) { + throw new VerifyAbort( + `The pulled cache is unreadable (${unreadable.length} problem(s)): ${unreadable.slice(0, 5).join("; ")}` + ); + } + + let filter: ModelFilterOptions | undefined; + try { + filter = await this.resolveFilter(); + } catch (error: any) { + if (error instanceof VerifyAbort) throw error; + throw new VerifyAbort(error?.message || String(error)); + } + + const collector = new FindingCollector(); + for (let i = 0; i < locales.length; i++) { + const locale = locales[i]; + this.deps.log(ansiColors.cyan(`🔎 Comparing ${locale}...`)); + const source = await this.deps.loadSide(sourceGuid, locale, filter); + const target = await this.deps.loadSide(targetGuid, locale); + const ctx = createVerifyContext({ + sourceGuid, + targetGuid, + locale, + mappings, + source, + target, + scoped: !!filter, + channels: channel ? [channel] : undefined, + }); + + if (i === 0) { + // Models, containers, templates, assets and galleries are instance-wide: check them once. + checkStructure(ctx, checkInstanceIntegrity(ctx, collector), collector); + } + const localePairs = checkLocaleIntegrity(ctx, collector); + checkContent(ctx, localePairs.content, collector); + checkPages(ctx, localePairs.pages, collector); + checkSitemap(ctx, collector); + } + return collector; + } +} + +/** A CMS link for a finding: the target entity when there is one, else the source entity. */ +export function cmsLinkFor(f: Finding, sourceGuid: string, targetGuid: string): string | undefined { + if (f.locale === "*") return undefined; + const pick = (): { guid: string; id: number } | undefined => { + if (typeof f.targetID === "number") return { guid: targetGuid, id: f.targetID }; + if (typeof f.sourceID === "number") return { guid: sourceGuid, id: f.sourceID }; + return undefined; + }; + const target = pick(); + if (!target) return undefined; + if (f.entity === "content") return getContentCmsLink(target.guid, f.locale, target.id); + if (f.entity === "page" || f.entity === "zone" || f.entity === "sitemap") { + return getPageCmsLink(target.guid, f.locale, target.id); + } + return undefined; +} + +// ─── command ───────────────────────────────────────────────────────────────── + +export interface VerifyCommandDeps { + createAuth?: () => { init(): Promise; validateCommand(commandType: "verify"): Promise }; + createVerify?: (options: VerifyOptions) => { run(): Promise }; + log?: (line: string) => void; +} + +function optionsFromState(channel: string | undefined): VerifyOptions { + return { + sourceGuid: state.sourceGuid, + targetGuid: state.targetGuid, + locales: (state.guidLocaleMap.get(state.sourceGuid) || state.locale || []).slice(), + channel: channel ? String(channel) : undefined, + models: state.models || "", + pages: state.pages || "", + containers: state.containers || "", + jsonSummary: state.jsonSummary || "", + rootPath: state.rootPath, + }; +} + +/** + * The `verify` command handler: prime state the same way every other command does, authenticate, + * validate read access to both instances, run, and return the exit code (the caller exits). + */ +export async function runVerifyCommand(argv: any, deps: VerifyCommandDeps = {}): Promise { + const log = deps.log ?? ((line: string) => console.log(line)); + resetState(); + argv = normalizeArgv(argv); + + const envPriming = primeFromEnv(); + if (envPriming.hasEnvFile && envPriming.primedValues.length > 0) { + log(ansiColors.cyan(`📄 Found .env file, primed: ${envPriming.primedValues.join(", ")}`)); + } + + const startedAt = new Date(); + // --channel has no default for verify: unset means "every channel". + const explicitChannel: string | undefined = argv.channel ? String(argv.channel) : undefined; + + const abort = (reason: string): number => { + const options = optionsFromState(explicitChannel); + const report = buildErrorReport({ + source: state.sourceGuid, + target: state.targetGuid, + options: new Verify(options).optionsSummary, + error: reason, + run: emptyRunInfo(startedAt), + }); + printVerifyReport(report, { log }); + const reportPath = + options.jsonSummary || + defaultReportPath(state.rootPath, state.sourceGuid || "unknown", state.targetGuid || "unknown", startedAt); + writeVerifyReport(reportPath, report); + return EXIT_CANNOT_VERIFY; + }; + + try { + setState(argv); + } catch (error: any) { + return abort(error?.message || String(error)); + } + + if (!state.sourceGuid || !state.targetGuid) { + return abort("verify requires both --sourceGuid and --targetGuid (or AGILITY_GUID / AGILITY_TARGET_GUID in .env)"); + } + if (state.sourceGuid === state.targetGuid) { + return abort("verify requires two different instances (sourceGuid equals targetGuid)"); + } + + const auth = deps.createAuth ? deps.createAuth() : new Auth(); + try { + if (!(await auth.init())) { + return abort("Authentication failed — could not resolve API keys for the requested instances"); + } + if (!(await auth.validateCommand("verify"))) { + return abort("Command validation failed — no read access to both instances, or a required setting is missing"); + } + } catch (error: any) { + return abort(`Authentication failed: ${error?.message || error}`); + } + + const options = optionsFromState(explicitChannel); + const verify = deps.createVerify ? deps.createVerify(options) : new Verify(options); + const report = await verify.run(); + return report.exitCode; +} diff --git a/src/index.ts b/src/index.ts index 43db5cf..e461834 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,11 +28,13 @@ import { primeFromEnv, systemArgs, workflowArgs, + verifyArgs, normalizeProcessArgs, normalizeArgv, } from "./core"; import { Pull } from "./core/pull"; import { Push } from "./core/push"; +import { runVerifyCommand } from "./core/verify"; import { WorkflowOperation } from "./lib/workflows"; import { initializeLogger, getLogger, finalizeLogger } from "./core/state"; @@ -57,6 +59,9 @@ yargs.command({ console.log( colors.white(" reverse-sync - Sync the target back to the source, reusing the original mapping files") ); + console.log( + colors.white(" verify - Compare a synced target against its source and report every difference") + ); console.log( colors.white(" workflowOperation - Perform workflow operations (publish, unpublish, approve, decline)") ); @@ -216,6 +221,21 @@ yargs.command({ }, }); +// Verify (PROD-2552): full pull of both instances, then an independent comparison of the target +// against the source. Exit 0 = match, 1 = differences found, 2 = could not verify. +yargs.command({ + command: "verify", + describe: + "Compare a synced target instance against its source (fresh full pull of both) and report every difference. Exit code 0 = match, 1 = differences found, 2 = could not verify. Never writes to either instance or to the mapping files.", + builder: { + ...verifyArgs, + }, + handler: async function (argv) { + const exitCode = await runVerifyCommand(argv); + process.exit(exitCode); + }, +}); + /** * Shared handler for push / sync / reverse-sync. * diff --git a/src/lib/verify/canonicalize.ts b/src/lib/verify/canonicalize.ts new file mode 100644 index 0000000..d811a75 --- /dev/null +++ b/src/lib/verify/canonicalize.ts @@ -0,0 +1,462 @@ +/** + * Neutral comparison form for `agility verify` (PROD-2552). + * + * Verify must not reuse the push transforms: a bug in ContentFieldMapper or translateZoneNames + * would then be invisible, because the check would apply the same bug to the expectation. Instead, + * both sides are projected into one neutral form and compared there: + * + * - IDs become TARGET IDs. A source content ID is translated through the item mapping (or becomes + * an `unmapped:` token that can never match); a target ID is used as-is. + * - Linked-content references, in any of their shapes ({contentid}, the scalar form, sortids, a + * companion value field), become an ordered list of target IDs under "@ids"; a full-list link + * becomes the target container reference name under "@list". + * - Asset attachments compare on url + label only (size/dimensions are re-derived by the target). + * - Asset URLs anywhere in a string are rewritten source→target (exact URL first, then container + * prefix), and every known alias of a target container (edge/origin/custom CDN host) collapses + * onto one canonical prefix, so the same file reached through two hosts compares equal. + * - null, "" and a missing key are equal; CRLF equals LF; keys compare case-insensitively; + * numbers and numeric strings compare equal, as do booleans and "true"/"false". + */ +import { getLinkedContentCompanionFields } from "../content/linked-content-companion-fields"; +import { AssetPrefixPair, stripTrailingSlash } from "./mapping-index"; + +export type Side = "source" | "target"; + +// ─── asset URLs ────────────────────────────────────────────────────────────── + +const URL_TAIL = `(?:/[^\\s"'<>()]*)?`; +const URL_END = `(?=$|[/\\s"'<>()])`; + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function safeDecode(url: string): string { + try { + return decodeURI(url); + } catch { + return url; + } +} + +/** One regex matching any of `prefixes` (longest first) followed by the rest of a URL. */ +function prefixRegex(prefixes: string[]): RegExp | null { + if (prefixes.length === 0) return null; + const alternation = prefixes + .slice() + .sort((a, b) => b.length - a.length) + .map(escapeRegExp) + .join("|"); + return new RegExp(`(?:${alternation})${URL_END}${URL_TAIL}`, "g"); +} + +function containerName(prefix: string): string { + const trimmed = stripTrailingSlash(prefix); + return trimmed.slice(trimmed.lastIndexOf("/") + 1).toLowerCase(); +} + +export interface AssetUrlTranslatorInput { + /** Exact URL source → target (mapping records, plus each mapped asset's edge/origin URLs). */ + exact: Map; + /** Container prefix pairs source → target. */ + prefixPairs: AssetPrefixPair[]; + /** Every container prefix known to belong to the SOURCE instance. */ + sourcePrefixes: string[]; + /** Every container prefix known to belong to the TARGET instance (edge, origin, custom host). */ + targetPrefixes: string[]; +} + +export class AssetUrlTranslator { + private readonly exact: Map; + private readonly prefixPairs: AssetPrefixPair[]; + private readonly sourceRegex: RegExp | null; + private readonly detectRegex: RegExp | null; + private readonly aliasRegex: RegExp | null; + private readonly aliasToCanonical = new Map(); + private readonly aliasesLongestFirst: string[]; + + constructor(input: AssetUrlTranslatorInput) { + this.exact = new Map(); + input.exact.forEach((target, source) => { + this.exact.set(source, target); + const decoded = safeDecode(source); + if (!this.exact.has(decoded)) this.exact.set(decoded, target); + }); + this.prefixPairs = input.prefixPairs + .map((p) => ({ source: stripTrailingSlash(p.source), target: stripTrailingSlash(p.target) })) + .sort((a, b) => b.source.length - a.source.length); + + const targetPrefixes = unique(input.targetPrefixes.map(stripTrailingSlash).filter(Boolean)); + const targetSet = new Set(targetPrefixes); + const sourcePrefixes = unique( + [...input.sourcePrefixes, ...this.prefixPairs.map((p) => p.source)].map(stripTrailingSlash).filter(Boolean) + ).filter((p) => !targetSet.has(p)); + + // Group target aliases by container name; the lexically smallest alias is the canonical one. + const groups = new Map(); + targetPrefixes.forEach((prefix) => { + const name = containerName(prefix); + if (!groups.has(name)) groups.set(name, []); + (groups.get(name) as string[]).push(prefix); + }); + groups.forEach((aliases) => { + const canonical = aliases.slice().sort()[0]; + aliases.forEach((alias) => this.aliasToCanonical.set(alias, canonical)); + }); + this.aliasesLongestFirst = Array.from(this.aliasToCanonical.keys()).sort((a, b) => b.length - a.length); + + // Exact source URLs without a known container prefix (legacy mapping records) still need to + // be found inside rich text, so they join the source alternation. + const exactOnly = Array.from(this.exact.keys()).filter( + (url) => !sourcePrefixes.some((p) => url.startsWith(p + "/")) + ); + this.sourceRegex = prefixRegex([...sourcePrefixes, ...exactOnly]); + this.detectRegex = prefixRegex(sourcePrefixes); + this.aliasRegex = prefixRegex(this.aliasesLongestFirst); + } + + /** Canonical form of a URL that is already a target URL. */ + canonicalTargetUrl(url: string): string { + const decoded = safeDecode(url); + for (const alias of this.aliasesLongestFirst) { + if (decoded === alias || decoded.startsWith(alias + "/")) { + return (this.aliasToCanonical.get(alias) as string) + decoded.slice(alias.length); + } + } + return decoded; + } + + /** Translate a source URL to the target URL it should have become, in canonical form. */ + translateSourceUrl(url: string): string { + const exact = this.exact.get(url) ?? this.exact.get(safeDecode(url)); + if (exact) return this.canonicalTargetUrl(exact); + const decoded = safeDecode(url); + for (const pair of this.prefixPairs) { + if (decoded === pair.source || decoded.startsWith(pair.source + "/")) { + return this.canonicalTargetUrl(pair.target + decoded.slice(pair.source.length)); + } + } + return this.canonicalTargetUrl(decoded); + } + + /** Rewrite every source asset URL inside a string, then canonicalize target aliases. */ + rewriteSourceText(text: string): string { + const rewritten = this.sourceRegex ? text.replace(this.sourceRegex, (url) => this.translateSourceUrl(url)) : text; + return this.canonicalTargetText(rewritten); + } + + canonicalTargetText(text: string): string { + return this.aliasRegex ? text.replace(this.aliasRegex, (url) => this.canonicalTargetUrl(url)) : text; + } + + /** Source-instance asset URLs still present in a (target) string. */ + findSourceUrls(text: string): string[] { + if (!this.detectRegex || typeof text !== "string") return []; + return Array.from(new Set(text.match(this.detectRegex) ?? [])); + } +} + +function unique(values: T[]): T[] { + return Array.from(new Set(values)); +} + +/** + * Build the translator from the mapping index's asset records plus both sides' asset lists. + * + * The pulled content does not always use the host the mapping recorded (a mapping can carry a + * custom CDN host while content uses the edge host), so each mapped asset contributes its edge and + * origin URLs as exact pairs, and its container's edge/origin prefixes as prefix pairs. + */ +export function buildAssetUrlTranslator(input: { + mappingUrls: Map; + mappingPrefixes: AssetPrefixPair[]; + assetIdMap: { target(sourceMediaID: number): number | undefined }; + sourceAssets: any[]; + targetAssets: any[]; +}): AssetUrlTranslator { + const exact = new Map(input.mappingUrls); + const prefixPairs: AssetPrefixPair[] = input.mappingPrefixes.slice(); + const pairKeys = new Set(prefixPairs.map((p) => `${p.source}\u0000${p.target}`)); + const addPair = (source: any, target: any) => { + if (typeof source !== "string" || typeof target !== "string" || !source || !target) return; + const key = `${stripTrailingSlash(source)}\u0000${stripTrailingSlash(target)}`; + if (pairKeys.has(key)) return; + pairKeys.add(key); + prefixPairs.push({ source, target }); + }; + + const targetById = new Map(); + input.targetAssets.forEach((a) => targetById.set(a?.mediaID, a)); + + const sourcePrefixes = new Set(); + const targetPrefixes = new Set(); + input.mappingPrefixes.forEach((p) => { + sourcePrefixes.add(p.source); + targetPrefixes.add(p.target); + }); + input.sourceAssets.forEach((a) => { + if (a?.containerEdgeUrl) sourcePrefixes.add(a.containerEdgeUrl); + if (a?.containerOriginUrl) sourcePrefixes.add(a.containerOriginUrl); + }); + input.targetAssets.forEach((a) => { + if (a?.containerEdgeUrl) targetPrefixes.add(a.containerEdgeUrl); + if (a?.containerOriginUrl) targetPrefixes.add(a.containerOriginUrl); + }); + + input.sourceAssets.forEach((s) => { + const targetID = input.assetIdMap.target(s?.mediaID); + const t = targetID !== undefined ? targetById.get(targetID) : undefined; + if (!t) return; + (["edgeUrl", "originUrl", "url"] as const).forEach((key) => { + if (typeof s[key] === "string" && typeof t[key] === "string" && !exact.has(s[key])) exact.set(s[key], t[key]); + }); + addPair(s.containerEdgeUrl, t.containerEdgeUrl); + addPair(s.containerOriginUrl, t.containerOriginUrl); + }); + + return new AssetUrlTranslator({ + exact, + prefixPairs, + sourcePrefixes: Array.from(sourcePrefixes), + targetPrefixes: Array.from(targetPrefixes), + }); +} + +// ─── values ────────────────────────────────────────────────────────────────── + +export interface CanonicalContext { + side: Side; + /** A content ID as it should appear on the target ("unmapped:" when it has no mapping). */ + contentRef(id: number): string; + /** A container reference name as it should appear on the target (lower-cased). */ + containerRef(referenceName: string): string; + /** A gallery (media grouping) ID as it should appear on the target. */ + galleryRef(id: number): string; + /** + * True when a full-list reference points at a list with no items on this side. Linking an empty + * list is the same as linking nothing: the target creates an empty nested list for a grid field + * the source never filled in. + */ + isEmptyList?(referenceName: string): boolean; + assets: AssetUrlTranslator; +} + +const NUMERIC = /^-?(?:0|[1-9]\d*)(?:\.\d+)?$/; +const ID_LIST = /^\s*\d+(?:\s*,\s*\d+)*\s*,?\s*$/; + +function canonicalNumber(text: string): string { + const n = Number(text); + return Number.isSafeInteger(n) || (Number.isFinite(n) && text.includes(".")) ? String(n) : text; +} + +/** Normalize a scalar. Returns undefined for "empty" (null, undefined, ""). */ +export function canonicalScalar(value: any): string | undefined { + if (value === null || value === undefined) return undefined; + if (typeof value === "boolean") return value ? "true" : "false"; + if (typeof value === "number") return Number.isFinite(value) ? canonicalNumber(String(value)) : String(value); + if (typeof value !== "string") return undefined; + const text = value.replace(/\r\n/g, "\n"); + if (text === "") return undefined; + if (NUMERIC.test(text)) return canonicalNumber(text); + const lowered = text.toLowerCase(); + if (lowered === "true" || lowered === "false") return lowered; + return text; +} + +function lowerKeys(obj: Record): Record { + const out: Record = {}; + Object.keys(obj).forEach((key) => { + const lower = key.toLowerCase(); + // A payload with both "Title" and "title" keeps the first non-empty one. + if (!(lower in out) || out[lower] === null || out[lower] === undefined || out[lower] === "") out[lower] = obj[key]; + }); + return out; +} + +function parseIdList(text: string): number[] { + return text + .split(",") + .map((id) => id.trim()) + .filter((id) => /^\d+$/.test(id)) + .map((id) => parseInt(id, 10)) + .filter((id) => id > 0); +} + +function isAssetObject(value: any): boolean { + return !!value && typeof value === "object" && !Array.isArray(value) && typeof value.url === "string"; +} + +function isLinkedRef(lowered: Record): boolean { + return "contentid" in lowered || "sortids" in lowered || "fulllist" in lowered; +} + +function canonicalLinkedRef(lowered: Record, ctx: CanonicalContext): any { + let ids: string[] = []; + const sortids = lowered.sortids; + if (typeof sortids === "string" && sortids.trim()) { + ids = parseIdList(sortids).map((id) => ctx.contentRef(id)); + } else if (typeof sortids === "number" && sortids > 0) { + ids = [ctx.contentRef(sortids)]; + } else { + const contentid = typeof lowered.contentid === "string" ? parseInt(lowered.contentid, 10) : lowered.contentid; + if (typeof contentid === "number" && contentid > 0) ids = [ctx.contentRef(contentid)]; + } + + const fullList = lowered.fulllist === true || String(lowered.fulllist).toLowerCase() === "true"; + const list = + fullList && typeof lowered.referencename === "string" && lowered.referencename + ? ctx.containerRef(lowered.referencename) + : undefined; + + const out: Record = {}; + if (ids.length > 0) out["@ids"] = ids; + if (list && !(ids.length === 0 && ctx.isEmptyList?.(lowered.referencename))) out["@list"] = list; + return Object.keys(out).length > 0 ? out : undefined; +} + +function canonicalAsset(value: any, ctx: CanonicalContext): any { + const url = + ctx.side === "source" ? ctx.assets.translateSourceUrl(value.url) : ctx.assets.canonicalTargetUrl(value.url); + const out: Record = { "@asset": url }; + const label = canonicalScalar(value.label); + if (label !== undefined) out.label = label; + return out; +} + +function canonicalText(text: string, ctx: CanonicalContext): string | undefined { + const scalar = canonicalScalar(text); + if (scalar === undefined || scalar !== text.replace(/\r\n/g, "\n")) return scalar; + return ctx.side === "source" ? ctx.assets.rewriteSourceText(scalar) : ctx.assets.canonicalTargetText(scalar); +} + +/** + * Canonical form of one value. `fieldType` is the model's field type for a top-level field, used + * to recognize the scalar shapes that are only references because the model says so (a Content + * field stored as "123" or "1,2,3", a PhotoGallery stored as a gallery ID). + */ +export function canonicalValue(value: any, ctx: CanonicalContext, fieldType?: string): any { + if (value === null || value === undefined) return undefined; + const type = (fieldType || "").toLowerCase(); + + if (typeof value === "string" || typeof value === "number") { + const text = String(value); + if (type === "content" && ID_LIST.test(text)) { + const ids = parseIdList(text).map((id) => ctx.contentRef(id)); + return ids.length > 0 ? { "@ids": ids } : undefined; + } + if (type === "photogallery" && /^\d+$/.test(text.trim()) && parseInt(text, 10) > 0) { + return { "@gallery": ctx.galleryRef(parseInt(text, 10)) }; + } + return typeof value === "string" ? canonicalText(value, ctx) : canonicalScalar(value); + } + + if (typeof value === "boolean") return canonicalScalar(value); + + if (Array.isArray(value)) { + const items = value.map((item) => { + const canonical = canonicalValue(item, ctx); + return canonical === undefined ? null : canonical; + }); + return items.length > 0 && items.some((i) => i !== null) ? items : undefined; + } + + if (typeof value === "object") { + if (isAssetObject(value)) return canonicalAsset(value, ctx); + const lowered = lowerKeys(value); + if (isLinkedRef(lowered)) return canonicalLinkedRef(lowered, ctx); + if (type === "photogallery" && typeof lowered.galleryid === "number") { + return { "@gallery": ctx.galleryRef(lowered.galleryid) }; + } + const out: Record = {}; + Object.keys(lowered) + .sort() + .forEach((key) => { + const canonical = canonicalValue(lowered[key], ctx); + if (canonical !== undefined) out[key] = canonical; + }); + return Object.keys(out).length > 0 ? out : undefined; + } + + return undefined; +} + +/** + * Canonical form of a content item's fields object, keyed by lower-cased field name. + * + * `model` is the item's model on ITS OWN side: it names the Content-typed fields and their + * companion value fields (LinkeContentDropdownValueField / SortIDFieldName). + */ +export function canonicalFields(fields: any, model: any, ctx: CanonicalContext): Record { + const out: Record = {}; + if (!fields || typeof fields !== "object") return out; + + const typeByField = new Map(); + (Array.isArray(model?.fields) ? model.fields : []).forEach((f: any) => { + if (f?.name) typeByField.set(String(f.name).toLowerCase(), String(f.type || "")); + }); + getLinkedContentCompanionFields(model).forEach(([main, companion]) => { + if (companion.toLowerCase() !== main.toLowerCase()) typeByField.set(companion.toLowerCase(), "Content"); + }); + + const lowered = lowerKeys(fields); + Object.keys(lowered) + .sort() + .forEach((key) => { + const canonical = canonicalValue(lowered[key], ctx, typeByField.get(key)); + if (canonical !== undefined) out[key] = canonical; + }); + return out; +} + +// ─── comparison ────────────────────────────────────────────────────────────── + +/** JSON with sorted keys, so two equal canonical values stringify identically. */ +export function stableStringify(value: any): string { + if (value === undefined) return "undefined"; + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; + return `{${Object.keys(value) + .sort() + .map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`) + .join(",")}}`; +} + +export function canonicalEquals(a: any, b: any): boolean { + return stableStringify(a) === stableStringify(b); +} + +export interface FieldDifference { + field: string; + expected: any; + actual: any; +} + +/** Top-level keys whose canonical values differ, in key order. */ +export function diffCanonical(expected: Record, actual: Record): FieldDifference[] { + const keys = Array.from(new Set([...Object.keys(expected), ...Object.keys(actual)])).sort(); + const out: FieldDifference[] = []; + keys.forEach((key) => { + if (!canonicalEquals(expected[key], actual[key])) { + out.push({ field: key, ...describeDifference(expected[key], actual[key]) }); + } + }); + return out; +} + +const SNIPPET_CONTEXT = 80; + +/** + * Report-friendly expected/actual: long strings are cut to a window around the first difference, + * so a one-character change in a 20KB rich-text field is readable. + */ +export function describeDifference(expected: any, actual: any): { expected: any; actual: any } { + if (typeof expected === "string" && typeof actual === "string" && (expected.length > 200 || actual.length > 200)) { + let i = 0; + while (i < expected.length && i < actual.length && expected[i] === actual[i]) i++; + const start = Math.max(0, i - SNIPPET_CONTEXT); + const cut = (s: string) => + `${start > 0 ? "…" : ""}${s.slice(start, i + SNIPPET_CONTEXT)}${i + SNIPPET_CONTEXT < s.length ? "…" : ""}`; + return { expected: cut(expected), actual: cut(actual) }; + } + return { expected, actual }; +} diff --git a/src/lib/verify/check-content.ts b/src/lib/verify/check-content.ts new file mode 100644 index 0000000..4067c79 --- /dev/null +++ b/src/lib/verify/check-content.ts @@ -0,0 +1,266 @@ +/** + * Content checks for one locale (PROD-2552). + * + * Per mapped pair: + * - contentModelMismatch the target item is of a different model (through the model mapping) + * - contentContainerMismatch the target item sits in a different container — skipped for items + * placed on a page, whose container name is auto-generated + * - fieldMismatch one per differing field, compared in canonical form + * - sourceAssetUrlInTarget a target field still points at an asset in the SOURCE instance + * - publishStateDifference (warning) the two items are in different workflow states + * Per container / model (unscoped runs only): + * - countMismatch failure when the target has fewer items, warning when it has more + * Per list container: + * - listOrderMismatch the items' order (itemOrder) differs, with both sequences + * + * Ignored: contentID, versionID, modified, and the item's own state (beyond the warning above). + */ +import { Finding, FindingCollector } from "./verify-report"; +import { ComparablePair, contentSourceDefect } from "./check-mapping-integrity"; +import { sourceCanonicalContext, targetCanonicalContext, VerifyContext } from "./verify-context"; +import { canonicalFields, diffCanonical } from "./canonicalize"; + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +/** Every string inside a value, with the dotted path to it. */ +function walkStrings(value: any, path: string, visit: (text: string, path: string) => void): void { + if (typeof value === "string") visit(value, path); + else if (Array.isArray(value)) value.forEach((v, i) => walkStrings(v, `${path}[${i}]`, visit)); + else if (value && typeof value === "object") + Object.keys(value).forEach((k) => walkStrings(value[k], `${path}.${k}`, visit)); +} + +export function checkContent(ctx: VerifyContext, pairs: ComparablePair[], collector: FindingCollector): void { + const { locale } = ctx; + const sourceCtx = sourceCanonicalContext(ctx); + const targetCtx = targetCanonicalContext(ctx); + + pairs.forEach(({ sourceID, targetID, source, target }) => { + const findings: Finding[] = []; + const name = source?.properties?.referenceName; + const base = { entity: "content" as const, locale, sourceID, targetID, name }; + + const sourceModelName = lower(source?.properties?.definitionName); + const targetModelName = lower(target?.properties?.definitionName); + const expectedModel = ctx.mappings.modelNames.target(sourceModelName) ?? sourceModelName; + if (expectedModel !== targetModelName) { + findings.push({ + ...base, + category: "contentModelMismatch", + severity: "failure", + path: "definitionName", + expected: expectedModel, + actual: targetModelName, + }); + } + + if (!ctx.source.pageBoundContentIDs.has(sourceID)) { + const sourceContainer = lower(source?.properties?.referenceName); + const expectedContainer = ctx.mappings.containerNames.target(sourceContainer) ?? sourceContainer; + const actualContainer = lower(target?.properties?.referenceName); + if (expectedContainer !== actualContainer) { + findings.push({ + ...base, + category: "contentContainerMismatch", + severity: "failure", + path: "referenceName", + expected: expectedContainer, + actual: actualContainer, + }); + } + } + + const sourceModel = ctx.source.modelsByRef.get(sourceModelName); + const targetModel = ctx.target.modelsByRef.get(targetModelName); + const expectedFields = canonicalFields(source?.fields, sourceModel, sourceCtx); + const actualFields = canonicalFields(target?.fields, targetModel, targetCtx); + diffCanonical(expectedFields, actualFields).forEach((diff) => { + findings.push({ + ...base, + category: "fieldMismatch", + severity: "failure", + path: diff.field, + expected: diff.expected, + actual: diff.actual, + }); + }); + + const leaked = new Map(); + walkStrings(target?.fields, "fields", (text, path) => { + const urls = ctx.assets.findSourceUrls(text); + if (urls.length > 0) leaked.set(path.replace(/^fields\./, ""), urls); + }); + leaked.forEach((urls, path) => { + findings.push({ + ...base, + category: "sourceAssetUrlInTarget", + severity: "failure", + path, + actual: urls, + message: "Target content references an asset URL in the source instance", + }); + }); + + // A difference explained by the source item itself (a required field it lacks) is listed, + // but does not fail the run. + const defect = findings.length > 0 ? contentSourceDefect(source, ctx.source.modelsByRef) : undefined; + findings.forEach((f) => { + if (defect) { + f.severity = "sourceDefect"; + f.message = f.message ? `${f.message} — ${defect}` : defect; + } + collector.add(f); + }); + collector.count(locale, "content", findings.length === 0 ? "matched" : defect ? "known" : "mismatched"); + + const sourceState = source?.properties?.state; + const targetState = target?.properties?.state; + if (sourceState !== undefined && targetState !== undefined && sourceState !== targetState) { + collector.add({ + ...base, + category: "publishStateDifference", + severity: "warning", + path: "state", + expected: sourceState, + actual: targetState, + }); + } + }); + + if (!ctx.scoped) checkCounts(ctx, collector); + checkListOrder(ctx, pairs, collector); +} + +/** Target items whose every mapped source is absent from the source pull. */ +function orphanedTargetIDs(ctx: VerifyContext): Set { + const out = new Set(); + ctx.target.contentById.forEach((_item, targetID) => { + const sources = ctx.localeMappings.content.sources(targetID); + if (sources.length > 0 && !sources.some((s) => ctx.source.contentById.has(s))) out.add(targetID); + }); + return out; +} + +function countBy(items: Iterable<[number, any]>, key: (item: any) => string, skip: Set): Map { + const out = new Map(); + for (const [id, item] of Array.from(items)) { + if (skip.has(id)) continue; + const k = key(item); + if (!k) continue; + out.set(k, (out.get(k) ?? 0) + 1); + } + return out; +} + +function checkCounts(ctx: VerifyContext, collector: FindingCollector): void { + const { locale } = ctx; + const orphaned = orphanedTargetIDs(ctx); + const none = new Set(); + + const groups: Array<{ + entity: "container" | "model"; + key: (item: any) => string; + translate: (name: string) => string; + }> = [ + { + entity: "container", + key: (item) => lower(item?.properties?.referenceName), + translate: (name) => ctx.mappings.containerNames.target(name) ?? name, + }, + { + entity: "model", + key: (item) => lower(item?.properties?.definitionName), + translate: (name) => ctx.mappings.modelNames.target(name) ?? name, + }, + ]; + + groups.forEach(({ entity, key, translate }) => { + const sourceCounts = countBy(ctx.source.contentById.entries(), key, none); + const targetCounts = countBy(ctx.target.contentById.entries(), key, orphaned); + Array.from(sourceCounts.keys()) + .sort() + .forEach((name) => { + const expected = sourceCounts.get(name) as number; + const targetName = translate(name); + const actual = targetCounts.get(targetName) ?? 0; + if (expected === actual) return; + collector.add({ + category: "countMismatch", + severity: actual < expected ? "failure" : "warning", + entity, + locale, + name, + path: "itemCount", + expected, + actual, + message: `${entity === "container" ? "Container" : "Model"} "${name}" has ${expected} item(s) in the source and ${actual} on the target`, + }); + }); + }); +} + +function itemOrder(item: any): number { + const order = item?.properties?.itemOrder; + return typeof order === "number" ? order : Number.MAX_SAFE_INTEGER; +} + +function checkListOrder(ctx: VerifyContext, pairs: ComparablePair[], collector: FindingCollector): void { + const { locale } = ctx; + const bySourceContainer = new Map(); + pairs.forEach((pair) => { + const name = lower(pair.source?.properties?.referenceName); + if (!name) return; + if (!bySourceContainer.has(name)) bySourceContainer.set(name, []); + (bySourceContainer.get(name) as ComparablePair[]).push(pair); + }); + + Array.from(bySourceContainer.keys()) + .sort() + .forEach((name) => { + const group = bySourceContainer.get(name) as ComparablePair[]; + if (group.length < 2) return; + const expectedContainer = ctx.mappings.containerNames.target(name) ?? name; + // Only items that landed in the matching target container take part; a misplaced item is + // already reported as contentContainerMismatch. + const comparable = group.filter((p) => lower(p.target?.properties?.referenceName) === expectedContainer); + if (comparable.length < 2) return; + + const expected = comparable + .slice() + .sort((a, b) => itemOrder(a.source) - itemOrder(b.source) || a.sourceID - b.sourceID); + // Rank by the source's itemOrder value, so items that tie in the source may come in either + // order on the target. Ties in the target's itemOrder are broken by that rank, so only a + // real disagreement is reported. + const sourceRank = new Map(); + let rank = -1; + let previous: number | undefined; + expected.forEach((p) => { + const order = itemOrder(p.source); + if (order !== previous) rank++; + previous = order; + sourceRank.set(p.targetID, rank); + }); + const rankOf = (p: ComparablePair) => sourceRank.get(p.targetID) as number; + const actual = comparable + .slice() + .sort((a, b) => itemOrder(a.target) - itemOrder(b.target) || rankOf(a) - rankOf(b)); + + if (actual.every((p, i) => i === 0 || rankOf(actual[i - 1]) <= rankOf(p))) return; + const expectedIDs = expected.map((p) => p.targetID); + const actualIDs = actual.map((p) => p.targetID); + + collector.add({ + category: "listOrderMismatch", + severity: "failure", + entity: "container", + locale, + name, + path: "itemOrder", + expected: expectedIDs, + actual: actualIDs, + message: "List items are in a different order on the target (target content IDs shown)", + }); + }); +} diff --git a/src/lib/verify/check-mapping-integrity.ts b/src/lib/verify/check-mapping-integrity.ts new file mode 100644 index 0000000..46e7cf5 --- /dev/null +++ b/src/lib/verify/check-mapping-integrity.ts @@ -0,0 +1,299 @@ +/** + * Mapping integrity (PROD-2552): does every source entity have a live counterpart on the target, + * and does every target entity trace back to a live source? + * + * - unmapped (failure) a source entity with no mapping record + * - danglingMapping (failure) a mapping record whose target entity is not on the target + * - duplicateTarget (failure) two live source entities mapped onto one target entity + * - extraInTarget (warning) a target entity no mapping points at + * - sourceNotInPull (warning) a target entity whose mapped source is absent from the source + * pull (deleted or unpublished there since the sync) + * + * A source entity that cannot be migrated as-is (see isContentSourceDefect) is reported with + * severity sourceDefect instead of failure. Entities that pass are returned as comparable pairs + * for the per-type checks; this check only tallies checked / missing / extra / known. + */ +import { BiMap } from "./mapping-index"; +import { EntityKind, FindingCollector, INSTANCE_LOCALE, Severity } from "./verify-report"; +import { VerifyContext } from "./verify-context"; +import { validateContentItemAgainstModel } from "../pushers/content-pusher/util/validate-required-fields"; + +export interface ComparablePair { + sourceID: number; + targetID: number; + source: any; + target: any; +} + +export interface IntegritySet { + entity: EntityKind; + locale: string; + source: Map; + target: Map; + map: BiMap; + name: (entity: any) => string | undefined; + /** When false (scoped runs), target-side completeness is not checked. */ + checkExtra: boolean; + /** Returns a reason when a source entity's absence on the target is explained by the source. */ + sourceDefect?: (source: any) => string | undefined; +} + +export function checkIntegrity(set: IntegritySet, collector: FindingCollector): ComparablePair[] { + const pairs: ComparablePair[] = []; + const { entity, locale } = set; + + set.source.forEach((source, sourceID) => { + collector.count(locale, entity, "checked"); + const targetID = set.map.target(sourceID); + const defect = targetID !== undefined && set.target.has(targetID) ? undefined : set.sourceDefect?.(source); + const severity: Severity = defect ? "sourceDefect" : "failure"; + + if (targetID === undefined) { + collector.count(locale, entity, defect ? "known" : "missing"); + collector.add({ + category: "unmapped", + severity, + entity, + locale, + sourceID, + targetID: null, + name: set.name(source), + message: defect ? `No mapping — ${defect}` : "Source entity has no mapping to the target", + }); + return; + } + + if (!set.target.has(targetID)) { + collector.count(locale, entity, defect ? "known" : "missing"); + collector.add({ + category: "danglingMapping", + severity, + entity, + locale, + sourceID, + targetID, + name: set.name(source), + message: defect + ? `Mapped target is missing — ${defect}` + : "Mapping points at a target entity that is not on the target (deleted or unpublished)", + }); + return; + } + + pairs.push({ sourceID, targetID, source, target: set.target.get(targetID) }); + }); + + set.map.duplicateTargets().forEach(({ target, sources }) => { + const live = sources.filter((s) => set.source.has(s)); + if (live.length < 2) return; + collector.add({ + category: "duplicateTarget", + severity: "failure", + entity, + locale, + sourceID: live.join(","), + targetID: target, + name: set.target.has(target) ? set.name(set.target.get(target)) : undefined, + expected: "one source per target", + actual: live, + message: `${live.length} source entities are mapped onto the same target entity`, + }); + }); + + if (set.checkExtra) { + set.target.forEach((target, targetID) => { + const sources = set.map.sources(targetID); + if (sources.length === 0) { + collector.count(locale, entity, "extra"); + collector.add({ + category: "extraInTarget", + severity: "warning", + entity, + locale, + sourceID: null, + targetID, + name: set.name(target), + message: "Target entity is not mapped from any source entity", + }); + } else if (!sources.some((s) => set.source.has(s))) { + collector.count(locale, entity, "extra"); + collector.add({ + category: "sourceNotInPull", + severity: "warning", + entity, + locale, + sourceID: sources.join(","), + targetID, + name: set.name(target), + message: "Mapped source entity is not in the source pull (deleted or unpublished in the source)", + }); + } + }); + } + + return pairs; +} + +/** + * Why a source content item cannot be migrated as-is, if it can't: a required scalar field with no + * value, which the target's Management API rejects on save. + */ +export function contentSourceDefect(item: any, sourceModelsByRef: Map): string | undefined { + const model = sourceModelsByRef.get(String(item?.properties?.definitionName || "").toLowerCase()); + const { missing } = validateContentItemAgainstModel(item?.fields, model); + if (missing.length === 0) return undefined; + return `source item is missing required field(s): ${missing.map((m) => `${m.name} (${m.type})`).join(", ")}`; +} + +function idMap(items: T[], key: (item: T) => any): Map { + const out = new Map(); + items.forEach((item) => { + const id = key(item); + if (typeof id === "number" && id > 0 && !out.has(id)) out.set(id, item); + }); + return out; +} + +export interface InstanceIntegrityResult { + models: ComparablePair[]; + containers: ComparablePair[]; + templates: ComparablePair[]; + assets: ComparablePair[]; + galleries: ComparablePair[]; +} + +/** Models, containers, templates, assets and galleries — instance-wide, checked once per run. */ +export function checkInstanceIntegrity(ctx: VerifyContext, collector: FindingCollector): InstanceIntegrityResult { + const s = ctx.source.entities; + const t = ctx.target.entities; + const checkExtra = !ctx.scoped; + const locale = INSTANCE_LOCALE; + return { + models: checkIntegrity( + { + entity: "model", + locale, + source: idMap(s.models, (m: any) => m.id), + target: idMap(t.models, (m: any) => m.id), + map: ctx.mappings.models, + name: (m) => m?.referenceName, + checkExtra, + }, + collector + ), + containers: checkIntegrity( + { + entity: "container", + locale, + source: idMap(s.containers, (c: any) => c.contentViewID), + target: idMap(t.containers, (c: any) => c.contentViewID), + map: ctx.mappings.containers, + name: (c) => c?.referenceName, + checkExtra, + }, + collector + ), + templates: checkIntegrity( + { + entity: "template", + locale, + source: idMap(s.templates, (x: any) => x.pageTemplateID), + target: idMap(t.templates, (x: any) => x.pageTemplateID), + map: ctx.mappings.templates, + name: (x) => x?.pageTemplateName, + checkExtra, + }, + collector + ), + assets: checkIntegrity( + { + entity: "asset", + locale, + source: idMap(s.assets, (a: any) => a.mediaID), + target: idMap(t.assets, (a: any) => a.mediaID), + map: ctx.mappings.assets, + name: (a) => a?.originKey || a?.fileName, + checkExtra, + }, + collector + ), + galleries: checkIntegrity( + { + entity: "gallery", + locale, + source: idMap(s.galleries, (g: any) => g.mediaGroupingID), + target: idMap(t.galleries, (g: any) => g.mediaGroupingID), + map: ctx.mappings.galleries, + name: (g) => g?.name, + checkExtra, + }, + collector + ), + }; +} + +export interface LocaleIntegrityResult { + content: ComparablePair[]; + pages: ComparablePair[]; +} + +/** + * Source items outside any live container (deleted or system container): the sync skips them, so + * they are listed once per container as a source defect instead of one "unmapped" per item. + */ +function reportContentOutsideLiveContainers(ctx: VerifyContext, collector: FindingCollector): void { + const byContainer = new Map(); + ctx.source.outsideLiveContainers.forEach((item) => { + const name = String(item?.properties?.referenceName || "").toLowerCase(); + byContainer.set(name, (byContainer.get(name) ?? 0) + 1); + }); + Array.from(byContainer.keys()) + .sort() + .forEach((name) => { + const count = byContainer.get(name) as number; + collector.count(ctx.locale, "content", "checked", count); + collector.count(ctx.locale, "content", "known", count); + collector.add({ + category: "contentNotInLiveContainer", + severity: "sourceDefect", + entity: "content", + locale: ctx.locale, + name, + actual: count, + message: `${count} source item(s) belong to container "${name}", which is deleted or a system container in the source — not migrated`, + }); + }); +} + +/** Content items and pages for one locale. */ +export function checkLocaleIntegrity(ctx: VerifyContext, collector: FindingCollector): LocaleIntegrityResult { + const checkExtra = !ctx.scoped; + reportContentOutsideLiveContainers(ctx, collector); + return { + content: checkIntegrity( + { + entity: "content", + locale: ctx.locale, + source: ctx.source.contentById, + target: ctx.target.contentById, + map: ctx.localeMappings.content, + name: (c) => c?.properties?.referenceName, + checkExtra, + sourceDefect: (item) => contentSourceDefect(item, ctx.source.modelsByRef), + }, + collector + ), + pages: checkIntegrity( + { + entity: "page", + locale: ctx.locale, + source: ctx.source.pagesById, + target: ctx.target.pagesById, + map: ctx.localeMappings.pages, + name: (p) => p?.name, + checkExtra, + }, + collector + ), + }; +} diff --git a/src/lib/verify/check-page-zones.ts b/src/lib/verify/check-page-zones.ts new file mode 100644 index 0000000..5121bd0 --- /dev/null +++ b/src/lib/verify/check-page-zones.ts @@ -0,0 +1,344 @@ +/** + * Page checks for one locale (PROD-2552) — page fields, and the components in each zone. + * + * The defect this exists for: a sync reported success while the components on a page were in the + * wrong order. So zones are compared component by component, in order, and a zone is never paired + * by position: + * + * source zone → target zone via the template section-ID mapping (PROD-2350), else by an exact + * (case-insensitive) zone name, else `unmappedZone`. Two templates that define their zones in a + * different order still pair correctly; a zone that cannot be paired is reported, not guessed. + * + * Each zone is then compared as an ordered list of (module, target content ID): + * - missingComponent expected in this zone, not on the target page at all + * - extraComponent on the target zone, not expected anywhere on the page + * - componentInWrongZone expected in one zone, found in another + * - componentOrderMismatch the shared components are in a different order (both sequences shown) + * A module whose content item is absent from the source pull cannot be migrated: it is listed as + * a sourceDefect and left out of the expectation. + * + * Page fields compared: title, menuText, name, pageType, template (through the template mapping), + * redirectUrl, visible, seo, scripts, securePage. A different workflow state is a warning + * (publishStateDifference), as for content. + */ +import { Finding, FindingCollector } from "./verify-report"; +import { ComparablePair } from "./check-mapping-integrity"; +import { moduleContentID, sourceCanonicalContext, targetCanonicalContext, VerifyContext } from "./verify-context"; +import { canonicalEquals, canonicalValue, describeDifference } from "./canonicalize"; + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +export const COMPARED_PAGE_FIELDS = [ + "title", + "menuText", + "name", + "pageType", + "redirectUrl", + "visible", + "seo", + "scripts", + "securePage", +] as const; + +interface ZoneEntry { + key: string; + label: string; +} + +function sections(template: any): any[] { + return Array.isArray(template?.contentSectionDefinitions) ? template.contentSectionDefinitions : []; +} + +/** The target template a source page should be on, by name (lower-case). */ +export function expectedTemplateName(ctx: VerifyContext, sourceTemplateName: any): string { + const name = lower(sourceTemplateName); + if (!name) return ""; + const byName = ctx.mappings.templateNames.target(name); + if (byName) return byName; + const sourceTemplate = ctx.source.templatesByName.get(name); + const targetID = sourceTemplate ? ctx.mappings.templates.target(sourceTemplate.pageTemplateID) : undefined; + const targetTemplate = targetID !== undefined ? ctx.target.templatesById.get(targetID) : undefined; + return targetTemplate ? lower(targetTemplate.pageTemplateName) : name; +} + +/** + * Pair a source zone with a target zone. Section-ID mapping first, then an exact name; never by + * position. Returns the lower-cased target zone name, or undefined when there is no pairing. + */ +export function resolveTargetZone( + ctx: VerifyContext, + sourceZone: string, + sourceTemplate: any, + targetTemplate: any, + targetZoneNames: string[] +): string | undefined { + const zone = lower(sourceZone); + const sourceSection = sections(sourceTemplate).find((s) => lower(s?.pageItemTemplateReferenceName) === zone); + if (sourceSection?.pageItemTemplateID != null) { + const targetSectionID = ctx.mappings.sections.target(sourceSection.pageItemTemplateID); + if (targetSectionID !== undefined) { + const targetSection = sections(targetTemplate).find((s) => s?.pageItemTemplateID === targetSectionID); + if (targetSection?.pageItemTemplateReferenceName) return lower(targetSection.pageItemTemplateReferenceName); + } + } + const candidates = new Set([ + ...sections(targetTemplate).map((s) => lower(s?.pageItemTemplateReferenceName)), + ...targetZoneNames.map(lower), + ]); + return candidates.has(zone) ? zone : undefined; +} + +export function checkPages(ctx: VerifyContext, pairs: ComparablePair[], collector: FindingCollector): void { + pairs.forEach((pair) => { + checkPageFields(ctx, pair, collector); + checkPageZones(ctx, pair, collector); + }); +} + +function checkPageFields(ctx: VerifyContext, pair: ComparablePair, collector: FindingCollector): void { + const { locale } = ctx; + const { sourceID, targetID, source, target } = pair; + const sourceCtx = sourceCanonicalContext(ctx); + const targetCtx = targetCanonicalContext(ctx); + let differs = false; + + const report = (path: string, expected: any, actual: any) => { + differs = true; + collector.add({ + category: "pageFieldMismatch", + severity: "failure", + entity: "page", + locale, + sourceID, + targetID, + name: source?.name, + path, + ...describeDifference(expected, actual), + }); + }; + + COMPARED_PAGE_FIELDS.forEach((field) => { + const expected = canonicalValue(source?.[field], sourceCtx); + const actual = canonicalValue(target?.[field], targetCtx); + if (!canonicalEquals(expected, actual)) report(field, expected, actual); + }); + + const expectedTemplate = expectedTemplateName(ctx, source?.templateName); + const actualTemplate = lower(target?.templateName); + if (expectedTemplate !== actualTemplate) report("template", expectedTemplate || null, actualTemplate || null); + + collector.count(locale, "page", differs ? "mismatched" : "matched"); + + const sourceState = source?.properties?.state; + const targetState = target?.properties?.state; + if (sourceState !== undefined && targetState !== undefined && sourceState !== targetState) { + collector.add({ + category: "publishStateDifference", + severity: "warning", + entity: "page", + locale, + sourceID, + targetID, + name: source?.name, + path: "state", + expected: sourceState, + actual: targetState, + }); + } +} + +function moduleLabel(moduleName: string, id: string): string { + return `${moduleName}#${id}`; +} + +function checkPageZones(ctx: VerifyContext, pair: ComparablePair, collector: FindingCollector): void { + const { locale } = ctx; + const { sourceID, targetID, source, target } = pair; + const base = { entity: "zone" as const, locale, sourceID, targetID, name: source?.name }; + + const sourceTemplate = ctx.source.templatesByName.get(lower(source?.templateName)); + const targetTemplate = ctx.target.templatesByName.get(lower(target?.templateName)); + const targetZones: Record = {}; + Object.keys(target?.zones || {}).forEach((zone) => { + targetZones[lower(zone)] = Array.isArray(target.zones[zone]) ? target.zones[zone] : []; + }); + + const expectedByZone = new Map(); + const unmappedKeys = new Set(); + const findings: Finding[] = []; + const zoneOf = new Map(); + const add = (zone: string, finding: Finding) => { + findings.push(finding); + zoneOf.set(finding, zone); + }; + + Object.keys(source?.zones || {}).forEach((sourceZone) => { + const modules = Array.isArray(source.zones[sourceZone]) ? source.zones[sourceZone] : []; + const entries: ZoneEntry[] = []; + modules.forEach((module: any) => { + const moduleName = ctx.mappings.modelNames.target(lower(module?.module)) ?? lower(module?.module); + const contentID = moduleContentID(module); + if (contentID !== undefined && !ctx.source.contentById.has(contentID)) { + add(lower(sourceZone), { + ...base, + category: "moduleContentMissingInSource", + severity: "sourceDefect", + path: sourceZone, + expected: moduleLabel(lower(module?.module), `source:${contentID}`), + message: `Module ${module?.module} references source content ${contentID}, which is not in the source pull`, + }); + return; + } + const mapped = contentID === undefined ? undefined : ctx.localeMappings.content.target(contentID); + const id = contentID === undefined ? "none" : mapped !== undefined ? String(mapped) : `unmapped:${contentID}`; + entries.push({ key: `${moduleName}|${id}`, label: moduleLabel(moduleName, id) }); + }); + + const targetZone = resolveTargetZone(ctx, sourceZone, sourceTemplate, targetTemplate, Object.keys(targetZones)); + if (!targetZone) { + if (entries.length === 0) return; + entries.forEach((e) => unmappedKeys.add(e.key)); + add(lower(sourceZone), { + ...base, + category: "unmappedZone", + severity: "failure", + path: sourceZone, + expected: entries.map((e) => e.label), + message: `Source zone "${sourceZone}" has no counterpart on the target template (no section mapping, no zone of that name)`, + }); + return; + } + expectedByZone.set(targetZone, [...(expectedByZone.get(targetZone) ?? []), ...entries]); + }); + + const actualByZone = new Map(); + Object.keys(targetZones).forEach((zone) => { + actualByZone.set( + zone, + targetZones[zone].map((module: any) => { + const moduleName = lower(module?.module); + const contentID = moduleContentID(module); + const id = contentID === undefined ? "none" : String(contentID); + return { key: `${moduleName}|${id}`, label: moduleLabel(moduleName, id) }; + }) + ); + }); + + // Pass 1: match within the paired zone. + const zones = Array.from(new Set([...Array.from(expectedByZone.keys()), ...Array.from(actualByZone.keys())])).sort(); + const leftoverExpected = new Map(); + const leftoverActual = new Map>(); + const matchedInZone = new Map>(); + zones.forEach((zone) => { + const expected = expectedByZone.get(zone) ?? []; + const available = new Map(); + (actualByZone.get(zone) ?? []).forEach((e) => available.set(e.key, (available.get(e.key) ?? 0) + 1)); + const leftovers: ZoneEntry[] = []; + const matched = new Map(); + expected.forEach((e) => { + const count = available.get(e.key) ?? 0; + if (count > 0) { + available.set(e.key, count - 1); + matched.set(e.key, (matched.get(e.key) ?? 0) + 1); + } else { + leftovers.push(e); + } + }); + leftoverExpected.set(zone, leftovers); + leftoverActual.set(zone, available); + matchedInZone.set(zone, matched); + }); + + // Pass 2: an expected component found in a different target zone is in the wrong zone. + zones.forEach((zone) => { + (leftoverExpected.get(zone) ?? []).forEach((e) => { + const otherZone = zones.find((z) => z !== zone && (leftoverActual.get(z)?.get(e.key) ?? 0) > 0); + if (otherZone) { + const available = leftoverActual.get(otherZone) as Map; + available.set(e.key, (available.get(e.key) as number) - 1); + add(zone, { + ...base, + category: "componentInWrongZone", + severity: "failure", + path: zone, + expected: `${e.label} in ${zone}`, + actual: `${e.label} in ${otherZone}`, + }); + } else { + add(zone, { + ...base, + category: "missingComponent", + severity: "failure", + path: zone, + expected: e.label, + actual: (actualByZone.get(zone) ?? []).map((a) => a.label), + }); + } + }); + }); + + // Pass 3: whatever is left on the target was not expected anywhere. + zones.forEach((zone) => { + const available = leftoverActual.get(zone) as Map; + (actualByZone.get(zone) ?? []).forEach((e) => { + const count = available.get(e.key) ?? 0; + if (count <= 0) return; + available.set(e.key, count - 1); + if (unmappedKeys.has(e.key)) return; // explained by the unmappedZone finding + add(zone, { + ...base, + category: "extraComponent", + severity: "failure", + path: zone, + actual: e.label, + expected: (expectedByZone.get(zone) ?? []).map((x) => x.label), + }); + }); + }); + + // Order: compare the components the two sides share in this zone. Each side contributes only as + // many occurrences of a component as were matched, so a duplicate (already an extraComponent) + // does not also read as an order change. + const shared = (entries: ZoneEntry[], matched: Map): string[] => { + const remaining = new Map(matched); + const keys: string[] = []; + entries.forEach((e) => { + const left = remaining.get(e.key) ?? 0; + if (left <= 0) return; + remaining.set(e.key, left - 1); + keys.push(e.key); + }); + return keys; + }; + zones.forEach((zone) => { + const matched = matchedInZone.get(zone) as Map; + const expectedKeys = shared(expectedByZone.get(zone) ?? [], matched); + if (expectedKeys.length < 2) return; + const actualKeys = shared(actualByZone.get(zone) ?? [], matched); + if (expectedKeys.every((k, i) => k === actualKeys[i])) return; + add(zone, { + ...base, + category: "componentOrderMismatch", + severity: "failure", + path: zone, + expected: (expectedByZone.get(zone) ?? []).map((e) => e.label), + actual: (actualByZone.get(zone) ?? []).map((e) => e.label), + message: "Components are in a different order on the target", + }); + }); + + findings.forEach((f) => collector.add(f)); + + const outcomeZones = new Set([...zones, ...Array.from(zoneOf.values())]); + outcomeZones.forEach((zone) => { + const zoneFindings = findings.filter((f) => zoneOf.get(f) === zone); + const hasEntries = (expectedByZone.get(zone)?.length ?? 0) > 0 || (actualByZone.get(zone)?.length ?? 0) > 0; + if (!hasEntries && zoneFindings.length === 0) return; + collector.count(locale, "zone", "checked"); + if (zoneFindings.some((f) => f.severity === "failure")) collector.count(locale, "zone", "mismatched"); + else if (zoneFindings.length > 0) collector.count(locale, "zone", "known"); + else collector.count(locale, "zone", "matched"); + }); +} diff --git a/src/lib/verify/check-sitemap.ts b/src/lib/verify/check-sitemap.ts new file mode 100644 index 0000000..96f4532 --- /dev/null +++ b/src/lib/verify/check-sitemap.ts @@ -0,0 +1,151 @@ +/** + * Sitemap checks for one locale, per channel (PROD-2552). + * + * The source nested sitemap is translated to target page IDs through the page mapping and compared + * with the target's nested sitemap for the same channel: + * - pageMissingFromSitemap a mapped page that exists on the target but is not in its sitemap + * - parentMismatch the page sits under a different parent (root = 0) + * - siblingOrderMismatch one finding per parent whose children are in a different order, + * with both sequences (target page IDs) + * Dynamic-page item entries (nodes carrying a contentID) are generated from content, not pages, + * and are skipped. Pages that are not mapped at all are reported by mapping integrity, not here. + */ +import { FindingCollector } from "./verify-report"; +import { ChannelSitemaps, VerifyContext } from "./verify-context"; +import { SitemapNode } from "../../types/syncAnalysis"; + +interface Placement { + parent: number; + path: string; + name: string; +} + +interface FlatSitemap { + placement: Map; + children: Map; +} + +function isPageNode(node: SitemapNode): boolean { + return typeof node?.pageID === "number" && node.pageID > 0 && !node.contentID; +} + +/** Flatten a nested sitemap into parent links and ordered child lists, keyed by page ID. */ +export function flattenSitemap(nodes: SitemapNode[] | null | undefined): FlatSitemap { + const flat: FlatSitemap = { placement: new Map(), children: new Map() }; + const visit = (list: SitemapNode[] | undefined, parent: number) => { + (list || []).forEach((node) => { + if (!isPageNode(node)) return; + if (!flat.placement.has(node.pageID)) { + flat.placement.set(node.pageID, { parent, path: node.path, name: node.name }); + if (!flat.children.has(parent)) flat.children.set(parent, []); + (flat.children.get(parent) as number[]).push(node.pageID); + } + visit(node.children, node.pageID); + }); + }; + visit(nodes || [], 0); + return flat; +} + +function channelKey(sitemaps: ChannelSitemaps): Map { + const out = new Map(); + Object.keys(sitemaps || {}).forEach((name) => out.set(name.toLowerCase(), name)); + return out; +} + +export function checkSitemap(ctx: VerifyContext, collector: FindingCollector): void { + const { locale } = ctx; + const sourceChannels = channelKey(ctx.source.entities.sitemaps); + const targetChannels = channelKey(ctx.target.entities.sitemaps); + const pageMap = ctx.localeMappings.pages; + // A scoped run compares only the pages the scope loaded on the source side. + const inScope = (sourcePageID: number) => !ctx.scoped || ctx.source.pagesById.has(sourcePageID); + + Array.from(sourceChannels.keys()) + .sort() + .forEach((channel) => { + if (ctx.channels && !ctx.channels.includes(channel)) return; + const source = flattenSitemap(ctx.source.entities.sitemaps[sourceChannels.get(channel) as string]); + const targetName = targetChannels.get(channel); + const target = flattenSitemap(targetName ? ctx.target.entities.sitemaps[targetName] : null); + const where = (path: string) => `${channel}:${path}`; + + const toTarget = (sourcePageID: number): number | undefined => + sourcePageID === 0 ? 0 : pageMap.target(sourcePageID); + + source.placement.forEach((placement, sourcePageID) => { + if (!inScope(sourcePageID)) return; + const targetPageID = pageMap.target(sourcePageID); + // Unmapped or dangling: mapping integrity already reports the page itself. + if (targetPageID === undefined || !ctx.target.pagesById.has(targetPageID)) return; + collector.count(locale, "sitemap", "checked"); + + const actual = target.placement.get(targetPageID); + if (!actual) { + collector.count(locale, "sitemap", "missing"); + collector.add({ + category: "pageMissingFromSitemap", + severity: "failure", + entity: "sitemap", + locale, + sourceID: sourcePageID, + targetID: targetPageID, + name: placement.name, + path: where(placement.path), + message: `Page is on the target but not in its "${channel}" sitemap`, + }); + return; + } + + const expectedParent = toTarget(placement.parent); + if (expectedParent !== actual.parent) { + collector.count(locale, "sitemap", "mismatched"); + collector.add({ + category: "parentMismatch", + severity: "failure", + entity: "sitemap", + locale, + sourceID: sourcePageID, + targetID: targetPageID, + name: placement.name, + path: where(placement.path), + expected: expectedParent === undefined ? `unmapped:${placement.parent}` : expectedParent, + actual: actual.parent, + message: "Page is under a different parent on the target (0 = root)", + }); + return; + } + collector.count(locale, "sitemap", "matched"); + }); + + source.children.forEach((sourceChildren, sourceParent) => { + if (sourceParent !== 0 && !inScope(sourceParent)) return; + const targetParent = toTarget(sourceParent); + if (targetParent === undefined) return; + const actualChildren = target.children.get(targetParent) ?? []; + const actualSet = new Set(actualChildren); + const expected = sourceChildren + .filter(inScope) + .map((id) => pageMap.target(id)) + .filter((id): id is number => id !== undefined && actualSet.has(id)); + if (expected.length < 2) return; + const expectedSet = new Set(expected); + const actual = actualChildren.filter((id) => expectedSet.has(id)); + if (expected.every((id, i) => id === actual[i])) return; + const parentPlacement = source.placement.get(sourceParent); + collector.add({ + category: "siblingOrderMismatch", + severity: "failure", + entity: "sitemap", + locale, + sourceID: sourceParent, + targetID: targetParent, + name: parentPlacement?.name ?? "(root)", + path: where(parentPlacement?.path ?? "/"), + expected, + actual, + message: "Child pages are in a different order on the target (target page IDs shown)", + }); + }); + }); +} diff --git a/src/lib/verify/check-structure.ts b/src/lib/verify/check-structure.ts new file mode 100644 index 0000000..d121d7b --- /dev/null +++ b/src/lib/verify/check-structure.ts @@ -0,0 +1,138 @@ +/** + * Structural checks (PROD-2552): models, containers and templates. + * + * Compared on what defines their shape, never on IDs, dates or who modified them: + * - model: reference name, and the ordered list of fields as (name, type) + * - container: reference name, and the model it holds (through the model mapping) + * - template: the ordered list of zone (section) reference names + * Assets and galleries are only checked for presence (mapping integrity); a matched pair counts + * as matched. + */ +import { FindingCollector, INSTANCE_LOCALE } from "./verify-report"; +import { InstanceIntegrityResult } from "./check-mapping-integrity"; +import { VerifyContext } from "./verify-context"; +import { canonicalEquals } from "./canonicalize"; + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +export function modelFieldSignature(model: any): string[] { + return (Array.isArray(model?.fields) ? model.fields : []).map((f: any) => `${lower(f?.name)}:${lower(f?.type)}`); +} + +export function templateSectionSignature(template: any): string[] { + return (Array.isArray(template?.contentSectionDefinitions) ? template.contentSectionDefinitions : []) + .slice() + .sort((a: any, b: any) => (a?.itemOrder ?? 0) - (b?.itemOrder ?? 0)) + .map((s: any) => lower(s?.pageItemTemplateReferenceName)); +} + +export function checkStructure(ctx: VerifyContext, pairs: InstanceIntegrityResult, collector: FindingCollector): void { + const locale = INSTANCE_LOCALE; + + pairs.models.forEach(({ sourceID, targetID, source, target }) => { + let differs = false; + if (lower(source.referenceName) !== lower(target.referenceName)) { + differs = true; + collector.add({ + category: "modelMismatch", + severity: "failure", + entity: "model", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "referenceName", + expected: source.referenceName, + actual: target.referenceName, + }); + } + const expected = modelFieldSignature(source); + const actual = modelFieldSignature(target); + if (!canonicalEquals(expected, actual)) { + differs = true; + collector.add({ + category: "modelMismatch", + severity: "failure", + entity: "model", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "fields", + expected, + actual, + message: "Model fields differ (name:type, in order)", + }); + } + collector.count(locale, "model", differs ? "mismatched" : "matched"); + }); + + pairs.containers.forEach(({ sourceID, targetID, source, target }) => { + let differs = false; + if (lower(source.referenceName) !== lower(target.referenceName)) { + differs = true; + collector.add({ + category: "containerMismatch", + severity: "failure", + entity: "container", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "referenceName", + expected: source.referenceName, + actual: target.referenceName, + }); + } + const expectedModelID = ctx.mappings.models.target(source.contentDefinitionID); + if (expectedModelID === undefined || expectedModelID !== target.contentDefinitionID) { + differs = true; + const sourceModel = ctx.source.modelsById.get(source.contentDefinitionID); + const targetModel = ctx.target.modelsById.get(target.contentDefinitionID); + collector.add({ + category: "containerMismatch", + severity: "failure", + entity: "container", + locale, + sourceID, + targetID, + name: source.referenceName, + path: "model", + expected: + expectedModelID === undefined + ? `unmapped model ${sourceModel?.referenceName ?? source.contentDefinitionID}` + : `${sourceModel?.referenceName ?? "?"} (#${expectedModelID})`, + actual: `${targetModel?.referenceName ?? "?"} (#${target.contentDefinitionID})`, + message: "Container holds a different model on the target", + }); + } + collector.count(locale, "container", differs ? "mismatched" : "matched"); + }); + + pairs.templates.forEach(({ sourceID, targetID, source, target }) => { + const expected = templateSectionSignature(source); + const actual = templateSectionSignature(target); + const differs = !canonicalEquals(expected, actual); + if (differs) { + collector.add({ + category: "templateMismatch", + severity: "failure", + entity: "template", + locale, + sourceID, + targetID, + name: source.pageTemplateName, + path: "sections", + expected, + actual, + message: "Template zones differ (reference name, in order)", + }); + } + collector.count(locale, "template", differs ? "mismatched" : "matched"); + }); + + pairs.assets.forEach(() => collector.count(locale, "asset", "matched")); + pairs.galleries.forEach(() => collector.count(locale, "gallery", "matched")); +} diff --git a/src/lib/verify/mapping-index.ts b/src/lib/verify/mapping-index.ts new file mode 100644 index 0000000..27664ce --- /dev/null +++ b/src/lib/verify/mapping-index.ts @@ -0,0 +1,213 @@ +/** + * Bidirectional, Map-backed indices over every mapping file of a source→target pair (PROD-2552). + * + * The mappers in lib/mappers look records up with Array.find, which is fine for the one lookup a + * pusher makes per item but quadratic for a verify pass that touches every item in both + * directions. These indices are built once per run and answer every lookup in O(1). + * + * Read-only: records come in through a reader function and nothing here can write a mapping file. + */ +import * as fs from "fs"; +import * as path from "path"; +import { fileOperations } from "../../core/fileOperations"; + +/** Mapping folder names as the mappers write them. Content and pages are per-locale. */ +export type MappingType = + | "models" + | "containers" + | "templates" + | "sections" + | "galleries" + | "assets" + | "urlredirections" + | "item" + | "page"; + +export type MappingReader = (type: MappingType, locale?: string) => any[]; + +/** + * A one-to-one-ish relation between source keys and target keys. Duplicate sources keep their + * FIRST target (the same record the mappers' Array.find would return); duplicate targets — two + * sources mapped onto one target — are kept and reported. + */ +export class BiMap { + private readonly s2t = new Map(); + private readonly t2s = new Map(); + + add(source: K | null | undefined, target: K | null | undefined): void { + if (!isUsableKey(source) || !isUsableKey(target)) return; + if (this.s2t.has(source as K)) return; + this.s2t.set(source as K, target as K); + const sources = this.t2s.get(target as K); + if (sources) sources.push(source as K); + else this.t2s.set(target as K, [source as K]); + } + + target(source: K | null | undefined): K | undefined { + return isUsableKey(source) ? this.s2t.get(source as K) : undefined; + } + + sources(target: K | null | undefined): K[] { + return isUsableKey(target) ? (this.t2s.get(target as K) ?? []) : []; + } + + hasSource(source: K): boolean { + return this.s2t.has(source); + } + + hasTarget(target: K): boolean { + return this.t2s.has(target); + } + + /** Targets that more than one source maps onto. */ + duplicateTargets(): Array<{ target: K; sources: K[] }> { + const out: Array<{ target: K; sources: K[] }> = []; + this.t2s.forEach((sources, target) => { + if (sources.length > 1) out.push({ target, sources: sources.slice() }); + }); + return out; + } + + entries(): Array<[K, K]> { + return Array.from(this.s2t.entries()); + } + + get size(): number { + return this.s2t.size; + } +} + +function isUsableKey(key: any): boolean { + if (key === null || key === undefined || key === "") return false; + if (typeof key === "number") return Number.isFinite(key) && key > 0; + return true; +} + +function lower(value: any): string | undefined { + return typeof value === "string" && value.length > 0 ? value.toLowerCase() : undefined; +} + +/** A source container-URL prefix and the target prefix it maps onto. */ +export interface AssetPrefixPair { + source: string; + target: string; +} + +export interface LocaleMappingIndex { + content: BiMap; + pages: BiMap; +} + +export interface MappingIndex { + models: BiMap; + /** Lower-cased reference names. */ + modelNames: BiMap; + containers: BiMap; + /** Lower-cased reference names. */ + containerNames: BiMap; + templates: BiMap; + /** Lower-cased template names. */ + templateNames: BiMap; + /** pageItemTemplateID (zone definition) source → target. */ + sections: BiMap; + galleries: BiMap; + assets: BiMap; + /** Exact asset URL source → target. */ + assetUrls: Map; + /** Container URL prefixes (edge and origin) source → target, from the mapping records. */ + assetPrefixes: AssetPrefixPair[]; + byLocale: Map; +} + +export function buildMappingIndex(read: MappingReader, locales: string[]): MappingIndex { + const index: MappingIndex = { + models: new BiMap(), + modelNames: new BiMap(), + containers: new BiMap(), + containerNames: new BiMap(), + templates: new BiMap(), + templateNames: new BiMap(), + sections: new BiMap(), + galleries: new BiMap(), + assets: new BiMap(), + assetUrls: new Map(), + assetPrefixes: [], + byLocale: new Map(), + }; + + (read("models") || []).forEach((m: any) => { + index.models.add(m.sourceID, m.targetID); + index.modelNames.add(lower(m.sourceReferenceName), lower(m.targetReferenceName)); + }); + + (read("containers") || []).forEach((c: any) => { + index.containers.add(c.sourceContentViewID, c.targetContentViewID); + index.containerNames.add(lower(c.sourceReferenceName), lower(c.targetReferenceName)); + }); + + (read("templates") || []).forEach((t: any) => { + index.templates.add(t.sourcePageTemplateID, t.targetPageTemplateID); + index.templateNames.add(lower(t.sourcePageTemplateName), lower(t.targetPageTemplateName)); + }); + + (read("sections") || []).forEach((s: any) => + index.sections.add(s.sourcePageItemTemplateID, s.targetPageItemTemplateID) + ); + + (read("galleries") || []).forEach((g: any) => index.galleries.add(g.sourceMediaGroupingID, g.targetMediaGroupingID)); + + const prefixKeys = new Set(); + const addPrefix = (source: any, target: any) => { + if (typeof source !== "string" || typeof target !== "string" || !source || !target) return; + const pair = { source: stripTrailingSlash(source), target: stripTrailingSlash(target) }; + const key = `${pair.source}\u0000${pair.target}`; + if (prefixKeys.has(key)) return; + prefixKeys.add(key); + index.assetPrefixes.push(pair); + }; + (read("assets") || []).forEach((a: any) => { + index.assets.add(a.sourceMediaID, a.targetMediaID); + if (typeof a.sourceUrl === "string" && typeof a.targetUrl === "string" && a.sourceUrl && a.targetUrl) { + if (!index.assetUrls.has(a.sourceUrl)) index.assetUrls.set(a.sourceUrl, a.targetUrl); + } + addPrefix(a.sourceContainerEdgeUrl, a.targetContainerEdgeUrl); + addPrefix(a.sourceContainerOriginUrl, a.targetContainerOriginUrl); + }); + + locales.forEach((locale) => { + const localeIndex: LocaleMappingIndex = { content: new BiMap(), pages: new BiMap() }; + (read("item", locale) || []).forEach((r: any) => localeIndex.content.add(r.sourceContentID, r.targetContentID)); + (read("page", locale) || []).forEach((r: any) => localeIndex.pages.add(r.sourcePageID, r.targetPageID)); + index.byLocale.set(locale, localeIndex); + }); + + return index; +} + +export function stripTrailingSlash(url: string): string { + return url.endsWith("/") ? url.slice(0, -1) : url; +} + +/** The on-disk directory holding one pair's mappings. */ +export function mappingPairDirectory(rootPath: string, sourceGuid: string, targetGuid: string): string { + return path.join(rootPath, "mappings", `${sourceGuid}-${targetGuid}`); +} + +/** + * Read mappings through fileOperations.getMappingFile, but only when the mappings.json exists: + * getMappingFile seeds an empty file when the folder exists without one, and verify must never + * write to a mapping directory. + */ +export function createFileMappingReader(rootPath: string, sourceGuid: string, targetGuid: string): MappingReader { + const fileOps = new fileOperations(sourceGuid); + const pairDir = mappingPairDirectory(rootPath, sourceGuid, targetGuid); + return (type: MappingType, locale?: string) => { + const file = path.join(pairDir, locale ?? "", type, "mappings.json"); + if (!fs.existsSync(file)) return []; + const records = fileOps.getMappingFile(type, sourceGuid, targetGuid, locale ?? null); + if (!Array.isArray(records)) { + throw new Error(`Mapping file ${file} does not contain an array`); + } + return records; + }; +} diff --git a/src/lib/verify/tests/canonicalize.test.ts b/src/lib/verify/tests/canonicalize.test.ts new file mode 100644 index 0000000..5982c79 --- /dev/null +++ b/src/lib/verify/tests/canonicalize.test.ts @@ -0,0 +1,227 @@ +import { + AssetUrlTranslator, + buildAssetUrlTranslator, + canonicalFields, + canonicalScalar, + canonicalValue, + CanonicalContext, + diffCanonical, + describeDifference, +} from "../canonicalize"; + +const SRC_EDGE = "https://cdn-usa2.aglty.io/src-container"; +const SRC_CUSTOM = "https://cdn.custom.com/src-container"; +const TGT_EDGE = "https://cdn-usa2.aglty.io/tgt-container"; +const TGT_CUSTOM = "https://cdn.custom.com/tgt-container"; +const TGT_ORIGIN = "https://origin.example.net/tgt-container"; + +function translator(): AssetUrlTranslator { + return buildAssetUrlTranslator({ + mappingUrls: new Map([[`${SRC_CUSTOM}/img/exact.png`, `${TGT_CUSTOM}/img/renamed.png`]]), + mappingPrefixes: [{ source: SRC_CUSTOM, target: TGT_CUSTOM }], + assetIdMap: { target: (id: number) => (id === 1 ? 101 : undefined) }, + sourceAssets: [{ mediaID: 1, containerEdgeUrl: SRC_EDGE, edgeUrl: `${SRC_EDGE}/a.png` }], + targetAssets: [ + { mediaID: 101, containerEdgeUrl: TGT_EDGE, containerOriginUrl: TGT_ORIGIN, edgeUrl: `${TGT_EDGE}/a.png` }, + ], + }); +} + +function contexts(): { source: CanonicalContext; target: CanonicalContext } { + const assets = translator(); + const contentMap = new Map([ + [1, 501], + [2, 502], + [3, 503], + ]); + return { + source: { + side: "source", + contentRef: (id) => (contentMap.has(id) ? String(contentMap.get(id)) : `unmapped:${id}`), + containerRef: (name) => (name.toLowerCase() === "posts" ? "poststarget" : `unmapped:${name.toLowerCase()}`), + galleryRef: (id) => String(id + 1000), + assets, + }, + target: { + side: "target", + contentRef: (id) => String(id), + containerRef: (name) => name.toLowerCase(), + galleryRef: (id) => String(id), + assets, + }, + }; +} + +function same(sourceValue: any, targetValue: any, fieldType?: string): boolean { + const { source, target } = contexts(); + return ( + JSON.stringify(canonicalValue(sourceValue, source, fieldType)) === + JSON.stringify(canonicalValue(targetValue, target, fieldType)) + ); +} + +describe("canonicalScalar", () => { + it("treats null, undefined and empty string as empty", () => { + expect(canonicalScalar(null)).toBeUndefined(); + expect(canonicalScalar(undefined)).toBeUndefined(); + expect(canonicalScalar("")).toBeUndefined(); + }); + + it("normalizes CRLF to LF", () => { + expect(canonicalScalar("a\r\nb")).toBe("a\nb"); + }); + + it("equates numbers and numeric strings", () => { + expect(canonicalScalar("13")).toBe(canonicalScalar(13)); + expect(canonicalScalar("37.80")).toBe(canonicalScalar(37.8)); + expect(canonicalScalar("-2")).toBe("-2"); + }); + + it("keeps leading-zero strings (zip codes, codes) as text", () => { + expect(canonicalScalar("007")).toBe("007"); + }); + + it("equates booleans and their string forms", () => { + expect(canonicalScalar(true)).toBe(canonicalScalar("True")); + expect(canonicalScalar(false)).toBe("false"); + }); +}); + +describe("canonicalValue", () => { + it("compares object keys case-insensitively and drops empty values", () => { + expect(same({ Href: "/x", Target: "", Text: "Go" }, { href: "/x", text: "Go", target: null })).toBe(true); + }); + + it("treats a missing value and an empty one as equal", () => { + expect(same("", undefined)).toBe(true); + expect(same(null, "")).toBe(true); + }); + + describe("linked content", () => { + it("translates a {contentid} selection to the mapped target ID", () => { + expect(same({ contentid: 1, fulllist: false }, { contentid: 501, fulllist: false })).toBe(true); + expect(same({ contentid: 1, fulllist: false }, { contentid: 502, fulllist: false })).toBe(false); + }); + + it("equates the object form with the scalar form on a Content field", () => { + expect(same({ contentid: 1, fulllist: false }, "501", "Content")).toBe(true); + expect(same("1", { contentid: 501 }, "Content")).toBe(true); + }); + + it("keeps sortids order significant", () => { + const source = { referencename: "posts", fulllist: true, sortids: "1,2,3" }; + expect(same(source, { referencename: "PostsTarget", fulllist: true, sortids: "501,502,503" })).toBe(true); + expect(same(source, { referencename: "PostsTarget", fulllist: true, sortids: "502,501,503" })).toBe(false); + }); + + it("maps a full-list reference name through the container mapping", () => { + expect(same({ referencename: "Posts", fulllist: true }, { referencename: "poststarget", fulllist: true })).toBe( + true + ); + expect(same({ referencename: "Other", fulllist: true }, { referencename: "other", fulllist: true })).toBe(false); + }); + + it("marks an unmapped source ID so it can never match", () => { + const { source } = contexts(); + expect(canonicalValue({ contentid: 99 }, source)).toEqual({ "@ids": ["unmapped:99"] }); + }); + + it("reads an empty selection as empty", () => { + expect(same({ fulllist: false }, undefined)).toBe(true); + }); + }); + + describe("asset attachments", () => { + it("compares url and label only", () => { + expect( + same( + { url: `${SRC_EDGE}/a.png`, label: "A", filesize: 10, height: 5, width: 5 }, + { url: `${TGT_EDGE}/a.png`, label: "A", filesize: 11, pixelHeight: "5" } + ) + ).toBe(true); + expect(same({ url: `${SRC_EDGE}/a.png`, label: "A" }, { url: `${TGT_EDGE}/a.png`, label: "B" })).toBe(false); + }); + + it("uses the exact mapping first, then the container prefix", () => { + expect(same({ url: `${SRC_CUSTOM}/img/exact.png` }, { url: `${TGT_CUSTOM}/img/renamed.png` })).toBe(true); + expect(same({ url: `${SRC_CUSTOM}/img/other.png` }, { url: `${TGT_CUSTOM}/img/other.png` })).toBe(true); + }); + + it("treats every alias of the target container as the same host", () => { + expect(same({ url: `${SRC_EDGE}/b.png` }, { url: `${TGT_ORIGIN}/b.png` })).toBe(true); + }); + }); + + describe("rich text", () => { + it("rewrites mapped source asset URLs inside text", () => { + expect( + same( + ` and `, + ` and ` + ) + ).toBe(true); + }); + + it("prefers the longest matching URL", () => { + const t = translator(); + expect(t.rewriteSourceText(`${SRC_CUSTOM}/img/exact.png`)).toBe( + t.canonicalTargetText(`${TGT_CUSTOM}/img/renamed.png`) + ); + }); + + it("does not match a container whose name merely starts with a known one", () => { + const t = translator(); + expect(t.rewriteSourceText(`${SRC_EDGE}-old/a.png`)).toBe(`${SRC_EDGE}-old/a.png`); + }); + + it("finds source-instance asset URLs left in target text", () => { + const t = translator(); + expect(t.findSourceUrls(``)).toEqual([ + `${SRC_EDGE}/left.png`, + ]); + }); + + it("equates CRLF and LF in text", () => { + expect(same("

a

\r\n

b

", "

a

\n

b

")).toBe(true); + }); + }); + + it("maps a PhotoGallery ID through the gallery mapping", () => { + expect(same(3, "1003", "PhotoGallery")).toBe(true); + }); +}); + +describe("canonicalFields", () => { + it("matches field keys case-insensitively", () => { + const { source, target } = contexts(); + expect(canonicalFields({ Title: "a" }, null, source)).toEqual(canonicalFields({ title: "a" }, null, target)); + }); + + it("treats a model's companion value field as a content ID list", () => { + const { source, target } = contexts(); + const model = { + fields: [{ name: "Game", type: "Content", settings: { LinkeContentDropdownValueField: "Game_ValueField" } }], + }; + const s = canonicalFields({ game: { contentid: 1 }, game_valuefield: "1,2" }, model, source); + const t = canonicalFields({ game: { contentid: 501 }, Game_ValueField: "501,502" }, model, target); + expect(s).toEqual(t); + expect(s.game_valuefield).toEqual({ "@ids": ["501", "502"] }); + }); +}); + +describe("diffCanonical / describeDifference", () => { + it("lists only the differing keys", () => { + expect(diffCanonical({ a: "1", b: "x" }, { a: "1", b: "y", c: "z" })).toEqual([ + { field: "b", expected: "x", actual: "y" }, + { field: "c", expected: undefined, actual: "z" }, + ]); + }); + + it("cuts long strings to a window around the first difference", () => { + const prefix = "a".repeat(500); + const { expected, actual } = describeDifference(`${prefix}XYZ`, `${prefix}XQZ`); + expect(expected.length).toBeLessThan(200); + expect(expected).toContain("XYZ"); + expect(actual).toContain("XQZ"); + }); +}); diff --git a/src/lib/verify/tests/check-content.test.ts b/src/lib/verify/tests/check-content.test.ts new file mode 100644 index 0000000..863e9e6 --- /dev/null +++ b/src/lib/verify/tests/check-content.test.ts @@ -0,0 +1,308 @@ +import { checkContent } from "../check-content"; +import { checkLocaleIntegrity } from "../check-mapping-integrity"; +import { FindingCollector } from "../verify-report"; +import { context, item, itemMappings, LOCALE, POST_MODEL_SOURCE, POST_MODEL_TARGET } from "./fixtures"; + +const MAPPINGS = { + models: [{ sourceID: 10, targetID: 110, sourceReferenceName: "Post", targetReferenceName: "Post" }], + containers: [ + { sourceContentViewID: 20, targetContentViewID: 120, sourceReferenceName: "posts", targetReferenceName: "posts" }, + ], +}; +const SOURCE_CONTAINERS = [{ contentViewID: 20, referenceName: "posts", contentDefinitionID: 10 }]; +const TARGET_CONTAINERS = [ + { contentViewID: 120, referenceName: "posts", contentDefinitionID: 110 }, + { contentViewID: 121, referenceName: "other", contentDefinitionID: 110 }, +]; + +function run(source: any[], target: any[], pairs: Array<[number, number]>, opts: { scoped?: boolean } = {}) { + const ctx = context({ + source: { models: [POST_MODEL_SOURCE], containers: SOURCE_CONTAINERS, content: source }, + target: { models: [POST_MODEL_TARGET], containers: TARGET_CONTAINERS, content: target }, + mappings: { ...MAPPINGS, locales: { [LOCALE]: { item: itemMappings(pairs) } } }, + scoped: opts.scoped, + }); + const collector = new FindingCollector(); + const { content } = checkLocaleIntegrity(ctx, collector); + checkContent(ctx, content, collector); + return collector; +} + +const byCategory = (collector: FindingCollector, category: string) => + collector.findings.filter((f) => f.category === category); + +describe("checkContent", () => { + it("ignores contentID, versionID and modified", () => { + const collector = run( + [item(1, { title: "A" }, { versionID: 5, modified: "2020-01-01" })], + [item(101, { title: "A" }, { versionID: 999, modified: "2026-09-09" })], + [[1, 101]] + ); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(LOCALE).content).toMatchObject({ checked: 1, matched: 1 }); + }); + + it("reports each differing field", () => { + const collector = run( + [item(1, { title: "A", body: "

x

" })], + [item(101, { title: "B", body: "

x

" })], + [[1, 101]] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ + category: "fieldMismatch", + path: "title", + expected: "A", + actual: "B", + sourceID: 1, + targetID: 101, + }), + ]); + expect(collector.talliesFor(LOCALE).content.mismatched).toBe(1); + }); + + it("translates linked content IDs before comparing", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B", related: { contentid: 1, fulllist: false } })], + [item(101, { title: "A" }), item(102, { title: "B", related: "101" })], + [ + [1, 101], + [2, 102], + ] + ); + expect(collector.findings).toEqual([]); + }); + + it("warns, without failing, on a publish-state difference", () => { + const collector = run( + [item(1, { title: "A" }, { state: 2 })], + [item(101, { title: "A" }, { state: 1 })], + [[1, 101]] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "publishStateDifference", severity: "warning", expected: 2, actual: 1 }), + ]); + }); + + it("reports an item in the wrong container", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A" }, { referenceName: "other" })], + [[1, 101]] + ); + expect(byCategory(collector, "contentContainerMismatch")).toHaveLength(1); + }); + + describe("counts", () => { + it("fails when the target container has fewer items", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B" })], + [item(101, { title: "A" })], + [[1, 101]] + ); + const counts = byCategory(collector, "countMismatch"); + expect(counts.map((f) => [f.entity, f.severity, f.expected, f.actual])).toEqual([ + ["container", "failure", 2, 1], + ["model", "failure", 2, 1], + ]); + expect(byCategory(collector, "unmapped")).toHaveLength(1); + }); + + it("only warns when the target has more", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A" }), item(102, { title: "X" })], + [[1, 101]] + ); + expect(byCategory(collector, "countMismatch").every((f) => f.severity === "warning")).toBe(true); + expect(byCategory(collector, "extraInTarget")).toHaveLength(1); + }); + + it("is skipped on a scoped run", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B" })], + [item(101, { title: "A" })], + [[1, 101]], + { + scoped: true, + } + ); + expect(byCategory(collector, "countMismatch")).toEqual([]); + }); + }); + + describe("list order", () => { + const source = [ + item(1, { title: "A" }, { itemOrder: 0 }), + item(2, { title: "B" }, { itemOrder: 1 }), + item(3, { title: "C" }, { itemOrder: 2 }), + ]; + const pairs: Array<[number, number]> = [ + [1, 101], + [2, 102], + [3, 103], + ]; + + it("passes when the target keeps the order", () => { + const target = [ + item(101, { title: "A" }, { itemOrder: 5 }), + item(102, { title: "B" }, { itemOrder: 6 }), + item(103, { title: "C" }, { itemOrder: 7 }), + ]; + expect(run(source, target, pairs).findings).toEqual([]); + }); + + it("reports a reordered list with both sequences", () => { + const target = [ + item(101, { title: "A" }, { itemOrder: 2 }), + item(102, { title: "B" }, { itemOrder: 1 }), + item(103, { title: "C" }, { itemOrder: 3 }), + ]; + expect(byCategory(run(source, target, pairs), "listOrderMismatch")).toEqual([ + expect.objectContaining({ + entity: "container", + name: "posts", + expected: [101, 102, 103], + actual: [102, 101, 103], + }), + ]); + }); + + it("lets items that tie in the source come in either order", () => { + const tied = source.map((i) => ({ ...i, properties: { ...i.properties, itemOrder: 0 } })); + const target = [ + item(101, { title: "A" }, { itemOrder: 3 }), + item(102, { title: "B" }, { itemOrder: 2 }), + item(103, { title: "C" }, { itemOrder: 1 }), + ]; + expect(byCategory(run(tied, target, pairs), "listOrderMismatch")).toEqual([]); + }); + }); + + describe("unpublished (state 7) items", () => { + it("drops a state-7 item on either side, as a full pull would", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "Gone" }, { state: 7 })], + [item(101, { title: "A" }), item(102, { title: "Gone" }, { state: 7 })], + [ + [1, 101], + [2, 102], + ] + ); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(LOCALE).content.checked).toBe(1); + }); + + it("unpublished in the source only: the target copy is a warning, not a failure", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A" }), item(102, { title: "Gone" })], + [ + [1, 101], + [2, 102], + ] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "sourceNotInPull", severity: "warning", targetID: 102 }), + ]); + }); + + it("unpublished in the target only: the source item is reported missing", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { title: "B" })], + [item(101, { title: "A" }), item(102, { title: "B" }, { state: 7 })], + [ + [1, 101], + [2, 102], + ] + ); + expect(byCategory(collector, "danglingMapping")).toEqual([ + expect.objectContaining({ severity: "failure", sourceID: 2, targetID: 102 }), + ]); + }); + }); + + it("treats a link to an empty list as no link", () => { + const collector = run( + [item(1, { title: "A" })], + [item(101, { title: "A", related: { referencename: "posts_related_auto", fulllist: true } })], + [[1, 101]] + ); + expect(collector.findings).toEqual([]); + }); + + it("leaves out items whose container is deleted in the source, listing them once per container", () => { + const collector = run( + [ + item(1, { title: "A" }), + item(2, { title: "x" }, { referenceName: "old_deleted" }), + item(3, {}, { referenceName: "old_deleted" }), + ], + [item(101, { title: "A" })], + [[1, 101]] + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ + category: "contentNotInLiveContainer", + severity: "sourceDefect", + name: "old_deleted", + actual: 2, + }), + ]); + expect(collector.talliesFor(LOCALE).content).toMatchObject({ checked: 3, matched: 1, known: 2, missing: 0 }); + }); + + it("flags a source asset URL left in target content", () => { + const sourceAsset = { mediaID: 1, containerEdgeUrl: "https://cdn/src-c", edgeUrl: "https://cdn/src-c/a.png" }; + const targetAsset = { mediaID: 2, containerEdgeUrl: "https://cdn/tgt-c", edgeUrl: "https://cdn/tgt-c/a.png" }; + const ctx = context({ + source: { + models: [POST_MODEL_SOURCE], + containers: SOURCE_CONTAINERS, + assets: [sourceAsset], + content: [item(1, { body: '' })], + }, + target: { + models: [POST_MODEL_TARGET], + containers: TARGET_CONTAINERS, + assets: [targetAsset], + content: [item(101, { body: '' })], + }, + mappings: { + ...MAPPINGS, + assets: [{ sourceMediaID: 1, targetMediaID: 2 }], + locales: { [LOCALE]: { item: itemMappings([[1, 101]]) } }, + }, + }); + const collector = new FindingCollector(); + checkContent(ctx, checkLocaleIntegrity(ctx, collector).content, collector); + expect(collector.findings.map((f) => f.category).sort()).toEqual(["fieldMismatch", "sourceAssetUrlInTarget"]); + }); +}); + +describe("source defects", () => { + it("classifies a missing target copy of an item lacking a required field as a source defect", () => { + const collector = run( + [item(1, { title: "A" }), item(2, { body: "no title" })], + [item(101, { title: "A" })], + [[1, 101]] + ); + const unmapped = byCategory(collector, "unmapped"); + expect(unmapped).toEqual([expect.objectContaining({ severity: "sourceDefect", sourceID: 2 })]); + expect(unmapped[0].message).toContain("Title (Text)"); + expect(collector.talliesFor(LOCALE).content).toMatchObject({ known: 1, missing: 0 }); + }); + + it("classifies a field difference on such an item as a source defect", () => { + const collector = run([item(1, { body: "x" })], [item(101, { body: "y" })], [[1, 101]]); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "fieldMismatch", severity: "sourceDefect" }), + ]); + expect(collector.talliesFor(LOCALE).content.known).toBe(1); + }); + + it("keeps an ordinary difference a failure", () => { + const collector = run([item(1, { title: "A", body: "x" })], [item(101, { title: "A", body: "y" })], [[1, 101]]); + expect(collector.findings).toEqual([expect.objectContaining({ category: "fieldMismatch", severity: "failure" })]); + }); +}); diff --git a/src/lib/verify/tests/check-page-zones.test.ts b/src/lib/verify/tests/check-page-zones.test.ts new file mode 100644 index 0000000..05c7926 --- /dev/null +++ b/src/lib/verify/tests/check-page-zones.test.ts @@ -0,0 +1,231 @@ +import { checkPages } from "../check-page-zones"; +import { checkLocaleIntegrity } from "../check-mapping-integrity"; +import { FindingCollector } from "../verify-report"; +import { context, item, itemMappings, LOCALE, module, page, pageMappings, section } from "./fixtures"; + +const SOURCE_TEMPLATE = { + pageTemplateID: 1, + pageTemplateName: "Standard", + contentSectionDefinitions: [section(11, "Main", 1), section(12, "Sidebar", 2)], +}; +const TARGET_TEMPLATE = { + pageTemplateID: 101, + pageTemplateName: "Standard", + contentSectionDefinitions: [section(111, "Main", 1), section(112, "Sidebar", 2)], +}; + +const MAPPINGS_BASE = { + models: [ + { sourceID: 1, targetID: 101, sourceReferenceName: "Hero", targetReferenceName: "Hero" }, + { sourceID: 2, targetID: 102, sourceReferenceName: "Text", targetReferenceName: "Text" }, + ], + templates: [ + { + sourcePageTemplateID: 1, + targetPageTemplateID: 101, + sourcePageTemplateName: "Standard", + targetPageTemplateName: "Standard", + }, + ], + sections: [ + { sourcePageItemTemplateID: 11, targetPageItemTemplateID: 111 }, + { sourcePageItemTemplateID: 12, targetPageItemTemplateID: 112 }, + ], +}; + +const SOURCE_CONTENT = [item(1, {}), item(2, {}), item(3, {})]; +const TARGET_CONTENT = [item(501, {}), item(502, {}), item(503, {})]; +const CONTENT_MAP = itemMappings([ + [1, 501], + [2, 502], + [3, 503], +]); + +function run(input: { + sourceZones: Record; + targetZones: Record; + sourceTemplate?: any; + targetTemplate?: any; + sections?: any[]; + sourcePage?: Record; + targetPage?: Record; + sourceContent?: any[]; +}) { + const ctx = context({ + source: { + templates: [input.sourceTemplate ?? SOURCE_TEMPLATE], + content: input.sourceContent ?? SOURCE_CONTENT, + pages: [page(10, input.sourceZones, input.sourcePage)], + }, + target: { + templates: [input.targetTemplate ?? TARGET_TEMPLATE], + content: TARGET_CONTENT, + pages: [page(90, input.targetZones, input.targetPage)], + }, + mappings: { + ...MAPPINGS_BASE, + sections: input.sections ?? MAPPINGS_BASE.sections, + locales: { [LOCALE]: { item: CONTENT_MAP, page: pageMappings([[10, 90]]) } }, + }, + }); + const collector = new FindingCollector(); + const pairs = checkLocaleIntegrity(ctx, collector); + checkPages(ctx, pairs.pages, collector); + return { findings: collector.findings, tallies: collector.talliesFor(LOCALE), collector }; +} + +const categories = (findings: any[]) => findings.map((f) => f.category).sort(); + +describe("checkPages — zones", () => { + it("reports nothing for identical pages", () => { + const { findings, tallies } = run({ + sourceZones: { Main: [module("Hero", 1), module("Text", 2)], Sidebar: [module("Text", 3)] }, + targetZones: { Main: [module("Hero", 501), module("Text", 502)], Sidebar: [module("Text", 503)] }, + }); + expect(findings).toEqual([]); + expect(tallies.zone).toEqual({ checked: 2, matched: 2, mismatched: 0, missing: 0, extra: 0, known: 0 }); + expect(tallies.page.matched).toBe(1); + }); + + it("reports swapped components as an order mismatch with both sequences", () => { + const { findings } = run({ + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { Main: [module("Text", 502), module("Hero", 501)] }, + }); + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + category: "componentOrderMismatch", + severity: "failure", + entity: "zone", + sourceID: 10, + targetID: 90, + path: "main", + expected: ["hero#501", "text#502"], + actual: ["text#502", "hero#501"], + }); + }); + + it("reports a component placed in another zone", () => { + const { findings } = run({ + sourceZones: { Main: [module("Hero", 1)], Sidebar: [module("Text", 3)] }, + targetZones: { Main: [module("Hero", 501), module("Text", 503)], Sidebar: [] }, + }); + expect(categories(findings)).toEqual(["componentInWrongZone"]); + expect(findings[0]).toMatchObject({ expected: "text#503 in sidebar", actual: "text#503 in main" }); + }); + + it("reports missing and extra components", () => { + const { findings } = run({ + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { Main: [module("Hero", 501), module("Text", 503)] }, + }); + expect(categories(findings)).toEqual(["extraComponent", "missingComponent"]); + }); + + it("pairs a renamed zone through the section-ID mapping", () => { + const targetTemplate = { + ...TARGET_TEMPLATE, + contentSectionDefinitions: [section(111, "MainContent", 1), section(112, "Sidebar", 2)], + }; + const { findings } = run({ + targetTemplate, + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { MainContent: [module("Hero", 501), module("Text", 502)] }, + }); + expect(findings).toEqual([]); + }); + + describe("PROD-2350: zones defined in a different order on the two templates", () => { + const targetTemplate = { + ...TARGET_TEMPLATE, + // Same zones, opposite itemOrder — a positional pairing would cross them over. + contentSectionDefinitions: [section(112, "Sidebar", 1), section(111, "Main", 2)], + }; + + it("pairs each zone by section ID, not position", () => { + const { findings } = run({ + targetTemplate, + sourceZones: { Main: [module("Hero", 1)], Sidebar: [module("Text", 3)] }, + targetZones: { Sidebar: [module("Text", 503)], Main: [module("Hero", 501)] }, + }); + expect(findings).toEqual([]); + }); + + it("catches content a positional pairing would have crossed over", () => { + const { findings } = run({ + targetTemplate, + sourceZones: { Main: [module("Hero", 1)], Sidebar: [module("Text", 3)] }, + targetZones: { Sidebar: [module("Hero", 501)], Main: [module("Text", 503)] }, + }); + expect(categories(findings)).toEqual(["componentInWrongZone", "componentInWrongZone"]); + }); + }); + + it("reports an unmapped zone and never falls back to position", () => { + const sourceTemplate = { + ...SOURCE_TEMPLATE, + contentSectionDefinitions: [section(11, "Main", 1), section(13, "Legacy", 2)], + }; + const targetTemplate = { + ...TARGET_TEMPLATE, + contentSectionDefinitions: [section(111, "Main", 1), section(114, "Other", 2)], + }; + const { findings } = run({ + sourceTemplate, + targetTemplate, + sections: [{ sourcePageItemTemplateID: 11, targetPageItemTemplateID: 111 }], + sourceZones: { Main: [module("Hero", 1)], Legacy: [module("Text", 2)] }, + // The leftover target zone holds what a positional fallback would have put there. + targetZones: { Main: [module("Hero", 501)], Other: [module("Text", 502)] }, + }); + expect(categories(findings)).toEqual(["unmappedZone"]); + expect(findings[0]).toMatchObject({ severity: "failure", path: "Legacy", expected: ["text#502"] }); + }); + + it("lists a module whose content is missing from the source pull as a source defect", () => { + const { findings, tallies } = run({ + sourceContent: [item(1, {})], + sourceZones: { Main: [module("Hero", 1), module("Text", 2)] }, + targetZones: { Main: [module("Hero", 501)] }, + }); + // The target copies of 2 and 3 are reported by mapping integrity (sourceNotInPull warnings). + const zoneFindings = findings.filter((f) => f.entity === "zone"); + expect(zoneFindings).toHaveLength(1); + expect(zoneFindings[0]).toMatchObject({ category: "moduleContentMissingInSource", severity: "sourceDefect" }); + expect(findings.some((f) => f.severity === "failure")).toBe(false); + expect(tallies.zone.known).toBe(1); + }); +}); + +describe("checkPages — page fields", () => { + it("compares the listed fields with normalization", () => { + const { findings } = run({ + sourceZones: {}, + targetZones: {}, + sourcePage: { redirectUrl: "", seo: { metaDescription: "d\r\nx" } }, + targetPage: { redirectUrl: null, seo: { MetaDescription: "d\nx", metaKeywords: null } }, + }); + expect(findings).toEqual([]); + }); + + it("reports a differing title and template", () => { + const { findings } = run({ + sourceZones: {}, + targetZones: {}, + targetPage: { title: "Other", templateName: "Different" }, + }); + expect(findings.map((f) => [f.category, f.path])).toEqual([ + ["pageFieldMismatch", "title"], + ["pageFieldMismatch", "template"], + ]); + }); + + it("compares template names through the template mapping, case-insensitively", () => { + const { findings } = run({ + sourceZones: {}, + targetZones: {}, + sourcePage: { templateName: "STANDARD" }, + }); + expect(findings).toEqual([]); + }); +}); diff --git a/src/lib/verify/tests/check-sitemap.test.ts b/src/lib/verify/tests/check-sitemap.test.ts new file mode 100644 index 0000000..8627534 --- /dev/null +++ b/src/lib/verify/tests/check-sitemap.test.ts @@ -0,0 +1,144 @@ +import { checkSitemap, flattenSitemap } from "../check-sitemap"; +import { FindingCollector } from "../verify-report"; +import { context, LOCALE, page, pageMappings } from "./fixtures"; + +function node(pageID: number, path: string, children?: any[], extra: Record = {}): any { + return { + title: path, + name: path.split("/").pop(), + pageID, + menuText: path, + visible: { menu: true, sitemap: true }, + path, + redirect: null, + isFolder: false, + ...(children ? { children } : {}), + ...extra, + }; +} + +// Source: /home, /about (→ /about/team, /about/jobs), /contact. Target IDs are source + 100. +const SOURCE_TREE = [ + node(1, "/home"), + node(2, "/about", [node(3, "/about/team"), node(4, "/about/jobs")]), + node(5, "/contact"), +]; +const MAP = pageMappings([ + [1, 101], + [2, 102], + [3, 103], + [4, 104], + [5, 105], +]); + +function run( + targetTree: any[] | null, + opts: { channels?: Record; targetChannels?: Record; only?: string[] } = {} +) { + const ids = [1, 2, 3, 4, 5]; + const ctx = context({ + source: { pages: ids.map((id) => page(id, {})), sitemaps: opts.channels ?? { website: SOURCE_TREE } }, + target: { + pages: ids.map((id) => page(id + 100, {})), + sitemaps: opts.targetChannels ?? { website: targetTree as any[] }, + }, + mappings: { locales: { [LOCALE]: { page: MAP } } }, + channels: opts.only, + }); + const collector = new FindingCollector(); + checkSitemap(ctx, collector); + return collector; +} + +const translate = (nodes: any[]): any[] => + nodes.map((n) => node(n.pageID + 100, n.path, n.children ? translate(n.children) : undefined)); + +describe("flattenSitemap", () => { + it("records parents and child order, skipping dynamic-item nodes", () => { + const flat = flattenSitemap([node(1, "/a", [node(2, "/a/b"), node(1, "/a/item", undefined, { contentID: 9 })])]); + expect(flat.placement.get(2)?.parent).toBe(1); + expect(flat.children.get(0)).toEqual([1]); + expect(flat.children.get(1)).toEqual([2]); + }); +}); + +describe("checkSitemap", () => { + it("reports nothing for an identical structure", () => { + const collector = run(translate(SOURCE_TREE)); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(LOCALE).sitemap).toMatchObject({ checked: 5, matched: 5 }); + }); + + it("reports one siblingOrderMismatch per parent with both sequences", () => { + const target = [ + node(101, "/home"), + node(102, "/about", [node(104, "/about/jobs"), node(103, "/about/team")]), + node(105, "/contact"), + ]; + const findings = run(target).findings; + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + category: "siblingOrderMismatch", + sourceID: 2, + targetID: 102, + expected: [103, 104], + actual: [104, 103], + }); + }); + + it("reports a reordered root level", () => { + const target = [ + node(105, "/contact"), + node(101, "/home"), + node(102, "/about", [node(103, "/a/t"), node(104, "/a/j")]), + ]; + const findings = run(target).findings; + expect(findings.map((f) => f.category)).toEqual(["siblingOrderMismatch"]); + expect(findings[0]).toMatchObject({ name: "(root)", expected: [101, 102, 105], actual: [105, 101, 102] }); + }); + + it("reports a reparented page", () => { + const target = [ + node(101, "/home", [node(103, "/home/team")]), + node(102, "/about", [node(104, "/about/jobs")]), + node(105, "/contact"), + ]; + const findings = run(target).findings; + expect(findings.map((f) => f.category)).toEqual(["parentMismatch"]); + expect(findings[0]).toMatchObject({ sourceID: 3, targetID: 103, expected: 102, actual: 101 }); + }); + + it("reports a mapped page missing from the target sitemap", () => { + const target = [node(101, "/home"), node(102, "/about", [node(103, "/about/team")]), node(105, "/contact")]; + const findings = run(target).findings; + expect(findings.map((f) => f.category)).toEqual(["pageMissingFromSitemap"]); + expect(findings[0]).toMatchObject({ sourceID: 4, targetID: 104 }); + }); + + describe("multiple channels", () => { + const channels = { website: SOURCE_TREE, mobile: [node(5, "/contact"), node(1, "/home")] }; + + it("compares each channel against the same channel on the target", () => { + const findings = run(null, { + channels, + targetChannels: { website: translate(SOURCE_TREE), mobile: [node(101, "/home"), node(105, "/contact")] }, + }).findings; + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ category: "siblingOrderMismatch", path: "mobile:/" }); + }); + + it("reports pages of a channel the target lacks", () => { + const findings = run(null, { channels, targetChannels: { website: translate(SOURCE_TREE) } }).findings; + expect(findings.map((f) => f.category)).toEqual(["pageMissingFromSitemap", "pageMissingFromSitemap"]); + }); + + it("checks only the requested channel when one is given", () => { + const findings = run(null, { + channels, + targetChannels: { website: translate(SOURCE_TREE) }, + only: ["Website"], + }).findings; + expect(findings).toEqual([]); + }); + }); +}); diff --git a/src/lib/verify/tests/check-structure.test.ts b/src/lib/verify/tests/check-structure.test.ts new file mode 100644 index 0000000..c2f0801 --- /dev/null +++ b/src/lib/verify/tests/check-structure.test.ts @@ -0,0 +1,220 @@ +import { checkInstanceIntegrity } from "../check-mapping-integrity"; +import { checkStructure } from "../check-structure"; +import { FindingCollector, INSTANCE_LOCALE } from "../verify-report"; +import { context, section } from "./fixtures"; + +function run(source: any, target: any, mappings: any) { + const ctx = context({ source, target, mappings }); + const collector = new FindingCollector(); + checkStructure(ctx, checkInstanceIntegrity(ctx, collector), collector); + return collector; +} + +const MODEL_MAP = [{ sourceID: 1, targetID: 11, sourceReferenceName: "Post", targetReferenceName: "Post" }]; +const fields = (...pairs: Array<[string, string]>) => + pairs.map(([name, type]) => ({ name, type, lastModifiedDate: "x" })); + +describe("models", () => { + it("compares reference name and ordered (name, type) fields, ignoring IDs and dates", () => { + const collector = run( + { + models: [ + { id: 1, referenceName: "Post", lastModifiedDate: "a", fields: fields(["Title", "Text"], ["Body", "HTML"]) }, + ], + }, + { + models: [ + { id: 11, referenceName: "post", lastModifiedDate: "b", fields: fields(["title", "Text"], ["body", "Html"]) }, + ], + }, + { models: MODEL_MAP } + ); + expect(collector.findings).toEqual([]); + expect(collector.talliesFor(INSTANCE_LOCALE).model).toMatchObject({ checked: 1, matched: 1 }); + }); + + it("reports reordered fields", () => { + const collector = run( + { models: [{ id: 1, referenceName: "Post", fields: fields(["Title", "Text"], ["Body", "HTML"]) }] }, + { models: [{ id: 11, referenceName: "Post", fields: fields(["Body", "HTML"], ["Title", "Text"]) }] }, + { models: MODEL_MAP } + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "modelMismatch", path: "fields", expected: ["title:text", "body:html"] }), + ]); + }); + + it("warns on a target model nothing maps to", () => { + const collector = run( + { models: [{ id: 1, referenceName: "Post", fields: [] }] }, + { + models: [ + { id: 11, referenceName: "Post", fields: [] }, + { id: 12, referenceName: "Extra", fields: [] }, + ], + }, + { models: MODEL_MAP } + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "extraInTarget", severity: "warning", targetID: 12 }), + ]); + }); +}); + +describe("containers", () => { + it("compares the held model through the model mapping", () => { + const mappings = { + models: MODEL_MAP, + containers: [ + { sourceContentViewID: 5, targetContentViewID: 55, sourceReferenceName: "posts", targetReferenceName: "posts" }, + ], + }; + const source = { + models: [{ id: 1, referenceName: "Post", fields: [] }], + containers: [{ contentViewID: 5, referenceName: "posts", contentDefinitionID: 1 }], + }; + const ok = run( + source, + { + models: [{ id: 11, referenceName: "Post", fields: [] }], + containers: [{ contentViewID: 55, referenceName: "posts", contentDefinitionID: 11 }], + }, + mappings + ); + expect(ok.findings).toEqual([]); + + const wrong = run( + source, + { + models: [ + { id: 11, referenceName: "Post", fields: [] }, + { id: 12, referenceName: "Other", fields: [] }, + ], + containers: [{ contentViewID: 55, referenceName: "posts", contentDefinitionID: 12 }], + }, + mappings + ); + expect(wrong.findings.filter((f) => f.category === "containerMismatch")).toEqual([ + expect.objectContaining({ path: "model", actual: "Other (#12)" }), + ]); + }); + + it("leaves out the fixed Agility system containers on both sides", () => { + const collector = run( + { containers: [{ contentViewID: 1, referenceName: "AgilityCSSFiles", contentDefinitionID: 3 }] }, + { containers: [{ contentViewID: 1, referenceName: "AgilityCSSFiles", contentDefinitionID: 3 }] }, + {} + ); + expect(collector.findings).toEqual([]); + }); + + it("leaves out deleted containers", () => { + const collector = run({ containers: [{ contentViewID: 9, referenceName: "gone", isDeleted: true }] }, {}, {}); + expect(collector.findings).toEqual([]); + }); +}); + +describe("templates", () => { + const mappings = { + templates: [ + { + sourcePageTemplateID: 1, + targetPageTemplateID: 2, + sourcePageTemplateName: "Std", + targetPageTemplateName: "Std", + }, + ], + }; + + it("compares section reference names in itemOrder", () => { + const collector = run( + { + templates: [ + { + pageTemplateID: 1, + pageTemplateName: "Std", + contentSectionDefinitions: [section(1, "Main", 1), section(2, "Side", 2)], + }, + ], + }, + { + templates: [ + { + pageTemplateID: 2, + pageTemplateName: "Std", + contentSectionDefinitions: [section(9, "Side", 2), section(8, "main", 1)], + }, + ], + }, + mappings + ); + expect(collector.findings).toEqual([]); + }); + + it("reports a differing zone list", () => { + const collector = run( + { + templates: [ + { + pageTemplateID: 1, + pageTemplateName: "Std", + contentSectionDefinitions: [section(1, "Main", 1), section(2, "Side", 2)], + }, + ], + }, + { + templates: [{ pageTemplateID: 2, pageTemplateName: "Std", contentSectionDefinitions: [section(8, "Main", 1)] }], + }, + mappings + ); + expect(collector.findings).toEqual([ + expect.objectContaining({ category: "templateMismatch", expected: ["main", "side"], actual: ["main"] }), + ]); + }); +}); + +describe("mapping integrity", () => { + it("reports unmapped, dangling and duplicate-target mappings", () => { + const collector = run( + { + models: [ + { id: 1, referenceName: "A", fields: [] }, + { id: 2, referenceName: "B", fields: [] }, + { id: 3, referenceName: "C", fields: [] }, + { id: 4, referenceName: "D", fields: [] }, + ], + }, + { models: [{ id: 11, referenceName: "A", fields: [] }] }, + { + models: [ + { sourceID: 1, targetID: 11 }, + { sourceID: 2, targetID: 11 }, + { sourceID: 3, targetID: 13 }, + ], + } + ); + const summary = collector.findings.map((f) => [f.category, f.sourceID, f.targetID]); + expect(summary).toEqual( + expect.arrayContaining([ + ["danglingMapping", 3, 13], + ["unmapped", 4, null], + ["duplicateTarget", "1,2", 11], + ]) + ); + expect(collector.talliesFor(INSTANCE_LOCALE).model).toMatchObject({ checked: 4, missing: 2 }); + }); + + it("does not report a duplicate whose second source no longer exists", () => { + const collector = run( + { models: [{ id: 1, referenceName: "A", fields: [] }] }, + { models: [{ id: 11, referenceName: "A", fields: [] }] }, + { + models: [ + { sourceID: 1, targetID: 11 }, + { sourceID: 2, targetID: 11 }, + ], + } + ); + expect(collector.findings).toEqual([]); + }); +}); diff --git a/src/lib/verify/tests/fixtures.ts b/src/lib/verify/tests/fixtures.ts new file mode 100644 index 0000000..9b4f572 --- /dev/null +++ b/src/lib/verify/tests/fixtures.ts @@ -0,0 +1,127 @@ +/** + * Small hand-built fixtures for the verify checks. Not a test file (no `.test.ts`), so Jest only + * loads it through the tests that import it. + */ +import { buildMappingIndex, MappingIndex, MappingType } from "../mapping-index"; +import { createVerifyContext, SideEntities, VerifyContext } from "../verify-context"; + +export const LOCALE = "en-us"; +export const SRC = "src-guid"; +export const TGT = "tgt-guid"; + +export function emptySide(overrides: Partial = {}): SideEntities { + return { + models: [], + containers: [], + templates: [], + assets: [], + galleries: [], + content: [], + pages: [], + sitemaps: {}, + ...overrides, + }; +} + +export interface MappingFixture { + models?: any[]; + containers?: any[]; + templates?: any[]; + sections?: any[]; + galleries?: any[]; + assets?: any[]; + /** Per locale: item / page mapping records. */ + locales?: Record; +} + +export function mappingIndex(fixture: MappingFixture, locales: string[] = [LOCALE]): MappingIndex { + return buildMappingIndex((type: MappingType, locale?: string) => { + if (locale) return (fixture.locales?.[locale] as any)?.[type] ?? []; + return (fixture as any)[type] ?? []; + }, locales); +} + +export function context(input: { + source: Partial; + target: Partial; + mappings: MappingFixture; + locale?: string; + scoped?: boolean; + channels?: string[]; +}): VerifyContext { + const locale = input.locale ?? LOCALE; + return createVerifyContext({ + sourceGuid: SRC, + targetGuid: TGT, + locale, + mappings: mappingIndex(input.mappings, [locale]), + source: emptySide(input.source), + target: emptySide(input.target), + scoped: input.scoped, + channels: input.channels, + }); +} + +/** Content mapping records from [sourceID, targetID] pairs. */ +export function itemMappings(pairs: Array<[number, number]>): any[] { + return pairs.map(([s, t]) => ({ sourceGuid: SRC, targetGuid: TGT, sourceContentID: s, targetContentID: t })); +} + +export function pageMappings(pairs: Array<[number, number]>): any[] { + return pairs.map(([s, t]) => ({ sourceGuid: SRC, targetGuid: TGT, sourcePageID: s, targetPageID: t })); +} + +export function item(contentID: number, fields: any, props: Partial> = {}): any { + return { + contentID, + properties: { + state: 2, + modified: "2026-01-01T00:00:00", + versionID: contentID * 10, + referenceName: "posts", + definitionName: "Post", + itemOrder: 0, + ...props, + }, + fields, + }; +} + +export const POST_MODEL_SOURCE = { + id: 10, + referenceName: "Post", + fields: [ + { name: "Title", type: "Text", settings: { Required: "True" } }, + { name: "Body", type: "HTML", settings: { Required: "False" } }, + { name: "Related", type: "Content", settings: { Required: "False" } }, + ], +}; + +export const POST_MODEL_TARGET = { ...POST_MODEL_SOURCE, id: 110 }; + +export function module(name: string, contentID: number): any { + return { title: name, module: name, item: { contentid: contentID, fulllist: false } }; +} + +export function section(id: number, referenceName: string, itemOrder: number): any { + return { pageItemTemplateID: id, pageItemTemplateReferenceName: referenceName, itemOrder }; +} + +export function page(pageID: number, zones: Record, overrides: Record = {}): any { + return { + pageID, + name: "home", + title: "Home", + menuText: "Home", + pageType: "static", + templateName: "Standard", + redirectUrl: "", + securePage: false, + visible: { menu: true, sitemap: true }, + seo: { metaDescription: "", metaKeywords: "" }, + scripts: { excludedFromGlobal: false, top: "", bottom: "" }, + properties: { state: 2, modified: "2026-01-01T00:00:00", versionID: pageID * 10 }, + zones, + ...overrides, + }; +} diff --git a/src/lib/verify/tests/mapping-index.test.ts b/src/lib/verify/tests/mapping-index.test.ts new file mode 100644 index 0000000..9491019 --- /dev/null +++ b/src/lib/verify/tests/mapping-index.test.ts @@ -0,0 +1,152 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { BiMap, buildMappingIndex, createFileMappingReader } from "../mapping-index"; +import { resetState, setState } from "../../../core/state"; + +describe("BiMap", () => { + it("looks up both directions", () => { + const map = new BiMap(); + map.add(1, 101); + map.add(2, 102); + expect(map.target(1)).toBe(101); + expect(map.sources(102)).toEqual([2]); + expect(map.target(3)).toBeUndefined(); + expect(map.sources(999)).toEqual([]); + }); + + it("keeps the first target for a duplicated source, like the mappers' Array.find", () => { + const map = new BiMap(); + map.add(1, 101); + map.add(1, 201); + expect(map.target(1)).toBe(101); + expect(map.hasTarget(201)).toBe(false); + }); + + it("reports targets that more than one source maps onto", () => { + const map = new BiMap(); + map.add(1, 101); + map.add(2, 101); + map.add(3, 103); + expect(map.duplicateTargets()).toEqual([{ target: 101, sources: [1, 2] }]); + }); + + it("ignores unusable keys (missing, zero, negative)", () => { + const map = new BiMap(); + map.add(undefined, 1); + map.add(1, null); + map.add(2, -1); + map.add(0, 5); + expect(map.size).toBe(0); + }); +}); + +describe("buildMappingIndex", () => { + const records: Record = { + models: [{ sourceID: 1, targetID: 11, sourceReferenceName: "Post", targetReferenceName: "Post" }], + containers: [ + { sourceContentViewID: 5, targetContentViewID: 55, sourceReferenceName: "Posts", targetReferenceName: "PostsT" }, + ], + templates: [ + { + sourcePageTemplateID: 2, + targetPageTemplateID: 22, + sourcePageTemplateName: "Std", + targetPageTemplateName: "Std", + }, + ], + sections: [{ sourcePageItemTemplateID: 7, targetPageItemTemplateID: 77 }], + galleries: [{ sourceMediaGroupingID: 3, targetMediaGroupingID: 33 }], + assets: [ + { + sourceMediaID: 4, + targetMediaID: 44, + sourceUrl: "https://cdn/src/a.png", + targetUrl: "https://cdn/tgt/a.png", + sourceContainerEdgeUrl: "https://cdn/src", + targetContainerEdgeUrl: "https://cdn/tgt", + sourceContainerOriginUrl: "https://origin/src/", + targetContainerOriginUrl: "https://origin/tgt/", + }, + { + sourceMediaID: 6, + targetMediaID: 66, + sourceContainerEdgeUrl: "https://cdn/src", + targetContainerEdgeUrl: "https://cdn/tgt", + }, + ], + }; + const locales: Record> = { + "en-us": { + item: [{ sourceContentID: 100, targetContentID: 900 }], + page: [{ sourcePageID: 8, targetPageID: 88 }], + }, + "fr-ca": { item: [{ sourceContentID: 100, targetContentID: 901 }] }, + }; + + const index = buildMappingIndex( + (type, locale) => (locale ? (locales[locale]?.[type] ?? []) : (records[type] ?? [])), + ["en-us", "fr-ca"] + ); + + it("indexes every guid-level mapping type", () => { + expect(index.models.target(1)).toBe(11); + expect(index.modelNames.target("post")).toBe("post"); + expect(index.containers.target(5)).toBe(55); + expect(index.containerNames.target("posts")).toBe("postst"); + expect(index.templates.target(2)).toBe(22); + expect(index.templateNames.target("std")).toBe("std"); + expect(index.sections.target(7)).toBe(77); + expect(index.galleries.target(3)).toBe(33); + expect(index.assets.target(4)).toBe(44); + }); + + it("collects exact asset URLs and de-duplicated container prefixes without trailing slashes", () => { + expect(index.assetUrls.get("https://cdn/src/a.png")).toBe("https://cdn/tgt/a.png"); + expect(index.assetPrefixes).toEqual([ + { source: "https://cdn/src", target: "https://cdn/tgt" }, + { source: "https://origin/src", target: "https://origin/tgt" }, + ]); + }); + + it("keeps content and page mappings per locale", () => { + expect(index.byLocale.get("en-us")?.content.target(100)).toBe(900); + expect(index.byLocale.get("fr-ca")?.content.target(100)).toBe(901); + expect(index.byLocale.get("en-us")?.pages.sources(88)).toEqual([8]); + expect(index.byLocale.get("fr-ca")?.pages.size).toBe(0); + }); +}); + +describe("createFileMappingReader", () => { + let root: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "verify-mappings-")); + resetState(); + setState({ rootPath: root }); + }); + + afterEach(() => fs.rmSync(root, { recursive: true, force: true })); + + it("reads existing mapping files and never seeds a missing one", () => { + const pair = path.join(root, "mappings", "a-b"); + fs.mkdirSync(path.join(pair, "models"), { recursive: true }); + fs.writeFileSync(path.join(pair, "models", "mappings.json"), JSON.stringify([{ sourceID: 1, targetID: 2 }])); + // A type folder without its mappings.json: fileOperations.getMappingFile would write "[]" here. + fs.mkdirSync(path.join(pair, "containers"), { recursive: true }); + + const read = createFileMappingReader(root, "a", "b"); + + expect(read("models")).toEqual([{ sourceID: 1, targetID: 2 }]); + expect(read("containers")).toEqual([]); + expect(fs.existsSync(path.join(pair, "containers", "mappings.json"))).toBe(false); + expect(read("item", "en-us")).toEqual([]); + }); + + it("throws on a malformed mapping file", () => { + const dir = path.join(root, "mappings", "a-b", "models"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "mappings.json"), "{ not json"); + expect(() => createFileMappingReader(root, "a", "b")("models")).toThrow(); + }); +}); diff --git a/src/lib/verify/tests/verify-report.test.ts b/src/lib/verify/tests/verify-report.test.ts new file mode 100644 index 0000000..afcb696 --- /dev/null +++ b/src/lib/verify/tests/verify-report.test.ts @@ -0,0 +1,233 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + buildErrorReport, + buildVerifyReport, + computeExitCode, + defaultReportPath, + emptyRunInfo, + EXIT_CANNOT_VERIFY, + EXIT_DIFFERENCES, + EXIT_PASS, + FindingCollector, + MAX_DETAILS_PER_CATEGORY, + printVerifyReport, + writeVerifyReport, +} from "../verify-report"; + +const OPTIONS = { locales: ["en-us"], channel: "", models: "", pages: "", containers: "" }; + +function collectorWith(severities: Array<"failure" | "warning" | "sourceDefect">): FindingCollector { + const collector = new FindingCollector(); + severities.forEach((severity, i) => + collector.add({ category: `c${i}`, severity, entity: "content", locale: "en-us", sourceID: i, targetID: 100 + i }) + ); + return collector; +} + +describe("exit codes", () => { + it("is 0 with no findings", () => { + expect(computeExitCode([])).toBe(EXIT_PASS); + }); + + it("is 0 when there are only warnings and source defects", () => { + expect(computeExitCode(collectorWith(["warning", "sourceDefect"]).findings)).toBe(EXIT_PASS); + }); + + it("is 1 with any failure", () => { + expect(computeExitCode(collectorWith(["warning", "failure"]).findings)).toBe(EXIT_DIFFERENCES); + }); + + it("is 2 for a run that could not verify", () => { + const report = buildErrorReport({ source: "a", target: "b", options: OPTIONS, error: "boom", run: emptyRunInfo() }); + expect(report).toMatchObject({ verdict: "error", exitCode: EXIT_CANNOT_VERIFY, error: "boom", findings: [] }); + }); +}); + +describe("buildVerifyReport — golden (without the volatile run block)", () => { + it("serializes findings, per-locale tallies and totals", () => { + const collector = new FindingCollector(); + collector.count("*", "model", "checked", 2); + collector.count("*", "model", "matched", 2); + collector.count("en-us", "page", "checked"); + collector.count("en-us", "page", "mismatched"); + collector.count("en-us", "zone", "checked"); + collector.count("en-us", "zone", "mismatched"); + collector.add({ + category: "componentOrderMismatch", + severity: "failure", + entity: "zone", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "main", + expected: ["hero#1", "text#2"], + actual: ["text#2", "hero#1"], + }); + collector.add({ + category: "publishStateDifference", + severity: "warning", + entity: "page", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "state", + expected: 2, + actual: 1, + }); + + const report = buildVerifyReport({ + source: "src", + target: "tgt", + options: OPTIONS, + collector, + run: emptyRunInfo(), + }); + const { run, ...stable } = report; + + expect(run).toEqual(expect.objectContaining({ cliVersion: expect.any(String), startedAt: expect.any(String) })); + expect(JSON.parse(JSON.stringify(stable))).toEqual({ + schemaVersion: 1, + command: "verify", + verdict: "fail", + exitCode: 1, + source: "src", + target: "tgt", + options: { locales: ["en-us"], channel: "", models: "", pages: "", containers: "" }, + totals: { + failures: 1, + warnings: 1, + sourceDefects: 0, + byCategory: { componentOrderMismatch: 1, publishStateDifference: 1 }, + entities: { + model: { checked: 2, matched: 2, mismatched: 0, missing: 0, extra: 0, known: 0 }, + page: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + zone: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + }, + }, + byLocale: { + "*": { + failures: 0, + warnings: 0, + sourceDefects: 0, + entities: { model: { checked: 2, matched: 2, mismatched: 0, missing: 0, extra: 0, known: 0 } }, + }, + "en-us": { + failures: 1, + warnings: 1, + sourceDefects: 0, + entities: { + page: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + zone: { checked: 1, matched: 0, mismatched: 1, missing: 0, extra: 0, known: 0 }, + }, + }, + }, + findings: [ + { + category: "componentOrderMismatch", + severity: "failure", + entity: "zone", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "main", + expected: ["hero#1", "text#2"], + actual: ["text#2", "hero#1"], + }, + { + category: "publishStateDifference", + severity: "warning", + entity: "page", + locale: "en-us", + sourceID: 10, + targetID: 90, + name: "home", + path: "state", + expected: 2, + actual: 1, + }, + ], + }); + }); + + it("orders findings deterministically regardless of insertion order", () => { + const a = collectorWith(["warning", "failure", "sourceDefect"]); + const b = new FindingCollector(); + a.findings + .slice() + .reverse() + .forEach((f) => b.add(f)); + const run = emptyRunInfo(); + expect(buildVerifyReport({ source: "s", target: "t", options: OPTIONS, collector: b, run }).findings).toEqual( + buildVerifyReport({ source: "s", target: "t", options: OPTIONS, collector: a, run }).findings + ); + }); +}); + +describe("writeVerifyReport / defaultReportPath", () => { + it("writes the report, creating directories, and never throws", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "verify-report-")); + const file = path.join(dir, "nested", "report.json"); + const report = buildErrorReport({ source: "a", target: "b", options: OPTIONS, error: "x", run: emptyRunInfo() }); + expect(writeVerifyReport(file, report)).toEqual({ written: true }); + expect(JSON.parse(fs.readFileSync(file, "utf8")).command).toBe("verify"); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it("defaults to agility-files/logs/verify-{src}-{tgt}-{timestamp}.json", () => { + expect(defaultReportPath("agility-files", "s", "t", new Date("2026-10-07T12:00:00.000Z"))).toBe( + path.join("agility-files", "logs", "verify-s-t-2026-10-07T12-00-00-000Z.json") + ); + }); +}); + +describe("printVerifyReport", () => { + it("prints a table per locale, capped details with links, and a verdict line", () => { + const collector = new FindingCollector(); + collector.count("en-us", "content", "checked", 30); + for (let i = 0; i < MAX_DETAILS_PER_CATEGORY + 5; i++) { + collector.add({ + category: "fieldMismatch", + severity: "failure", + entity: "content", + locale: "en-us", + sourceID: i, + targetID: 500 + i, + path: "title", + expected: "a", + actual: "b", + }); + } + const lines: string[] = []; + printVerifyReport( + buildVerifyReport({ source: "s", target: "t", options: OPTIONS, collector, run: emptyRunInfo() }), + { + log: (l) => lines.push(l.replace(/\x1b\[[0-9;]*m/g, "")), + linkFor: (f) => `link-${f.targetID}`, + } + ); + const text = lines.join("\n"); + expect(text).toContain("Locale en-us: 25 failure(s)"); + expect(text).toMatch(/entity\s+checked\s+matched\s+mismatched\s+missing\s+extra\s+known/); + expect(text).toContain("FAILURE fieldMismatch (25)"); + expect(lines.filter((l) => l.includes("link-")).length).toBe(MAX_DETAILS_PER_CATEGORY); + expect(text).toContain("…and 5 more"); + expect(text).toContain("VERDICT: FAIL"); + }); + + it("prints the reason for a run that could not verify", () => { + const lines: string[] = []; + printVerifyReport( + buildErrorReport({ source: "s", target: "t", options: OPTIONS, error: "no auth", run: emptyRunInfo() }), + { + log: (l) => lines.push(l), + } + ); + expect(lines.join("\n")).toContain("no auth"); + expect(lines.join("\n")).toContain("exit 2"); + }); +}); diff --git a/src/lib/verify/verify-context.ts b/src/lib/verify/verify-context.ts new file mode 100644 index 0000000..d6742cf --- /dev/null +++ b/src/lib/verify/verify-context.ts @@ -0,0 +1,334 @@ +/** + * Everything one locale's verify checks need, loaded once (PROD-2552). + * + * Reads only: the pulled caches through GuidDataLoader / SitemapHierarchy, and the mapping index + * built by mapping-index.ts. Building the context is split from loading it so tests can hand in + * entities directly. + * + * Unpublished items: a full pull omits state-7 (unpublished) items, on both sides. Anything with + * state 7 that is still on disk (an older incremental pull) is dropped here, on BOTH sides, so the + * two sides are always compared on the same terms — what a fresh pull of each instance returns. + */ +import * as fs from "fs"; +import * as path from "path"; +import { GuidDataLoader, GuidEntities, ModelFilterOptions } from "../pushers/guid-data-loader"; +import { SitemapHierarchy } from "../pushers/page-pusher/sitemap-hierarchy"; +import { SitemapNode } from "../../types/syncAnalysis"; +import { LocaleMappingIndex, MappingIndex } from "./mapping-index"; +import { AssetUrlTranslator, buildAssetUrlTranslator, CanonicalContext } from "./canonicalize"; + +export const UNPUBLISHED_STATE = 7; + +/** + * Containers every instance has and the sync never migrates (container-pusher skips them by name), + * so they are left out on both sides. + */ +export const SYSTEM_CONTAINER_NAMES = new Set([ + "agilitycssfiles", + "agilityjavascriptfiles", + "agilityglobalcodetemplates", + "agilitymodulecodetemplates", + "agilitypagecodetemplates", +]); + +export type ChannelSitemaps = { [channel: string]: SitemapNode[] | null }; + +export interface SideEntities { + models: any[]; + containers: any[]; + templates: any[]; + assets: any[]; + galleries: any[]; + content: any[]; + pages: any[]; + sitemaps: ChannelSitemaps; +} + +export interface SideIndex { + entities: SideEntities; + modelsById: Map; + /** Lower-cased reference name → model. */ + modelsByRef: Map; + containersById: Map; + /** Lower-cased reference name → container. */ + containersByRef: Map; + templatesById: Map; + /** Lower-cased template name → template. */ + templatesByName: Map; + contentById: Map; + pagesById: Map; + /** Content IDs placed as a module on some page zone (their container is auto-generated). */ + pageBoundContentIDs: Set; + /** Lower-cased container reference name → number of (live) items in it, for this locale. */ + contentCountByContainer: Map; + /** + * Content items whose container is not a live container on this side (deleted, or a system + * container). The pull still returns them, but they are not reachable content and the sync skips + * them (no container mapping), so they are left out of every comparison. + */ + outsideLiveContainers: any[]; +} + +export interface VerifyContext { + sourceGuid: string; + targetGuid: string; + locale: string; + mappings: MappingIndex; + localeMappings: LocaleMappingIndex; + source: SideIndex; + target: SideIndex; + assets: AssetUrlTranslator; + /** + * A --models / --pages / --containers scope narrowed the SOURCE side. Checks that compare + * whole sets (extra target entities, per-container counts) are skipped: the target side is not + * narrowed, so they would report everything outside the scope. + */ + scoped: boolean; + /** Restrict sitemap checks to these channels (lower-case); undefined = every channel. */ + channels?: string[]; +} + +export function isPulled(entity: any): boolean { + return entity?.properties?.state !== UNPUBLISHED_STATE; +} + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +/** Module entries on a page, across every zone. */ +export function zoneModules(page: any): Array<{ zone: string; module: any }> { + const out: Array<{ zone: string; module: any }> = []; + const zones = page?.zones; + if (!zones || typeof zones !== "object") return out; + Object.keys(zones).forEach((zone) => { + const modules = zones[zone]; + if (Array.isArray(modules)) modules.forEach((module) => out.push({ zone, module })); + }); + return out; +} + +/** The content ID a zone module points at, whatever the key's casing. */ +export function moduleContentID(module: any): number | undefined { + const item = module?.item; + if (!item || typeof item !== "object") return undefined; + const raw = item.contentid ?? item.contentID ?? item.ContentID; + const id = typeof raw === "string" ? parseInt(raw, 10) : raw; + return typeof id === "number" && id > 0 ? id : undefined; +} + +export function buildSideIndex(entities: SideEntities, outsideLiveContainers: any[] = []): SideIndex { + const index: SideIndex = { + entities, + modelsById: new Map(), + modelsByRef: new Map(), + containersById: new Map(), + containersByRef: new Map(), + templatesById: new Map(), + templatesByName: new Map(), + contentById: new Map(), + pagesById: new Map(), + pageBoundContentIDs: new Set(), + contentCountByContainer: new Map(), + outsideLiveContainers, + }; + entities.models.forEach((m) => { + if (typeof m?.id === "number") index.modelsById.set(m.id, m); + if (m?.referenceName && !index.modelsByRef.has(lower(m.referenceName))) + index.modelsByRef.set(lower(m.referenceName), m); + }); + entities.containers.forEach((c) => { + if (typeof c?.contentViewID === "number") index.containersById.set(c.contentViewID, c); + if (c?.referenceName && !index.containersByRef.has(lower(c.referenceName))) + index.containersByRef.set(lower(c.referenceName), c); + }); + entities.templates.forEach((t) => { + if (typeof t?.pageTemplateID === "number") index.templatesById.set(t.pageTemplateID, t); + if (t?.pageTemplateName && !index.templatesByName.has(lower(t.pageTemplateName))) + index.templatesByName.set(lower(t.pageTemplateName), t); + }); + entities.content.forEach((c) => { + if (typeof c?.contentID === "number") index.contentById.set(c.contentID, c); + const container = lower(c?.properties?.referenceName); + if (container) + index.contentCountByContainer.set(container, (index.contentCountByContainer.get(container) ?? 0) + 1); + }); + entities.pages.forEach((p) => { + if (typeof p?.pageID === "number") index.pagesById.set(p.pageID, p); + zoneModules(p).forEach(({ module }) => { + const id = moduleContentID(module); + if (id) index.pageBoundContentIDs.add(id); + }); + }); + return index; +} + +export function createVerifyContext(input: { + sourceGuid: string; + targetGuid: string; + locale: string; + mappings: MappingIndex; + source: SideEntities; + target: SideEntities; + scoped?: boolean; + channels?: string[]; +}): VerifyContext { + const clean = (side: SideEntities): SideIndex => { + const containers = side.containers.filter( + (c) => c?.isDeleted !== true && !SYSTEM_CONTAINER_NAMES.has(lower(c?.referenceName)) + ); + const liveContainers = new Set(containers.map((c) => lower(c?.referenceName))); + const pulled = side.content.filter(isPulled); + // Without a container list (nothing pulled, or a hand-built fixture) every item counts as live. + const isLive = (item: any) => containers.length === 0 || liveContainers.has(lower(item?.properties?.referenceName)); + return buildSideIndex( + { + ...side, + content: pulled.filter(isLive), + pages: side.pages.filter(isPulled), + containers, + templates: side.templates.filter((t) => t?.isDeleted !== true), + assets: side.assets.filter((a) => a?.isDeleted !== true && a?.isFolder !== true), + }, + pulled.filter((item) => !isLive(item)) + ); + }; + const source = clean(input.source); + const target = clean(input.target); + const localeMappings = input.mappings.byLocale.get(input.locale); + if (!localeMappings) { + throw new Error(`No mapping index for locale ${input.locale}`); + } + + return { + sourceGuid: input.sourceGuid, + targetGuid: input.targetGuid, + locale: input.locale, + mappings: input.mappings, + localeMappings, + source, + target, + assets: buildAssetUrlTranslator({ + mappingUrls: input.mappings.assetUrls, + mappingPrefixes: input.mappings.assetPrefixes, + assetIdMap: input.mappings.assets, + sourceAssets: source.entities.assets, + targetAssets: target.entities.assets, + }), + scoped: input.scoped === true, + channels: input.channels?.map((c) => c.toLowerCase()), + }; +} + +/** Canonicalization context for SOURCE values: every ID is translated to its target. */ +export function sourceCanonicalContext(ctx: VerifyContext): CanonicalContext { + return { + side: "source", + contentRef: (id) => { + const target = ctx.localeMappings.content.target(id); + return target !== undefined ? String(target) : `unmapped:${id}`; + }, + containerRef: (name) => { + const key = lower(name); + const mapped = ctx.mappings.containerNames.target(key); + if (mapped) return mapped; + // Auto-named nested lists keep their reference name on the target, so an identical name is + // accepted. A name that is not a live container in the source either (a dangling reference + // the sync copies verbatim) is compared literally. Only a live, unmapped source container + // that the target lacks can never match. + if (ctx.target.containersByRef.has(key) || !ctx.source.containersByRef.has(key)) return key; + return `unmapped:${key}`; + }, + galleryRef: (id) => { + const target = ctx.mappings.galleries.target(id); + return target !== undefined ? String(target) : `unmapped:${id}`; + }, + isEmptyList: (name) => !ctx.source.contentCountByContainer.get(lower(name)), + assets: ctx.assets, + }; +} + +/** Canonicalization context for TARGET values: IDs are already target IDs. */ +export function targetCanonicalContext(ctx: VerifyContext): CanonicalContext { + return { + side: "target", + contentRef: (id) => String(id), + containerRef: (name) => lower(name), + galleryRef: (id) => String(id), + isEmptyList: (name) => !ctx.target.contentCountByContainer.get(lower(name)), + assets: ctx.assets, + }; +} + +// ─── loading ───────────────────────────────────────────────────────────────── + +function toSideEntities(entities: GuidEntities, sitemaps: ChannelSitemaps): SideEntities { + return { + models: entities.models || [], + containers: entities.containers || [], + templates: entities.templates || [], + assets: entities.assets || [], + galleries: entities.galleries || [], + content: entities.content || [], + pages: entities.pages || [], + sitemaps, + }; +} + +/** + * Load one side of one locale from the pulled cache. With `filter`, the existing scope filters + * (--models / --pages / --containers) narrow what is loaded, exactly as a scoped sync would. + */ +export async function loadSideEntities( + guid: string, + locale: string, + filter?: ModelFilterOptions +): Promise { + const loader = new GuidDataLoader(guid); + const entities = filter + ? await loader.loadGuidEntities(locale, filter) + : await loader.loadCompleteGuidEntities(locale); + // Sitemaps are always loaded whole: a scoped run narrows the sitemap check to the scoped pages + // itself (see check-sitemap.ts), and siblings outside the scope are still needed for ordering. + const sitemaps = new SitemapHierarchy().loadAllSitemaps(guid, locale); + return toSideEntities(entities, sitemaps); +} + +/** + * The loaders skip a JSON file they cannot parse with only a console warning, which would make a + * corrupt cache read as "entity missing". Verify refuses to compare on top of that: every file a + * check reads is parsed here first, and any failure is a "could not verify" (exit 2). + */ +export function findUnreadableCacheFiles(rootPath: string, guid: string, locales: string[]): string[] { + const problems: string[] = []; + const base = path.join(rootPath, guid); + if (!fs.existsSync(base)) return [`${base} does not exist (the pull wrote nothing)`]; + + const folders = ["models", "containers", "templates", "galleries", path.join("assets", "json")]; + locales.forEach((locale) => + folders.push(path.join(locale, "item"), path.join(locale, "page"), path.join(locale, "nestedsitemap")) + ); + + if (!fs.existsSync(path.join(base, "models"))) problems.push(`${path.join(base, "models")} is missing`); + + folders.forEach((folder) => { + const dir = path.join(base, folder); + if (!fs.existsSync(dir)) return; + let files: string[]; + try { + files = fs.readdirSync(dir).filter((f) => f.endsWith(".json")); + } catch (error: any) { + problems.push(`${dir}: ${error.message}`); + return; + } + files.forEach((file) => { + const full = path.join(dir, file); + try { + JSON.parse(fs.readFileSync(full, "utf8")); + } catch (error: any) { + problems.push(`${full}: ${error.message}`); + } + }); + }); + return problems; +} diff --git a/src/lib/verify/verify-report.ts b/src/lib/verify/verify-report.ts new file mode 100644 index 0000000..dc06603 --- /dev/null +++ b/src/lib/verify/verify-report.ts @@ -0,0 +1,394 @@ +/** + * Findings, tallies and the report for `agility verify` (PROD-2552). + * + * The report is the product of the command: CI reads the exit code, a person reads the console + * table, and a harness asserts on the JSON. All three are derived from the same findings list, so + * they cannot disagree. + * + * Exit codes: + * 0 — every check passed (warnings and source defects do not fail a run) + * 1 — at least one failure-severity finding + * 2 — the run could not verify at all (auth, status guard, pull, missing mappings, unreadable cache) + */ +import * as path from "path"; +import ansiColors from "ansi-colors"; +import { writeJsonSummary, resolveCliVersion } from "../../core/json-summary"; + +/** Bump when a field is removed or its meaning changes. Additions do not require a bump. */ +export const VERIFY_REPORT_SCHEMA_VERSION = 1; + +export const EXIT_PASS = 0; +export const EXIT_DIFFERENCES = 1; +export const EXIT_CANNOT_VERIFY = 2; + +/** + * - failure: the target differs from the source in a way the migration should have prevented. + * - warning: worth knowing, does not fail the run (extra target items, publish-state drift). + * - sourceDefect: a difference explained by the source item itself (e.g. a missing required + * field the target would reject). Listed, does not fail the run. + */ +export type Severity = "failure" | "warning" | "sourceDefect"; + +export type EntityKind = + | "model" + | "container" + | "template" + | "asset" + | "gallery" + | "content" + | "page" + | "zone" + | "sitemap"; + +export interface Finding { + category: string; + severity: Severity; + entity: EntityKind; + /** Locale for locale-scoped entities; "*" for instance-wide ones (models, containers, ...). */ + locale: string; + sourceID?: number | string | null; + targetID?: number | string | null; + name?: string; + /** A field name, zone name, or sitemap path — what inside the entity differs. */ + path?: string; + expected?: any; + actual?: any; + /** Human sentence for the console. */ + message?: string; +} + +export interface Tally { + checked: number; + matched: number; + mismatched: number; + missing: number; + extra: number; + known: number; +} + +/** Instance-wide entities (models, containers, templates, assets, galleries) are tallied here. */ +export const INSTANCE_LOCALE = "*"; + +export function emptyTally(): Tally { + return { checked: 0, matched: 0, mismatched: 0, missing: 0, extra: 0, known: 0 }; +} + +/** + * Collects findings and per-(locale, entity) tallies as the checks run. Each check decides what an + * entity's outcome is; the collector only counts. + */ +export class FindingCollector { + readonly findings: Finding[] = []; + private readonly tallies = new Map>(); + + add(finding: Finding): void { + this.findings.push(finding); + } + + tally(locale: string, entity: EntityKind): Tally { + let byEntity = this.tallies.get(locale); + if (!byEntity) { + byEntity = new Map(); + this.tallies.set(locale, byEntity); + } + let tally = byEntity.get(entity); + if (!tally) { + tally = emptyTally(); + byEntity.set(entity, tally); + } + return tally; + } + + count(locale: string, entity: EntityKind, outcome: keyof Tally, by: number = 1): void { + this.tally(locale, entity)[outcome] += by; + } + + /** Locales in a stable order: instance-wide first, then alphabetical. */ + locales(): string[] { + return Array.from(this.tallies.keys()).sort((a, b) => + a === INSTANCE_LOCALE ? -1 : b === INSTANCE_LOCALE ? 1 : a.localeCompare(b) + ); + } + + talliesFor(locale: string): Record { + const out: Record = {}; + const byEntity = this.tallies.get(locale); + if (!byEntity) return out; + Array.from(byEntity.keys()) + .sort() + .forEach((entity) => (out[entity] = { ...(byEntity.get(entity) as Tally) })); + return out; + } +} + +export function computeExitCode(findings: Finding[]): number { + return findings.some((f) => f.severity === "failure") ? EXIT_DIFFERENCES : EXIT_PASS; +} + +export interface SeverityCounts { + failures: number; + warnings: number; + sourceDefects: number; +} + +function countSeverities(findings: Finding[]): SeverityCounts { + return { + failures: findings.filter((f) => f.severity === "failure").length, + warnings: findings.filter((f) => f.severity === "warning").length, + sourceDefects: findings.filter((f) => f.severity === "sourceDefect").length, + }; +} + +export interface StatusSnapshot { + guid: string; + mode: string; + checkedAt: string; + inProgress: boolean; + lastContentVersionID: number | null; + maxChangeDate: string | null; +} + +export interface VerifyRunInfo { + cliVersion: string; + startedAt: string; + finishedAt: string; + elapsedMs: number; + pull: { attempts: number; elapsedMs: number[] }; + status: { before: StatusSnapshot[]; after: StatusSnapshot[] }; + compareElapsedMs: number; +} + +export interface VerifyOptionsSummary { + locales: string[]; + channel: string; + models: string; + pages: string; + containers: string; +} + +export interface VerifyReport { + schemaVersion: number; + command: "verify"; + verdict: "pass" | "fail" | "error"; + exitCode: number; + /** Set when exitCode is 2: why the run could not verify. */ + error?: string; + source: string; + target: string; + options: VerifyOptionsSummary; + totals: SeverityCounts & { + byCategory: Record; + entities: Record; + }; + byLocale: Record }>; + findings: Finding[]; + /** Everything that differs between two otherwise-identical runs. Drop this for golden files. */ + run: VerifyRunInfo; +} + +/** Stable ordering, so two runs over the same data produce byte-identical findings. */ +export function sortFindings(findings: Finding[]): Finding[] { + const key = (v: any) => (v === undefined || v === null ? "" : String(v)); + const severityRank: Record = { failure: 0, sourceDefect: 1, warning: 2 }; + return findings.slice().sort((a, b) => { + return ( + key(a.locale).localeCompare(key(b.locale)) || + severityRank[a.severity] - severityRank[b.severity] || + a.category.localeCompare(b.category) || + a.entity.localeCompare(b.entity) || + key(a.sourceID).localeCompare(key(b.sourceID), undefined, { numeric: true }) || + key(a.targetID).localeCompare(key(b.targetID), undefined, { numeric: true }) || + key(a.path).localeCompare(key(b.path)) + ); + }); +} + +export function emptyRunInfo(startedAt: Date = new Date()): VerifyRunInfo { + return { + cliVersion: resolveCliVersion(), + startedAt: startedAt.toISOString(), + finishedAt: startedAt.toISOString(), + elapsedMs: 0, + pull: { attempts: 0, elapsedMs: [] }, + status: { before: [], after: [] }, + compareElapsedMs: 0, + }; +} + +export function buildVerifyReport(input: { + source: string; + target: string; + options: VerifyOptionsSummary; + collector: FindingCollector; + run: VerifyRunInfo; +}): VerifyReport { + const findings = sortFindings(input.collector.findings); + const exitCode = computeExitCode(findings); + + const byCategory: Record = {}; + findings.forEach((f) => (byCategory[f.category] = (byCategory[f.category] || 0) + 1)); + const sortedByCategory: Record = {}; + Object.keys(byCategory) + .sort() + .forEach((k) => (sortedByCategory[k] = byCategory[k])); + + const byLocale: VerifyReport["byLocale"] = {}; + const entityTotals: Record = {}; + input.collector.locales().forEach((locale) => { + const entities = input.collector.talliesFor(locale); + byLocale[locale] = { ...countSeverities(findings.filter((f) => f.locale === locale)), entities }; + Object.keys(entities).forEach((entity) => { + const total = entityTotals[entity] ?? emptyTally(); + (Object.keys(total) as Array).forEach((k) => (total[k] += entities[entity][k])); + entityTotals[entity] = total; + }); + }); + const sortedEntityTotals: Record = {}; + Object.keys(entityTotals) + .sort() + .forEach((k) => (sortedEntityTotals[k] = entityTotals[k])); + + return { + schemaVersion: VERIFY_REPORT_SCHEMA_VERSION, + command: "verify", + verdict: exitCode === EXIT_PASS ? "pass" : "fail", + exitCode, + source: input.source, + target: input.target, + options: input.options, + totals: { ...countSeverities(findings), byCategory: sortedByCategory, entities: sortedEntityTotals }, + byLocale, + findings, + run: input.run, + }; +} + +/** Report for a run that could not verify (exit 2). Same schema, no findings. */ +export function buildErrorReport(input: { + source: string; + target: string; + options: VerifyOptionsSummary; + error: string; + run: VerifyRunInfo; +}): VerifyReport { + return { + schemaVersion: VERIFY_REPORT_SCHEMA_VERSION, + command: "verify", + verdict: "error", + exitCode: EXIT_CANNOT_VERIFY, + error: input.error, + source: input.source, + target: input.target, + options: input.options, + totals: { failures: 0, warnings: 0, sourceDefects: 0, byCategory: {}, entities: {} }, + byLocale: {}, + findings: [], + run: input.run, + }; +} + +export function defaultReportPath(rootPath: string, sourceGuid: string, targetGuid: string, at: Date): string { + const timestamp = at.toISOString().replace(/[:.]/g, "-"); + return path.join(rootPath, "logs", `verify-${sourceGuid}-${targetGuid}-${timestamp}.json`); +} + +/** Never throws — an unwritable report path is a warning, not a different verdict. */ +export function writeVerifyReport(filePath: string, report: VerifyReport): { written: boolean; error?: string } { + return writeJsonSummary(filePath, report); +} + +// ─── console output ────────────────────────────────────────────────────────── + +/** How many findings to print per category before summarizing the rest. */ +export const MAX_DETAILS_PER_CATEGORY = 20; + +const TABLE_COLUMNS: Array = ["checked", "matched", "mismatched", "missing", "extra", "known"]; + +function formatValue(value: any, max: number = 160): string { + if (value === undefined) return "(none)"; + let text: string; + if (Array.isArray(value)) { + text = `[${value.map((v) => (typeof v === "string" ? v : JSON.stringify(v))).join(", ")}]`; + } else if (typeof value === "string") { + text = JSON.stringify(value); + } else { + text = JSON.stringify(value); + } + return text.length > max ? `${text.slice(0, max - 1)}…` : text; +} + +export function renderTallyTable(entities: Record): string[] { + const header = ["entity", ...TABLE_COLUMNS]; + const rows = Object.keys(entities).map((entity) => [ + entity, + ...TABLE_COLUMNS.map((c) => String(entities[entity][c])), + ]); + const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length))); + const line = (cells: string[]) => + cells.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join(" "); + return [line(header), widths.map((w) => "-".repeat(w)).join(" "), ...rows.map(line)]; +} + +export interface PrintOptions { + /** Builds a CMS link for a finding, or returns undefined when there is none. */ + linkFor?: (finding: Finding) => string | undefined; + log?: (line: string) => void; +} + +export function printVerifyReport(report: VerifyReport, options: PrintOptions = {}): void { + const log = options.log ?? ((line: string) => console.log(line)); + + log(ansiColors.cyan(`\n🔎 Verify: ${report.source} → ${report.target}`)); + + if (report.exitCode === EXIT_CANNOT_VERIFY) { + log(ansiColors.red(`\n✗ Could not verify: ${report.error}`)); + log(ansiColors.red(`VERDICT: ERROR (exit ${report.exitCode})`)); + return; + } + + Object.keys(report.byLocale).forEach((locale) => { + const section = report.byLocale[locale]; + const title = locale === INSTANCE_LOCALE ? "Instance-wide" : `Locale ${locale}`; + log( + ansiColors.bold( + `\n${title}: ${section.failures} failure(s), ${section.warnings} warning(s), ${section.sourceDefects} source defect(s)` + ) + ); + renderTallyTable(section.entities).forEach((row) => log(` ${row}`)); + }); + + const byCategory = new Map(); + report.findings.forEach((f) => { + const key = `${f.severity}:${f.category}`; + if (!byCategory.has(key)) byCategory.set(key, []); + (byCategory.get(key) as Finding[]).push(f); + }); + + byCategory.forEach((findings, key) => { + const [severity, category] = key.split(":"); + const color = + severity === "failure" ? ansiColors.red : severity === "warning" ? ansiColors.yellow : ansiColors.magenta; + log(color(`\n${severity.toUpperCase()} ${category} (${findings.length})`)); + findings.slice(0, MAX_DETAILS_PER_CATEGORY).forEach((f) => { + const where = f.locale === INSTANCE_LOCALE ? "" : `[${f.locale}] `; + const ids = `source ${f.sourceID ?? "-"} → target ${f.targetID ?? "-"}`; + const label = [f.name, f.path].filter(Boolean).join(" / "); + log(` • ${where}${f.entity} ${label ? `"${label}" ` : ""}(${ids})`); + if (f.message) log(ansiColors.gray(` ${f.message}`)); + if (f.expected !== undefined || f.actual !== undefined) { + log(ansiColors.gray(` expected: ${formatValue(f.expected)}`)); + log(ansiColors.gray(` actual: ${formatValue(f.actual)}`)); + } + const link = options.linkFor?.(f); + if (link) log(ansiColors.gray(` ${link}`)); + }); + if (findings.length > MAX_DETAILS_PER_CATEGORY) { + log(ansiColors.gray(` …and ${findings.length - MAX_DETAILS_PER_CATEGORY} more (see the JSON report)`)); + } + }); + + const t = report.totals; + const verdictLine = `VERDICT: ${report.verdict.toUpperCase()} — ${t.failures} failure(s), ${t.warnings} warning(s), ${t.sourceDefects} source defect(s) (exit ${report.exitCode})`; + log(""); + log(report.exitCode === EXIT_PASS ? ansiColors.green(`✓ ${verdictLine}`) : ansiColors.red(`✗ ${verdictLine}`)); +} diff --git a/src/tests/sync/verify-after-sync.test.ts b/src/tests/sync/verify-after-sync.test.ts new file mode 100644 index 0000000..5b973d4 --- /dev/null +++ b/src/tests/sync/verify-after-sync.test.ts @@ -0,0 +1,171 @@ +/** + * PROD-2552: `agility verify` against the output of a real (hermetic) sync. + * + * The sync half is the real push pipeline from full-instance.test.ts, extended with a page that + * carries two components in one zone. Its target is then projected to disk the way a pull would + * write it, and verify runs over the two caches with the pull replaced by a no-op — everything + * else (loaders, mapping files, checks, report) is the real code. + * + * 1. Right after the sync, verify passes (exit 0). + * 2. Swap the two components in the target page's zone — the reported defect — and verify + * fails (exit 1) with a componentOrderMismatch naming both sequences. + */ +import * as fs from "fs"; +import * as path from "path"; +import { SyncHarness } from "./helpers/run-sync"; +import { installNetworkStubs, InstalledNetworkStubs } from "./helpers/network-stubs"; +import { + ALL_ELEMENTS, + CONTENT_HELLO, + FIXTURE_LOCALE, + fullInstanceFixture, + PAGE_HOME, +} from "./helpers/full-instance-fixture"; +import { Verify } from "core/verify"; +import { EXIT_DIFFERENCES, EXIT_PASS, VerifyReport } from "lib/verify/verify-report"; + +jest.mock("lib/pushers/batch-polling", () => ({ + pollBatchUntilComplete: jest.fn(), + extractContentBatchResults: jest.fn(), + extractPageBatchResults: jest.fn(), +})); +jest.mock("axios", () => ({ post: jest.fn(), get: jest.fn(), default: { post: jest.fn() } })); +jest.mock("lib/pushers/url-redirection-api", () => ({ + MAX_URL_REDIRECTION_BATCH_SIZE: 250, + saveUrlRedirections: jest.fn(), +})); + +const CONTENT_WORLD = { + ...CONTENT_HELLO, + contentID: 501, + properties: { ...CONTENT_HELLO.properties, versionID: 901, itemOrder: 1 }, + fields: { title: "World" }, +}; + +const PAGE_WITH_ZONE = { + ...PAGE_HOME, + zones: { + Main: [ + { module: "Post", item: { contentid: 500, fulllist: false } }, + { module: "Post", item: { contentid: 501, fulllist: false } }, + ], + }, +}; + +function fixture() { + const base = fullInstanceFixture(); + base.source[`${FIXTURE_LOCALE}/item/501.json`] = CONTENT_WORLD; + base.source[`${FIXTURE_LOCALE}/page/800.json`] = PAGE_WITH_ZONE; + return base; +} + +/** The target's nested sitemap, as a pull would write it: the fake does not keep one. */ +function writeTargetSitemap(harness: SyncHarness): void { + const pageMapping = (harness.readMappings("page", FIXTURE_LOCALE) ?? [])[0]; + const file = path.join(harness.rootPath, harness.targetGuid, FIXTURE_LOCALE, "nestedsitemap", "website.json"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync( + file, + JSON.stringify([ + { + title: "Home", + name: "home", + pageID: pageMapping.targetPageID, + menuText: "Home", + visible: { menu: true, sitemap: true }, + path: "/home", + redirect: null, + isFolder: false, + }, + ]) + ); +} + +function targetPagePath(harness: SyncHarness): string { + const pageMapping = (harness.readMappings("page", FIXTURE_LOCALE) ?? [])[0]; + return path.join(harness.rootPath, harness.targetGuid, FIXTURE_LOCALE, "page", `${pageMapping.targetPageID}.json`); +} + +async function verify(harness: SyncHarness): Promise { + return new Verify( + { + sourceGuid: harness.sourceGuid, + targetGuid: harness.targetGuid, + locales: [FIXTURE_LOCALE], + models: "", + pages: "", + containers: "", + jsonSummary: path.join(harness.rootPath, "verify-report.json"), + rootPath: harness.rootPath, + }, + { + getStatus: async () => ({ inProgress: false, lastContentVersionID: 1, maxChangeDate: "x" }) as any, + // Hermetic: the caches are already on disk, so the pull is a no-op that reports success. + pull: async () => ({ + success: true, + results: [ + { guidProcessed: harness.sourceGuid, failed: [] }, + { guidProcessed: harness.targetGuid, failed: [] }, + ], + }), + print: false, + log: () => {}, + } + ).run(); +} + +describe("verify after a hermetic sync", () => { + let harness: SyncHarness; + let net: InstalledNetworkStubs; + let mappingsBefore: Record; + + beforeEach(async () => { + net = installNetworkStubs(); + jest.spyOn(console, "log").mockImplementation(() => {}); + jest.spyOn(console, "warn").mockImplementation(() => {}); + jest.spyOn(console, "error").mockImplementation(() => {}); + + harness = new SyncHarness({ elements: ALL_ELEMENTS, fixture: fixture() }); + const run = await harness.run(); + expect(run.totals.failed).toBe(0); + harness.fake.projectToDisk(harness.rootPath, harness.targetGuid); + writeTargetSitemap(harness); + mappingsBefore = harness.readAllMappings(); + }); + + afterEach(() => { + harness?.cleanup(); + net.restore(); + jest.restoreAllMocks(); + }); + + it("passes right after the sync", async () => { + const report = await verify(harness); + + expect(report.findings.filter((f) => f.severity === "failure")).toEqual([]); + expect(report.exitCode).toBe(EXIT_PASS); + expect(report.totals.entities.zone).toMatchObject({ checked: 1, matched: 1 }); + expect(report.totals.entities.content).toMatchObject({ checked: 2, matched: 2 }); + expect(net.fetchCalls).toEqual([]); + }); + + it("fails with componentOrderMismatch when the target zone is reordered", async () => { + const file = targetPagePath(harness); + const targetPage = JSON.parse(fs.readFileSync(file, "utf8")); + const zone = Object.keys(targetPage.zones)[0]; + targetPage.zones[zone].reverse(); + fs.writeFileSync(file, JSON.stringify(targetPage)); + + const report = await verify(harness); + + expect(report.exitCode).toBe(EXIT_DIFFERENCES); + const failures = report.findings.filter((f) => f.severity === "failure"); + expect(failures.map((f) => f.category)).toEqual(["componentOrderMismatch"]); + expect(failures[0].actual).toEqual([...(failures[0].expected as string[])].reverse()); + }); + + it("leaves the mapping files untouched", async () => { + await verify(harness); + expect(harness.readAllMappings()).toEqual(mappingsBefore); + }); +}); From d25472f0d69b5c716a096db10bed1652034870bf Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 14 Oct 2026 14:15:16 -0400 Subject: [PATCH 2/3] PROD-2773: warn when content references a CDN container owned by neither the source nor the target PROD-2773: warn about asset URLs into a CDN container owned by neither instance Content synced before the PROD-915 fix kept the upstream instance's CDN URLs, so a source can reference a third instance's container (e.g. cdn-usa2.aglty.io/ upstream-qa/...). The asset mapping cannot map those, so they passed through silently: image/file fields arrived empty on the target and rich-text links kept pointing at the other instance. The content pusher now scans each locale's syncable items (also in --preflight and for unchanged items) and reports every foreign CDN URL once per run, with the items/fields/locales using it. Reporting only: payloads, counts, success and exit codes are unchanged. - foreign-cdn-detector: pure URL extraction (aglty.io, azurewebsites origin, cdn*.agilitycms.com, learned custom edge hosts; JSON-escaped, &, query and percent-encoded variants) and container-ownership check. Source/target containers come from asset mappings and pulled assets; nothing is reported while either side's container is unknown. - foreign-cdn-report: per-run dedupe and rendering; attachment-field URLs are called out separately from text occurrences. - Surfaces: logger warning on first sighting, end-of-run "Foreign CDN references: N URLs in M items" block, additive --jsonSummary warnings (code "foreignCdnReference", url, container, references), and a preflight warnings section that does not affect totals or the exit code. Preflight totals now include PROD-2542's "invalid" count; the push test expects it. Co-Authored-By: Claude Opus 5.5 --- src/core/json-summary.ts | 15 + src/core/push.ts | 19 +- src/core/tests/push.test.ts | 82 ++++++ src/lib/content/foreign-cdn-detector.ts | 256 ++++++++++++++++++ src/lib/content/foreign-cdn-report.ts | 186 +++++++++++++ .../tests/foreign-cdn-detector.test.ts | 215 +++++++++++++++ .../content/tests/foreign-cdn-report.test.ts | 211 +++++++++++++++ src/lib/mappers/asset-mapper.ts | 5 + src/lib/preflight/preflight-report.ts | 25 ++ .../preflight/tests/preflight-report.test.ts | 39 +++ .../pushers/content-pusher/content-pusher.ts | 14 + .../util/report-foreign-cdn-references.ts | 74 +++++ .../report-foreign-cdn-references.test.ts | 78 ++++++ 13 files changed, 1218 insertions(+), 1 deletion(-) create mode 100644 src/lib/content/foreign-cdn-detector.ts create mode 100644 src/lib/content/foreign-cdn-report.ts create mode 100644 src/lib/content/tests/foreign-cdn-detector.test.ts create mode 100644 src/lib/content/tests/foreign-cdn-report.test.ts create mode 100644 src/lib/pushers/content-pusher/util/report-foreign-cdn-references.ts create mode 100644 src/lib/pushers/content-pusher/util/tests/report-foreign-cdn-references.test.ts diff --git a/src/core/json-summary.ts b/src/core/json-summary.ts index 13acf80..801b95b 100644 --- a/src/core/json-summary.ts +++ b/src/core/json-summary.ts @@ -44,6 +44,21 @@ export interface JsonSummaryDetail { model?: string; /** PROD-2542: the required fields it was missing, as "name (Type)". */ fields?: string[]; + /** + * PROD-2773 (additive): set on warnings that are not item failures, e.g. "foreignCdnReference" + * — one entry per asset URL into a CDN container owned by neither instance, with the URL, its + * container and every item/field/locale that references it. + */ + code?: string; + url?: string; + container?: string; + references?: Array<{ + contentID: number; + referenceName: string; + locale: string; + field: string; + kind: "attachment" | "text"; + }>; } /** diff --git a/src/core/push.ts b/src/core/push.ts index 19e544c..1b00072 100644 --- a/src/core/push.ts +++ b/src/core/push.ts @@ -19,6 +19,8 @@ import { Pull } from "./pull"; import { preflightReport } from "../lib/preflight/preflight-report"; import { buildJsonSummary, writeJsonSummary, JsonSummaryDetail, JsonSummaryPhase } from "./json-summary"; import type { FailureDetail } from "../types/sourceData"; +import { foreignCdnReport } from "../lib/content/foreign-cdn-report"; +import { resetForeignCdnDetectorCache } from "../lib/pushers/content-pusher/util/report-foreign-cdn-references"; /** * PROD-2310: Decide whether a set of auto-publish errors should fail the sync exit code. @@ -62,6 +64,9 @@ export class Push { // Preflight (PROD-2203): start each run with a clean report of planned actions preflightReport.reset(); + // PROD-2773: foreign CDN references are reported once per URL per run + foreignCdnReport.reset(); + resetForeignCdnDetectorCache(); // Initialize logger for push operation // Determine if this is a sync operation by checking if both source and target GUIDs exist @@ -299,6 +304,14 @@ export class Push { }); } + // PROD-2773: asset URLs into a CDN container owned by neither instance — passed through + // unchanged, so non-blocking, but an image/file field holding one may arrive empty. + if (foreignCdnReport.hasEntries()) { + const [headline, ...rest] = foreignCdnReport.renderSummaryLines(); + console.log(ansiColors.yellow(`\n ${headline}`)); + rest.forEach((line) => console.log(ansiColors.yellow(` ${line}`))); + } + // PROD-2311: post-publish mapping/version bookkeeping notices are NOT publish // failures — show them under a non-blocking header so they don't read as broken. if (mappingWarnings.length > 0) { @@ -320,6 +333,9 @@ export class Push { // Preflight (PROD-2203): print the report of actions that WOULD have been // taken, and signal conflicts via a non-zero exit code so automation can gate on it. if (state.preflight) { + if (foreignCdnReport.hasEntries()) { + preflightReport.recordWarning(`${foreignCdnReport.headline()} — see the list above`); + } preflightReport.print(); if (preflightReport.hasConflicts()) { process.exitCode = 1; @@ -334,7 +350,8 @@ export class Push { exitCode: typeof process.exitCode === "number" ? process.exitCode : success ? 0 : 1, phases: results.flatMap((r) => r.phases ?? []), failures: syncFailureDetails, - warnings: syncWarningDetails, + // PROD-2773: one additive entry per foreign CDN URL (code "foreignCdnReference") + warnings: [...syncWarningDetails, ...foreignCdnReport.toWarningDetails(sourceGuid)], operationErrors: syncErrors, autoPublishErrors, startedAt, diff --git a/src/core/tests/push.test.ts b/src/core/tests/push.test.ts index 63d4cc3..8a61459 100644 --- a/src/core/tests/push.test.ts +++ b/src/core/tests/push.test.ts @@ -1,5 +1,8 @@ import { Push, hasBlockingAutoPublishErrors, formatFailureName } from "../push"; import { resetState, setState } from "../state"; +import { Pull } from "../pull"; +import { Pushers } from "../../lib/pushers/orchestrate-pushers"; +import { foreignCdnReport } from "../../lib/content/foreign-cdn-report"; beforeEach(() => { resetState(); @@ -286,6 +289,85 @@ describe("Push.pushInstances — --jsonSummary", () => { expect(parsed.operationErrors).toEqual([{ type: "fatal", error: "Model validation failed" }]); }); + describe("foreign CDN references (PROD-2773)", () => { + const FOREIGN = "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png"; + + /** Have the mocked orchestrator record what the content pusher would have found. */ + async function runWithForeignReference(extraState: Record) { + jest.spyOn(Pull.prototype, "pullInstances").mockResolvedValue(undefined as any); + jest.spyOn(Pushers.prototype, "instanceOrchestrator").mockImplementation(async () => { + foreignCdnReport.setOwnContainers(["site-qa"], ["site-cat"]); + foreignCdnReport.record( + { url: FOREIGN, container: "upstream-qa", field: "mobileBackgroundImage", kind: "attachment" }, + { contentID: 474, referenceName: "home_homebanner_homebanner", locale: "en-us" } + ); + return [pushResult({ warningDetails: [{ name: "Home", error: "Dropped Component Hero — unresolved" }] })] as any; + }); + setState({ sourceGuid: "src-u", targetGuid: "tgt-u", ...extraState }); + return new Push().pushInstances(); + } + + it("adds one warning per URL to --jsonSummary without affecting success", async () => { + const out = path.join(tmp, "summary.json"); + const result = await runWithForeignReference({ jsonSummary: out }); + + const parsed = JSON.parse(fs.readFileSync(out, "utf8")); + expect(result.success).toBe(true); + expect(parsed.success).toBe(true); + expect(parsed.exitCode).toBe(0); + expect(parsed.schemaVersion).toBe(1); + expect(parsed.failures).toEqual([]); + expect(parsed.warnings).toHaveLength(2); + expect(parsed.warnings[0]).toEqual({ name: "Home", error: "Dropped Component Hero — unresolved" }); + expect(parsed.warnings[1]).toMatchObject({ + name: FOREIGN, + type: "content", + contentID: 474, + guid: "src-u", + locale: "en-us", + code: "foreignCdnReference", + url: FOREIGN, + container: "upstream-qa", + references: [ + { + contentID: 474, + referenceName: "home_homebanner_homebanner", + locale: "en-us", + field: "mobileBackgroundImage", + kind: "attachment", + }, + ], + }); + expect(parsed.warnings[1].error).toContain("belongs to neither the source (site-qa)"); + + const printed = (console.log as jest.Mock).mock.calls.map((c) => String(c[0])).join("\n"); + expect(printed).toContain("Foreign CDN references: 1 URL in 1 item (non-blocking):"); + expect(printed).toContain(FOREIGN); + }); + + it("lists the count in the preflight report without changing its totals or exit code", async () => { + const out = path.join(tmp, "summary.json"); + await runWithForeignReference({ jsonSummary: out, preflight: true }); + + const parsed = JSON.parse(fs.readFileSync(out, "utf8")); + expect(parsed.exitCode).toBe(0); + expect(process.exitCode).toBeUndefined(); + expect(parsed.preflight.totals).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(parsed.preflight.hasConflicts).toBe(false); + expect(parsed.warnings.some((w: any) => w.code === "foreignCdnReference")).toBe(true); + + const printed = (console.log as jest.Mock).mock.calls.map((c) => String(c[0])).join("\n"); + expect(printed).toContain("Warnings (1, non-blocking — not counted above):"); + }); + + it("starts each run with an empty report", async () => { + const out = path.join(tmp, "summary.json"); + await runWithForeignReference({}); + await runPush([pushResult()], { jsonSummary: out }); + expect(JSON.parse(fs.readFileSync(out, "utf8")).warnings).toEqual([]); + }); + }); + it("does not fail the run when the summary path cannot be written", async () => { // A reporting artifact must never turn a good sync into a failure. const blocker = path.join(tmp, "not-a-dir"); diff --git a/src/lib/content/foreign-cdn-detector.ts b/src/lib/content/foreign-cdn-detector.ts new file mode 100644 index 0000000..2be8371 --- /dev/null +++ b/src/lib/content/foreign-cdn-detector.ts @@ -0,0 +1,256 @@ +/** + * PROD-2773: detect asset URLs that point at an Agility CDN container belonging to NEITHER the + * source NOR the target instance of a sync. + * + * Content synced before the PROD-915 fix kept the URLs of the instance it was copied FROM, so a + * source can carry URLs into a third instance's container (e.g. `cdn-usa2.aglty.io/upstream-qa/…` + * in an instance whose own container is `site-qa`). The asset mapping only knows the + * source's own container, so those URLs pass through unchanged and silently: an image/file field + * holding one may be dropped by the target, and a link in rich text keeps pointing at the other + * instance. This module only DETECTS them — it never changes a value. + * + * Container ownership rule: + * - A URL is an Agility CDN URL when its host is a generic Agility CDN host (`cdn*.aglty.io`, + * `agility-cdn*.azurewebsites.net`, `cdn*.agilitycms.com`) or a custom edge host seen in the + * source/target container URLs (asset mappings, pulled assets). Non-CDN Agility hosts such as + * `api.aglty.io` or `manager.agilitycms.com` are deliberately not matched. + * - Its container is the first path segment (case-insensitive), whatever the host — the same + * container is served from the aglty.io edge, the azurewebsites origin and any custom domain. + * - It is foreign when that container is in neither the source's nor the target's known set. + * - When either side's container is unknown (no asset mappings and no pulled assets for it), any + * unclaimed container could plausibly be that side's, so nothing is reported at all. + */ +import * as mgmtApi from "@agility/management-sdk"; + +/** Generic Agility CDN hosts; anything else must be learned from the instances' own URLs. */ +const GENERIC_CDN_HOST_PATTERNS: RegExp[] = [ + /^cdn[a-z0-9-]*\.aglty\.io$/, + /^agility-?cdn[a-z0-9-]*\.azurewebsites\.net$/, + /^cdn[a-z0-9-]*\.agilitycms\.com$/, +]; + +// host, then the path up to the first character that cannot be part of a URL in text/HTML/JSON +const URL_PATTERN = /(https?:)?\/\/([a-z0-9-]+(?:\.[a-z0-9-]+)+)(?::\d+)?\/([^\s"'<>()\\[\]{}|^`]*)/gi; + +export interface CdnUrlMatch { + /** The asset URL as found (unescaped, query string and fragment removed). */ + url: string; + host: string; + /** Lower-cased first path segment. */ + container: string; +} + +export interface CdnOwnership { + sourceContainers: string[]; + targetContainers: string[]; + /** Custom (non-generic) edge hosts used by the source or target, lower-cased. */ + customHosts: string[]; +} + +export function isGenericAgilityCdnHost(host: string): boolean { + const h = host.toLowerCase(); + return GENERIC_CDN_HOST_PATTERNS.some((p) => p.test(h)); +} + +/** + * Undo the encodings a URL picks up when stored in a field: JSON-escaped slashes (`https:\/\/`), + * HTML-escaped ampersands (`&`) and percent-encoded nested URLs (`?src=https%3A%2F%2F…`). + * Only used for detection; field values are never rewritten. + */ +function normalizeText(text: string): string { + return text.replace(/\\+\//g, "/").replace(/&/gi, "&").replace(/%3A/gi, ":").replace(/%2F/gi, "/"); +} + +/** + * Every Agility CDN asset URL in a string (a plain URL, rich text, HTML or a JSON blob). + * A bare container URL (no file path) is not an asset reference and is ignored; on a custom host + * the last segment must also look like a file, since that host may serve ordinary pages too. + */ +export function extractAgilityCdnUrls(text: string, customHosts: Iterable = []): CdnUrlMatch[] { + if (!text || typeof text !== "string" || !text.includes("/")) return []; + const custom = new Set(Array.from(customHosts, (h) => h.toLowerCase())); + const matches: CdnUrlMatch[] = []; + + const normalized = normalizeText(text); + const pattern = new RegExp(URL_PATTERN.source, URL_PATTERN.flags); + let m: RegExpExecArray | null; + while ((m = pattern.exec(normalized)) !== null) { + const scheme = m[1] || ""; + const host = m[2].toLowerCase(); + const pathStart = m.index + m[0].length - m[3].length; + const path = m[3].split(/[?#]/)[0].replace(/[.,;:!*]+$/, ""); + // resume scanning inside this URL's query string, where another URL may be nested + // (e.g. an image proxy's ?src=https://cdn.aglty.io/...) + pattern.lastIndex = pathStart + path.length; + + const isGeneric = isGenericAgilityCdnHost(host); + if (!isGeneric && !custom.has(host)) continue; + + const segments = path.split("/").filter((s) => s.length > 0); + if (segments.length < 2) continue; + if (!isGeneric && !/\.[a-z0-9]{1,8}$/i.test(segments[segments.length - 1])) continue; + + matches.push({ url: `${scheme}//${m[2]}/${path}`, host, container: segments[0].toLowerCase() }); + } + return matches; +} + +/** Host and container of a container/asset URL such as `https://cdn-usa2.aglty.io/my-container`. */ +export function parseContainerUrl(url: string | null | undefined): { host: string; container: string } | null { + if (!url || typeof url !== "string") return null; + try { + const parsed = new URL(url); + const container = parsed.pathname.split("/").filter((s) => s.length > 0)[0]; + if (!container) return null; + return { host: parsed.hostname.toLowerCase(), container: decodeURIComponent(container).toLowerCase() }; + } catch { + return null; + } +} + +interface ContainerUrlRecord { + sourceContainerEdgeUrl?: string; + targetContainerEdgeUrl?: string; + sourceContainerOriginUrl?: string; + targetContainerOriginUrl?: string; + sourceUrl?: string; + targetUrl?: string; +} + +/** + * Collect each side's containers (and custom edge hosts) from the asset mapping records and the + * pulled asset lists. Either input may be empty; a side with nothing to learn from stays unknown. + */ +export function buildCdnOwnership(input: { + assetMappings?: ContainerUrlRecord[]; + sourceAssets?: Array>; + targetAssets?: Array>; +}): CdnOwnership { + const source = new Set(); + const target = new Set(); + const hosts = new Set(); + + const add = (side: Set, url: string | undefined) => { + const parsed = parseContainerUrl(url); + if (!parsed) return; + side.add(parsed.container); + if (!isGenericAgilityCdnHost(parsed.host)) hosts.add(parsed.host); + }; + + for (const m of input.assetMappings ?? []) { + if (!m) continue; + const hasContainerUrls = m.sourceContainerEdgeUrl || m.sourceContainerOriginUrl; + add(source, m.sourceContainerEdgeUrl); + add(source, m.sourceContainerOriginUrl); + add(target, m.targetContainerEdgeUrl); + add(target, m.targetContainerOriginUrl); + // legacy records (before container URLs were tracked) only have the per-asset URLs + if (!hasContainerUrls) add(source, m.sourceUrl); + if (!m.targetContainerEdgeUrl && !m.targetContainerOriginUrl) add(target, m.targetUrl); + } + + for (const [assets, side] of [ + [input.sourceAssets, source], + [input.targetAssets, target], + ] as const) { + for (const a of assets ?? []) { + if (!a) continue; + add(side, a.containerEdgeUrl); + add(side, a.containerOriginUrl); + add(side, a.edgeUrl); + add(side, a.originUrl); + } + } + + return { + sourceContainers: Array.from(source), + targetContainers: Array.from(target), + customHosts: Array.from(hosts), + }; +} + +export class ForeignCdnDetector { + private source: Set; + private target: Set; + private customHosts: Set; + + constructor(ownership: CdnOwnership) { + this.source = new Set(ownership.sourceContainers.map((c) => c.toLowerCase())); + this.target = new Set(ownership.targetContainers.map((c) => c.toLowerCase())); + this.customHosts = new Set(ownership.customHosts.map((h) => h.toLowerCase())); + } + + /** False when either side's container is unknown — see the ownership rule above. */ + isEnabled(): boolean { + return this.source.size > 0 && this.target.size > 0; + } + + get sourceContainers(): string[] { + return Array.from(this.source); + } + + get targetContainers(): string[] { + return Array.from(this.target); + } + + isForeignContainer(container: string): boolean { + const c = container.toLowerCase(); + return this.isEnabled() && !this.source.has(c) && !this.target.has(c); + } + + findForeignUrls(text: string): CdnUrlMatch[] { + if (!this.isEnabled()) return []; + return extractAgilityCdnUrls(text, this.customHosts).filter( + (m) => !this.source.has(m.container) && !this.target.has(m.container) + ); + } +} + +/** "attachment": the URL of an image/file/attachment-list field. "text": inside a string field. */ +export type ForeignCdnFieldKind = "attachment" | "text"; + +export interface ForeignCdnFieldReference extends CdnUrlMatch { + /** Field path, named the way ContentFieldMapper names nested values (`field[0]`, `field.sub`). */ + field: string; + kind: ForeignCdnFieldKind; +} + +/** + * Walk a content item's fields and return every foreign CDN reference, one per (field, URL). + * Mirrors ContentFieldMapper's shapes: an object with a string `url` is an attachment (ImageAttachment, + * FileAttachment, an AttachmentList entry); any other string is scanned as text. + */ +export function collectForeignCdnReferences(fields: any, detector: ForeignCdnDetector): ForeignCdnFieldReference[] { + if (!detector.isEnabled() || !fields || typeof fields !== "object") return []; + const refs: ForeignCdnFieldReference[] = []; + const seen = new Set(); + + const push = (field: string, kind: ForeignCdnFieldKind, matches: CdnUrlMatch[]) => { + for (const m of matches) { + const key = `${field}\u0000${m.url}`; + if (seen.has(key)) continue; + seen.add(key); + refs.push({ ...m, field, kind }); + } + }; + + const walk = (path: string, value: any) => { + if (value === null || value === undefined) return; + if (typeof value === "string") { + push(path, "text", detector.findForeignUrls(value)); + } else if (Array.isArray(value)) { + value.forEach((v, i) => walk(`${path}[${i}]`, v)); + } else if (typeof value === "object") { + for (const [key, v] of Object.entries(value)) { + if (key === "url" && typeof v === "string") { + push(path, "attachment", detector.findForeignUrls(v)); + } else { + walk(`${path}.${key}`, v); + } + } + } + }; + + for (const [name, value] of Object.entries(fields)) walk(name, value); + return refs; +} diff --git a/src/lib/content/foreign-cdn-report.ts b/src/lib/content/foreign-cdn-report.ts new file mode 100644 index 0000000..c5e9ebe --- /dev/null +++ b/src/lib/content/foreign-cdn-report.ts @@ -0,0 +1,186 @@ +/** + * PROD-2773: per-run collector for foreign CDN references (see foreign-cdn-detector.ts). + * + * The content pusher records each reference it finds; this collector dedupes them to one entry + * per distinct URL, aggregating the items/fields/locales that use it, and renders that entry for + * the console/log, the end-of-run summary, --jsonSummary and the --preflight report. + * + * Reporting only: nothing here affects item payloads, counts, `success` or the exit code. + * A process-level singleton, like preflightReport, because pushers share state globally. + */ +import type { ForeignCdnFieldKind, ForeignCdnFieldReference } from "./foreign-cdn-detector"; + +export interface ForeignCdnOccurrence { + contentID: number; + referenceName: string; + locale: string; + field: string; + kind: ForeignCdnFieldKind; +} + +export interface ForeignCdnUrlEntry { + url: string; + container: string; + occurrences: ForeignCdnOccurrence[]; +} + +/** Shape of a --jsonSummary `warnings` entry (JsonSummaryDetail plus additive fields). */ +export interface ForeignCdnWarningDetail { + name: string; + error: string; + type: "content"; + contentID?: number; + guid?: string; + locale?: string; + code: "foreignCdnReference"; + url: string; + container: string; + references: ForeignCdnOccurrence[]; +} + +export const FOREIGN_CDN_WARNING_CODE = "foreignCdnReference"; + +class ForeignCdnReport { + private entries = new Map(); + private sourceContainers: string[] = []; + private targetContainers: string[] = []; + + reset(): void { + this.entries.clear(); + this.sourceContainers = []; + this.targetContainers = []; + } + + /** The containers the run treated as its own, quoted in every message. */ + setOwnContainers(sourceContainers: string[], targetContainers: string[]): void { + this.sourceContainers = sourceContainers; + this.targetContainers = targetContainers; + } + + /** + * Record one reference. Returns "new-url" the first time a URL is seen in this run, + * "new-occurrence" for a further item/field/locale using it, and "duplicate" otherwise. + */ + record( + ref: Pick, + item: { contentID: number; referenceName: string; locale: string } + ): "new-url" | "new-occurrence" | "duplicate" { + const key = ref.url.toLowerCase(); + let entry = this.entries.get(key); + const isNewUrl = !entry; + if (!entry) { + entry = { url: ref.url, container: ref.container, occurrences: [] }; + this.entries.set(key, entry); + } + const exists = entry.occurrences.some( + (o) => o.contentID === item.contentID && o.locale === item.locale && o.field === ref.field + ); + if (exists) return "duplicate"; + entry.occurrences.push({ ...item, field: ref.field, kind: ref.kind }); + return isNewUrl ? "new-url" : "new-occurrence"; + } + + /** Distinct URLs in first-seen order, those used by an image/file field (higher risk) first. */ + getEntries(): ForeignCdnUrlEntry[] { + const rank = (e: ForeignCdnUrlEntry) => (e.occurrences.some((o) => o.kind === "attachment") ? 0 : 1); + return Array.from(this.entries.values()).sort((a, b) => rank(a) - rank(b)); + } + + /** Distinct source content items (by contentID; locales of one item count once). */ + getItemCount(): number { + const ids = new Set(); + this.entries.forEach((e) => e.occurrences.forEach((o) => ids.add(o.contentID))); + return ids.size; + } + + hasEntries(): boolean { + return this.entries.size > 0; + } + + /** "Foreign CDN references: N URLs in M items" */ + headline(): string { + const urls = this.entries.size; + const items = this.getItemCount(); + return `Foreign CDN references: ${urls} URL${urls === 1 ? "" : "s"} in ${items} item${items === 1 ? "" : "s"}`; + } + + /** What is wrong with the URL and what happens to it; shared by every surface. */ + describe(container: string, kinds: ForeignCdnFieldKind[]): string { + const own = (list: string[]) => (list.length > 0 ? list.join(", ") : "unknown"); + const consequences: string[] = []; + if (kinds.includes("attachment")) { + consequences.push("it is set in an image/file field, which the target may drop (the field arrives empty)"); + } + if (kinds.includes("text")) { + consequences.push( + "it is embedded in a text/rich-text field, so the synced content keeps linking to that container" + ); + } + return ( + `references CDN container '${container}', which belongs to neither the source (${own(this.sourceContainers)}) ` + + `nor the target (${own(this.targetContainers)}). The URL is passed through unchanged; ${consequences.join("; ")}. ` + + `Repoint it at the source instance's own asset.` + ); + } + + /** One log line for a single reference, used the first time its URL is seen. */ + formatOccurrence( + ref: Pick, + item: { contentID: number; referenceName: string; locale: string } + ): string { + const where = ref.kind === "attachment" ? "image/file field" : "text field"; + return ( + `⚠️ Foreign CDN reference: ${item.referenceName} (contentID: ${item.contentID}) [${item.locale}] ${where} '${ref.field}' ` + + `→ ${ref.url} ${this.describe(ref.container, [ref.kind])}` + ); + } + + private formatOccurrenceList(occurrences: ForeignCdnOccurrence[]): string { + return occurrences + .map( + (o) => + `${o.referenceName} #${o.contentID} [${o.locale}] ${o.field}${o.kind === "attachment" ? " (image/file)" : ""}` + ) + .join("; "); + } + + /** Lines for the end-of-run summary: headline, then each URL with the items using it. */ + renderSummaryLines(): string[] { + if (!this.hasEntries()) return []; + const lines = [`${this.headline()} (non-blocking):`]; + for (const e of this.getEntries()) { + const kinds = Array.from(new Set(e.occurrences.map((o) => o.kind))); + const risk = kinds.includes("attachment") ? "image/file field" : "text only"; + lines.push(` • ${e.url} — container '${e.container}', ${risk}`); + lines.push(` used by: ${this.formatOccurrenceList(e.occurrences)}`); + } + lines.push( + ` These URLs belong to neither the source (${this.sourceContainers.join(", ") || "unknown"}) nor the target ` + + `(${this.targetContainers.join(", ") || "unknown"}) and are passed through unchanged. Repoint them at the source instance's own assets.` + ); + return lines; + } + + /** One --jsonSummary warning per distinct URL, in getEntries() order. */ + toWarningDetails(sourceGuid?: string): ForeignCdnWarningDetail[] { + return this.getEntries().map((e) => { + const kinds = Array.from(new Set(e.occurrences.map((o) => o.kind))); + const first = e.occurrences[0]; + return { + name: e.url, + error: `${this.describe(e.container, kinds)} Used by: ${this.formatOccurrenceList(e.occurrences)}.`, + type: "content" as const, + contentID: first?.contentID, + guid: sourceGuid, + locale: first?.locale, + code: FOREIGN_CDN_WARNING_CODE, + url: e.url, + container: e.container, + references: e.occurrences, + }; + }); + } +} + +/** Process-level singleton shared by the content pusher and the end-of-run summary. */ +export const foreignCdnReport = new ForeignCdnReport(); diff --git a/src/lib/content/tests/foreign-cdn-detector.test.ts b/src/lib/content/tests/foreign-cdn-detector.test.ts new file mode 100644 index 0000000..c05603d --- /dev/null +++ b/src/lib/content/tests/foreign-cdn-detector.test.ts @@ -0,0 +1,215 @@ +import { + buildCdnOwnership, + collectForeignCdnReferences, + extractAgilityCdnUrls, + ForeignCdnDetector, + isGenericAgilityCdnHost, + parseContainerUrl, +} from "lib/content/foreign-cdn-detector"; + +// A clone pair: source container on the aglty.io edge and a custom domain, target likewise. +const SOURCE = "clone-qa-20261002"; +const TARGET = "clone-cat-20261002"; +const mapping = { + sourceUrl: `https://cdn.example.com/${SOURCE}/a/b.png`, + targetUrl: `https://cdn.example.com/${TARGET}/a/b.png`, + sourceContainerEdgeUrl: `https://cdn.example.com/${SOURCE}`, + targetContainerEdgeUrl: `https://cdn.example.com/${TARGET}`, + sourceContainerOriginUrl: `https://agility-cdn-usa2.azurewebsites.net/${SOURCE}`, + targetContainerOriginUrl: `https://agility-cdn-usa2.azurewebsites.net/${TARGET}`, +}; + +function detector(): ForeignCdnDetector { + return new ForeignCdnDetector(buildCdnOwnership({ assetMappings: [mapping] })); +} + +describe("isGenericAgilityCdnHost", () => { + it("matches the Agility CDN edge, origin and legacy hosts", () => { + expect(isGenericAgilityCdnHost("cdn.aglty.io")).toBe(true); + expect(isGenericAgilityCdnHost("cdn-usa2.aglty.io")).toBe(true); + expect(isGenericAgilityCdnHost("agility-cdn-usa2.azurewebsites.net")).toBe(true); + expect(isGenericAgilityCdnHost("cdn.agilitycms.com")).toBe(true); + }); + + it("does not match non-CDN Agility hosts or unrelated hosts", () => { + expect(isGenericAgilityCdnHost("api.aglty.io")).toBe(false); + expect(isGenericAgilityCdnHost("manager.agilitycms.com")).toBe(false); + expect(isGenericAgilityCdnHost("myapp.azurewebsites.net")).toBe(false); + expect(isGenericAgilityCdnHost("www.example.org")).toBe(false); + }); +}); + +describe("parseContainerUrl", () => { + it("returns the host and lower-cased first path segment", () => { + expect(parseContainerUrl("https://cdn-usa2.aglty.io/Upstream-QA/x.png")).toEqual({ + host: "cdn-usa2.aglty.io", + container: "upstream-qa", + }); + }); + + it("returns null for missing or unparseable input", () => { + expect(parseContainerUrl(undefined)).toBeNull(); + expect(parseContainerUrl("not a url")).toBeNull(); + expect(parseContainerUrl("https://cdn.aglty.io/")).toBeNull(); + }); +}); + +describe("buildCdnOwnership", () => { + it("learns each side's containers and custom edge hosts from asset mappings", () => { + const own = buildCdnOwnership({ assetMappings: [mapping] }); + expect(own.sourceContainers).toEqual([SOURCE]); + expect(own.targetContainers).toEqual([TARGET]); + expect(own.customHosts).toEqual(["cdn.example.com"]); + }); + + it("learns containers from pulled assets when there are no mappings", () => { + const own = buildCdnOwnership({ + sourceAssets: [ + { containerEdgeUrl: "https://cdn-usa2.aglty.io/src-c", edgeUrl: "https://cdn-usa2.aglty.io/src-c/x.png" }, + ], + targetAssets: [{ originUrl: "https://agility-cdn-usa2.azurewebsites.net/tgt-c/y.png" }], + }); + expect(own.sourceContainers).toEqual(["src-c"]); + expect(own.targetContainers).toEqual(["tgt-c"]); + expect(own.customHosts).toEqual([]); + }); + + it("falls back to per-asset URLs for legacy mapping records", () => { + const own = buildCdnOwnership({ + assetMappings: [ + { sourceUrl: "https://cdn.aglty.io/old-src/a.png", targetUrl: "https://cdn.aglty.io/old-tgt/a.png" }, + ], + }); + expect(own.sourceContainers).toEqual(["old-src"]); + expect(own.targetContainers).toEqual(["old-tgt"]); + }); +}); + +describe("extractAgilityCdnUrls", () => { + it("finds an aglty.io URL and its container", () => { + expect(extractAgilityCdnUrls("https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png")).toEqual([ + { + url: "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png", + host: "cdn-usa2.aglty.io", + container: "upstream-qa", + }, + ]); + }); + + it("finds an azurewebsites origin URL", () => { + const [m] = extractAgilityCdnUrls(''); + expect(m.container).toBe("other"); + }); + + it("ignores custom-domain URLs unless the host is known, and then requires a file name", () => { + const html = + 'x '; + expect(extractAgilityCdnUrls(html)).toEqual([]); + expect(extractAgilityCdnUrls(html, ["cdn.example.com"]).map((m) => m.url)).toEqual([ + "https://cdn.example.com/foo/x.svg", + ]); + }); + + it("unescapes JSON-escaped slashes", () => { + const json = '{"img":"https:\\/\\/cdn.aglty.io\\/other\\/a.png"}'; + expect(extractAgilityCdnUrls(json).map((m) => m.url)).toEqual(["https://cdn.aglty.io/other/a.png"]); + }); + + it("strips query strings (including & variants) and fragments", () => { + const html = + ''; + expect(extractAgilityCdnUrls(html).map((m) => m.url)).toEqual([ + "https://cdn.aglty.io/other/a.png", + "https://cdn.aglty.io/other/b.png", + ]); + }); + + it("finds percent-encoded URLs nested in a query string", () => { + const text = "https://images.example.com/resize?src=https%3A%2F%2Fcdn.aglty.io%2Fother%2Fa.png&w=10"; + expect(extractAgilityCdnUrls(text).map((m) => m.container)).toEqual(["other"]); + }); + + it("finds protocol-relative URLs and drops trailing punctuation", () => { + expect(extractAgilityCdnUrls("see //cdn.aglty.io/other/a.png.").map((m) => m.url)).toEqual([ + "//cdn.aglty.io/other/a.png", + ]); + }); + + it("ignores bare container URLs and non-CDN Agility hosts", () => { + expect(extractAgilityCdnUrls("https://cdn.aglty.io/other")).toEqual([]); + expect(extractAgilityCdnUrls("https://api.aglty.io/abc-u/fetch/en-us/item/1")).toEqual([]); + expect(extractAgilityCdnUrls("https://manager.agilitycms.com/instance/abc/x")).toEqual([]); + }); + + it("returns every occurrence, duplicates included", () => { + expect(extractAgilityCdnUrls("https://cdn.aglty.io/o/a.png https://cdn.aglty.io/o/a.png")).toHaveLength(2); + }); +}); + +describe("ForeignCdnDetector", () => { + it("does not flag the source's or target's own containers on any host", () => { + const d = detector(); + const text = [ + `https://cdn-usa2.aglty.io/${SOURCE}/x.png`, + `https://cdn.example.com/${SOURCE}/x.png`, + `https://agility-cdn-usa2.azurewebsites.net/${SOURCE}/x.png`, + `https://cdn-usa2.aglty.io/${TARGET}/x.png`, + `https://cdn.example.com/${TARGET.toUpperCase()}/x.png`, + ].join(" "); + expect(d.findForeignUrls(text)).toEqual([]); + }); + + it("flags a third instance's container on aglty.io and on the shared custom domain", () => { + const d = detector(); + const text = + "https://cdn-usa2.aglty.io/upstream-qa/a.png https://cdn.example.com/site-qa/upload/b.svg"; + expect(d.findForeignUrls(text).map((m) => m.container)).toEqual(["upstream-qa", "site-qa"]); + }); + + it("reports nothing when either side's container is unknown", () => { + const sourceOnly = new ForeignCdnDetector({ sourceContainers: [SOURCE], targetContainers: [], customHosts: [] }); + const targetOnly = new ForeignCdnDetector({ sourceContainers: [], targetContainers: [TARGET], customHosts: [] }); + expect(sourceOnly.isEnabled()).toBe(false); + expect(targetOnly.isEnabled()).toBe(false); + expect(sourceOnly.findForeignUrls("https://cdn.aglty.io/other/a.png")).toEqual([]); + expect(targetOnly.findForeignUrls("https://cdn.aglty.io/other/a.png")).toEqual([]); + }); +}); + +describe("collectForeignCdnReferences", () => { + const foreign = "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png"; + + it("classifies ImageAttachment, AttachmentList, plain URL and rich text fields", () => { + const fields = { + mobileBackgroundImage: { label: "Hero Image", url: foreign, target: null }, + gallery: [{ url: `https://cdn.aglty.io/upstream-qa/g1.png` }, { url: `https://cdn.aglty.io/${SOURCE}/g2.png` }], + imagesrc: "https://cdn.aglty.io/othersite/footer/find-a-store.png", + body: `

`, + title: "No URLs here", + }; + const refs = collectForeignCdnReferences(fields, detector()); + expect(refs.map(({ field, kind, container }) => ({ field, kind, container }))).toEqual([ + { field: "mobileBackgroundImage", kind: "attachment", container: "upstream-qa" }, + { field: "gallery[0]", kind: "attachment", container: "upstream-qa" }, + { field: "imagesrc", kind: "text", container: "othersite" }, + { field: "body", kind: "text", container: "upstream-qa" }, + ]); + }); + + it("reports a URL repeated within one field once", () => { + const refs = collectForeignCdnReferences({ body: `` }, detector()); + expect(refs).toHaveLength(1); + }); + + it("never mutates the fields it scans", () => { + const fields = { a: { url: foreign }, b: [{ url: foreign }], c: `` }; + const before = JSON.stringify(fields); + collectForeignCdnReferences(Object.freeze(fields), detector()); + expect(JSON.stringify(fields)).toBe(before); + }); + + it("returns nothing when the detector is disabled", () => { + const off = new ForeignCdnDetector({ sourceContainers: [], targetContainers: [], customHosts: [] }); + expect(collectForeignCdnReferences({ a: { url: foreign } }, off)).toEqual([]); + }); +}); diff --git a/src/lib/content/tests/foreign-cdn-report.test.ts b/src/lib/content/tests/foreign-cdn-report.test.ts new file mode 100644 index 0000000..1085a77 --- /dev/null +++ b/src/lib/content/tests/foreign-cdn-report.test.ts @@ -0,0 +1,211 @@ +import { foreignCdnReport } from "lib/content/foreign-cdn-report"; +import { ForeignCdnDetector } from "lib/content/foreign-cdn-detector"; +import { ContentFieldMapper } from "lib/content/content-field-mapper"; +import { reportForeignCdnReferences } from "lib/pushers/content-pusher/util/report-foreign-cdn-references"; + +const SOURCE = "site-qa"; +const TARGET = "site-cat"; +const FOREIGN = "https://cdn-usa2.aglty.io/upstream-qa/hero-background-image.png"; +const FOREIGN_TEXT = "https://cdn-usa2.aglty.io/upstream-qa/navigation-icons/contact-us-success-image.png"; + +const detector = new ForeignCdnDetector({ sourceContainers: [SOURCE], targetContainers: [TARGET], customHosts: [] }); + +function item(contentID: number, referenceName: string, fields: any): any { + return { contentID, properties: { referenceName }, fields }; +} + +function makeLogger() { + return { warning: jest.fn(), fileOnly: jest.fn() }; +} + +beforeEach(() => { + foreignCdnReport.reset(); + jest.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +describe("foreignCdnReport", () => { + it("dedupes to one entry per URL, aggregating items, fields and locales", () => { + const logger = makeLogger(); + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger, detector }; + const banner = (id: number) => + item(id, "home_homebanner_homebanner", { mobileBackgroundImage: { label: "x", url: FOREIGN } }); + + reportForeignCdnReferences([banner(474), banner(475)], "en-us", opts); + reportForeignCdnReferences([banner(474)], "pt-br", opts); + // the same item/locale/field scanned again (e.g. a second pass) is not a new occurrence + reportForeignCdnReferences([banner(474)], "pt-br", opts); + + const entries = foreignCdnReport.getEntries(); + expect(entries).toHaveLength(1); + expect(entries[0].container).toBe("upstream-qa"); + expect(entries[0].occurrences).toEqual([ + { + contentID: 474, + referenceName: "home_homebanner_homebanner", + locale: "en-us", + field: "mobileBackgroundImage", + kind: "attachment", + }, + { + contentID: 475, + referenceName: "home_homebanner_homebanner", + locale: "en-us", + field: "mobileBackgroundImage", + kind: "attachment", + }, + { + contentID: 474, + referenceName: "home_homebanner_homebanner", + locale: "pt-br", + field: "mobileBackgroundImage", + kind: "attachment", + }, + ]); + expect(foreignCdnReport.getItemCount()).toBe(2); + expect(foreignCdnReport.headline()).toBe("Foreign CDN references: 1 URL in 2 items"); + + // console/log warning once per URL; further occurrences go to the log file only + expect(logger.warning).toHaveBeenCalledTimes(1); + expect(logger.warning.mock.calls[0][0]).toContain("references CDN container 'upstream-qa'"); + expect(logger.fileOnly).toHaveBeenCalledTimes(2); + }); + + it("distinguishes image/file fields from text fields in the message", () => { + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger: makeLogger(), detector }; + reportForeignCdnReferences( + [ + item(474, "banner", { img: { url: FOREIGN } }), + item(1271, "contactus", { body: `

` }), + ], + "en-us", + opts + ); + + const [attachment, text] = foreignCdnReport.toWarningDetails("src-u"); + expect(attachment.error).toContain( + `references CDN container 'upstream-qa', which belongs to neither the source (${SOURCE}) nor the target (${TARGET}). The URL is passed through unchanged` + ); + expect(attachment.error).toContain("image/file field, which the target may drop"); + expect(attachment.error).toContain("Repoint it at the source instance's own asset."); + expect(attachment.error).not.toContain("text/rich-text field"); + expect(text.error).toContain("embedded in a text/rich-text field"); + expect(text.error).not.toContain("image/file field"); + }); + + it("builds additive --jsonSummary warning entries, attachment-field URLs first", () => { + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger: makeLogger(), detector }; + reportForeignCdnReferences( + [ + item(1271, "contactus", { body: `` }), + item(474, "banner", { img: { url: FOREIGN } }), + ], + "en-us", + opts + ); + + const details = foreignCdnReport.toWarningDetails("src-u"); + expect(details.map((d) => d.url)).toEqual([FOREIGN, FOREIGN_TEXT]); + expect(details[0]).toMatchObject({ + name: FOREIGN, + type: "content", + contentID: 474, + guid: "src-u", + locale: "en-us", + code: "foreignCdnReference", + container: "upstream-qa", + references: [{ contentID: 474, referenceName: "banner", locale: "en-us", field: "img", kind: "attachment" }], + }); + expect(details[0].error).toContain("Used by: banner #474 [en-us] img (image/file)"); + }); + + it("renders the end-of-run summary block", () => { + const opts = { sourceGuid: "src-u", targetGuid: "tgt-u", logger: makeLogger(), detector }; + reportForeignCdnReferences([item(474, "banner", { img: { url: FOREIGN } })], "en-us", opts); + const lines = foreignCdnReport.renderSummaryLines(); + expect(lines[0]).toBe("Foreign CDN references: 1 URL in 1 item (non-blocking):"); + expect(lines.join("\n")).toContain(`• ${FOREIGN} — container 'upstream-qa', image/file field`); + expect(lines.join("\n")).toContain("used by: banner #474 [en-us] img (image/file)"); + }); + + it("records nothing for the instances' own containers or when a side is unknown", () => { + const own = `https://cdn-usa2.aglty.io/${SOURCE}/a.png https://cdn.aglty.io/${TARGET}/b.png`; + reportForeignCdnReferences( + [item(1, "x", { body: own, img: { url: `https://cdn.aglty.io/${SOURCE}/c.png` } })], + "en-us", + { + sourceGuid: "s", + targetGuid: "t", + detector, + } + ); + const unknownTarget = new ForeignCdnDetector({ sourceContainers: [SOURCE], targetContainers: [], customHosts: [] }); + reportForeignCdnReferences([item(2, "x", { img: { url: FOREIGN } })], "en-us", { + sourceGuid: "s", + targetGuid: "t", + detector: unknownTarget, + }); + expect(foreignCdnReport.hasEntries()).toBe(false); + expect(foreignCdnReport.renderSummaryLines()).toEqual([]); + }); +}); + +describe("foreign CDN detection alongside ContentFieldMapper", () => { + // Detection is reporting only: the payload the field mapper builds must be exactly what it + // was before, for every field shape that can carry a foreign URL. + const fields = { + mobileBackgroundImage: { + label: "Hero Image", + url: FOREIGN, + target: null, + pixelHeight: null, + pixelWidth: null, + }, + attachments: [ + { label: "a", url: FOREIGN }, + { label: "b", url: "https://cdn.aglty.io/upstream-qa/b.pdf" }, + ], + imagesrc: "https://cdn.aglty.io/othersite/footer/find-a-store.png", + richText: `

`, + }; + + const context = (): any => ({ + referenceMapper: { getContentItemMappingByContentID: jest.fn().mockReturnValue(null) }, + assetMapper: { + getAssetMappingByMediaUrl: jest.fn().mockReturnValue(null), + remapUrlByContainer: jest.fn().mockReturnValue(null), + isKnownAssetUrl: jest.fn().mockReturnValue(false), + }, + }); + + it("finds every foreign reference while the mapped payload stays identical", () => { + const original = JSON.parse(JSON.stringify(fields)); + const before = new ContentFieldMapper().mapContentFields(original, context()); + + reportForeignCdnReferences([item(474, "banner", original)], "en-us", { + sourceGuid: "s", + targetGuid: "t", + logger: makeLogger(), + detector, + }); + + const after = new ContentFieldMapper().mapContentFields(original, context()); + expect(after).toEqual(before); + expect(after.mappedFields).toEqual(fields); + expect(original).toEqual(fields); + + const refs = foreignCdnReport.getEntries().flatMap((e) => e.occurrences.map((o) => `${o.field}:${o.kind}`)); + expect(refs.sort()).toEqual( + [ + "attachments[0]:attachment", + "attachments[1]:attachment", + "imagesrc:text", + "mobileBackgroundImage:attachment", + "richText:text", + ].sort() + ); + }); +}); diff --git a/src/lib/mappers/asset-mapper.ts b/src/lib/mappers/asset-mapper.ts index 47637ca..ead4904 100644 --- a/src/lib/mappers/asset-mapper.ts +++ b/src/lib/mappers/asset-mapper.ts @@ -49,6 +49,11 @@ export class AssetMapper { return mapping; } + /** PROD-2773: every record, read-only — the foreign CDN detector reads each side's container URLs. */ + getMappings(): ReadonlyArray { + return this.mappings; + } + getAssetMappingByMediaUrl(url: string, type: "source" | "target"): AssetMapping | null { // Try exact match first const exact = this.mappings.find((m: AssetMapping) => diff --git a/src/lib/preflight/preflight-report.ts b/src/lib/preflight/preflight-report.ts index c736b48..00dba94 100644 --- a/src/lib/preflight/preflight-report.ts +++ b/src/lib/preflight/preflight-report.ts @@ -63,6 +63,9 @@ const ACTION_GLYPH: Record = { class PreflightReport { private entries: PreflightEntry[] = []; + // PROD-2773: non-blocking notices shown under their own heading. Kept apart from `entries` + // so they never change a phase count, the totals, or the conflict exit code. + private warnings: string[] = []; /** Whether preflight mode is active for the current run. */ isEnabled(): boolean { @@ -72,6 +75,7 @@ class PreflightReport { /** Clear all recorded actions (called per-command via resetState). */ reset(): void { this.entries = []; + this.warnings = []; } /** @@ -83,6 +87,16 @@ class PreflightReport { this.entries.push(entry); } + /** PROD-2773: record a non-blocking warning line. No-op when preflight mode is off. */ + recordWarning(message: string): void { + if (!this.isEnabled()) return; + this.warnings.push(message); + } + + getWarnings(): string[] { + return this.warnings; + } + /** True if any item would conflict (used to set a non-zero exit code). */ hasConflicts(): boolean { return this.entries.some((e) => e.action === "conflict"); @@ -176,6 +190,7 @@ class PreflightReport { const phases = this.getPhaseSummaries(); if (phases.length === 0) { lines.push(ansiColors.gray("\nNothing to do — no creates, updates, skips, or conflicts detected.\n")); + lines.push(...this.renderWarnings()); return lines.join("\n"); } @@ -221,11 +236,21 @@ class PreflightReport { if (this.hasConflicts()) { lines.push(ansiColors.red("\n⚠️ Conflicts detected — a real sync would require --overwrite (exit code 1).")); } + lines.push(...this.renderWarnings()); lines.push(ansiColors.cyan(bar)); return lines.join("\n"); } + /** PROD-2773: the warnings section — not counted in the totals and never affects the exit code. */ + private renderWarnings(): string[] { + if (this.warnings.length === 0) return []; + return [ + ansiColors.yellow(`\nWarnings (${this.warnings.length}, non-blocking — not counted above):`), + ...this.warnings.map((w) => ansiColors.yellow(` ⚠ ${w}`)), + ]; + } + /** Print the report to stdout. */ print(): void { console.log(this.renderTable()); diff --git a/src/lib/preflight/tests/preflight-report.test.ts b/src/lib/preflight/tests/preflight-report.test.ts index ac2318a..ab10e6b 100644 --- a/src/lib/preflight/tests/preflight-report.test.ts +++ b/src/lib/preflight/tests/preflight-report.test.ts @@ -420,3 +420,42 @@ describe("invalid action (PROD-2542)", () => { expect(preflightReport.renderTable()).not.toContain("invalid"); }); }); + +// ─── warnings (PROD-2773) ───────────────────────────────────────────────────── + +describe("recordWarning", () => { + it("is a no-op when preflight is disabled", () => { + preflightReport.recordWarning("Foreign CDN references: 1 URL in 1 item"); + expect(preflightReport.getWarnings()).toEqual([]); + }); + + it("renders a warnings section without changing counts, totals or the conflict signal", () => { + setState({ preflight: true }); + preflightReport.record({ phase: "Content", action: "update", name: "banner", locale: "en-us" }); + const before = { totals: preflightReport.getTotals(), json: preflightReport.toJSON() }; + + preflightReport.recordWarning("Foreign CDN references: 14 URLs in 11 items — see the list above"); + + expect(preflightReport.getTotals()).toEqual(before.totals); + expect(preflightReport.toJSON()).toEqual(before.json); + expect(preflightReport.hasConflicts()).toBe(false); + const table = preflightReport.renderTable(); + expect(table).toContain("Warnings (1, non-blocking — not counted above):"); + expect(table).toContain("Foreign CDN references: 14 URLs in 11 items"); + }); + + it("renders warnings even when there is nothing to do", () => { + setState({ preflight: true }); + preflightReport.recordWarning("Foreign CDN references: 1 URL in 1 item"); + const table = preflightReport.renderTable(); + expect(table).toContain("Nothing to do"); + expect(table).toContain("Foreign CDN references: 1 URL in 1 item"); + }); + + it("is cleared by reset", () => { + setState({ preflight: true }); + preflightReport.recordWarning("x"); + preflightReport.reset(); + expect(preflightReport.getWarnings()).toEqual([]); + }); +}); diff --git a/src/lib/pushers/content-pusher/content-pusher.ts b/src/lib/pushers/content-pusher/content-pusher.ts index 40f4769..1c2c076 100644 --- a/src/lib/pushers/content-pusher/content-pusher.ts +++ b/src/lib/pushers/content-pusher/content-pusher.ts @@ -9,6 +9,7 @@ import { ModelMapper } from "lib/mappers/model-mapper"; import { ContentItem, Model } from "@agility/management-sdk"; import { ContainerMapper } from "lib/mappers/container-mapper"; import { getApiClient } from "core/state"; +import { reportForeignCdnReferences } from "./util/report-foreign-cdn-references"; /** * Push content to the target instance @@ -40,6 +41,19 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content logger: logger as any, }); + // PROD-2773: warn about asset URLs into a CDN container owned by neither instance. Reporting + // only — runs before change detection (and in --preflight) and never alters an item. Items + // skipped above for missing container/model mappings are not synced, so they are not scanned. + try { + reportForeignCdnReferences([...linkedContentItems, ...normalContentItems], locale, { + sourceGuid: sourceGuidStr, + targetGuid: targetGuidStr, + logger, + }); + } catch (scanError: any) { + logger?.fileOnly?.(`Foreign CDN reference scan failed: ${scanError?.message || scanError}`); + } + let totalSuccessful = 0; let totalFailed = 0; let totalSkipped = 0; diff --git a/src/lib/pushers/content-pusher/util/report-foreign-cdn-references.ts b/src/lib/pushers/content-pusher/util/report-foreign-cdn-references.ts new file mode 100644 index 0000000..ef420b0 --- /dev/null +++ b/src/lib/pushers/content-pusher/util/report-foreign-cdn-references.ts @@ -0,0 +1,74 @@ +import { ContentItem } from "@agility/management-sdk"; +import { AssetMapper } from "lib/mappers/asset-mapper"; +import { fileOperations } from "core/fileOperations"; +import { getAssetsFromFileSystem } from "lib/getters/filesystem/get-assets"; +import { buildCdnOwnership, collectForeignCdnReferences, ForeignCdnDetector } from "lib/content/foreign-cdn-detector"; +import { foreignCdnReport } from "lib/content/foreign-cdn-report"; + +// One detector per source/target pair per process: building it reads every pulled asset file. +// Asset mappings created later in the same run only add containers already known from the pulled +// assets, so reusing it across the content phases of every locale is safe. +let cachedDetector: { key: string; detector: ForeignCdnDetector } | null = null; + +export function resetForeignCdnDetectorCache(): void { + cachedDetector = null; +} + +export function getForeignCdnDetector(sourceGuid: string, targetGuid: string): ForeignCdnDetector { + const key = `${sourceGuid}->${targetGuid}`; + if (cachedDetector?.key === key) return cachedDetector.detector; + + const read = (fn: () => T[]): T[] => { + try { + return fn() || []; + } catch { + return []; // best-effort: an unreadable side just stays unknown (no warnings) + } + }; + const assetMappings = read(() => [...new AssetMapper(sourceGuid, targetGuid).getMappings()]); + const sourceAssets = read(() => getAssetsFromFileSystem(new fileOperations(sourceGuid))); + const targetAssets = read(() => getAssetsFromFileSystem(new fileOperations(targetGuid))); + + const detector = new ForeignCdnDetector(buildCdnOwnership({ assetMappings, sourceAssets, targetAssets })); + cachedDetector = { key, detector }; + return detector; +} + +/** + * PROD-2773: scan the content items of one locale for foreign CDN references and record them in + * foreignCdnReport. Runs in both real and --preflight runs, before change detection, so an + * unchanged item still carrying a foreign URL is reported too. Reporting only — the items are not + * touched. Logs a warning the first time each URL is seen; repeat sightings go to the log file only. + */ +export function reportForeignCdnReferences( + contentItems: ContentItem[], + locale: string, + opts: { sourceGuid: string; targetGuid: string; logger?: any; detector?: ForeignCdnDetector } +): number { + const detector = opts.detector ?? getForeignCdnDetector(opts.sourceGuid, opts.targetGuid); + if (!detector.isEnabled()) return 0; + foreignCdnReport.setOwnContainers(detector.sourceContainers, detector.targetContainers); + + let found = 0; + for (const contentItem of contentItems) { + const refs = collectForeignCdnReferences(contentItem?.fields, detector); + for (const ref of refs) { + const item = { + contentID: contentItem.contentID, + referenceName: contentItem.properties?.referenceName ?? "unknown", + locale, + }; + const outcome = foreignCdnReport.record(ref, item); + if (outcome === "duplicate") continue; + found++; + const message = foreignCdnReport.formatOccurrence(ref, item); + if (outcome === "new-url") { + if (opts.logger?.warning) opts.logger.warning(message); + else console.warn(message); + } else { + opts.logger?.fileOnly?.(message); + } + } + } + return found; +} diff --git a/src/lib/pushers/content-pusher/util/tests/report-foreign-cdn-references.test.ts b/src/lib/pushers/content-pusher/util/tests/report-foreign-cdn-references.test.ts new file mode 100644 index 0000000..4044157 --- /dev/null +++ b/src/lib/pushers/content-pusher/util/tests/report-foreign-cdn-references.test.ts @@ -0,0 +1,78 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { resetState, setState } from "core/state"; +import { foreignCdnReport } from "lib/content/foreign-cdn-report"; +import { + getForeignCdnDetector, + reportForeignCdnReferences, + resetForeignCdnDetectorCache, +} from "../report-foreign-cdn-references"; + +let tmpDir: string; + +function writeJson(rel: string, data: any) { + const file = path.join(tmpDir, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); +} + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agility-foreign-cdn-")); + resetState(); + setState({ rootPath: tmpDir }); + resetForeignCdnDetectorCache(); + foreignCdnReport.reset(); + jest.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + jest.restoreAllMocks(); +}); + +describe("getForeignCdnDetector", () => { + it("learns the source container from asset mappings and the target's from its pulled assets", () => { + writeJson("mappings/src-u-tgt-u/assets/mappings.json", [ + { + sourceMediaID: 1, + targetMediaID: 2, + sourceContainerEdgeUrl: "https://cdn.example-media.com/src-container", + sourceContainerOriginUrl: "https://agility-cdn-usa2.azurewebsites.net/src-container", + }, + ]); + writeJson("tgt-u/assets/json/1.json", { + assetMedias: [{ mediaID: 2, containerEdgeUrl: "https://cdn-usa2.aglty.io/tgt-container" }], + }); + + const detector = getForeignCdnDetector("src-u", "tgt-u"); + expect(detector.isEnabled()).toBe(true); + expect(detector.sourceContainers).toEqual(["src-container"]); + expect(detector.targetContainers).toEqual(["tgt-container"]); + // the custom edge host is learned, so a third instance's container on it is recognised + expect(detector.findForeignUrls("https://cdn.example-media.com/other/a.png").map((m) => m.container)).toEqual([ + "other", + ]); + }); + + it("is disabled when the target has no assets and no mappings yet", () => { + writeJson("src-u/assets/json/1.json", { + assetMedias: [{ mediaID: 1, containerEdgeUrl: "https://cdn-usa2.aglty.io/src-container" }], + }); + const detector = getForeignCdnDetector("src-u", "tgt-u"); + expect(detector.isEnabled()).toBe(false); + + reportForeignCdnReferences( + [ + { + contentID: 1, + properties: { referenceName: "x" }, + fields: { img: { url: "https://cdn.aglty.io/other/a.png" } }, + } as any, + ], + "en-us", + { sourceGuid: "src-u", targetGuid: "tgt-u" } + ); + expect(foreignCdnReport.hasEntries()).toBe(false); + }); +}); From c72b8a526913d984a4dcfa0e0a46f2b4f95deb84 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 14 Oct 2026 14:15:36 -0400 Subject: [PATCH 3/3] PROD-2782: republish a page's other locale when a save in one locale puts it in staging A page's zones are shared by all of its locales. When a save in one locale adds, removes or reorders components, the server cuts a new page version and gives every other language a staging row on it (ClearPageContent / ReorderPageContent), so the page drops to Staging in its other locales. --autoPublish only republished pages the run saved in that locale, so the other locale stayed in Staging, and the PROD-2628 refresh then recorded its new version as in sync, so later runs skipped it as up to date. refreshOtherLocaleVersions already re-reads the page in each other locale of the run after a save. It now records the page for republishing in that locale when it was Published on the target when pulled, is Staging now, and is Published on the source (a page left in staging on the source stays in staging). executeAutoPublish adds those pages to that locale's publish list (pages/both), de-duplicated, and updates their mappings as usual. A page whose own save in that locale fails is never republished, whichever order the saves ran in. Without auto-publish of pages, the run lists them as non-blocking "staging" warnings in the console and --jsonSummary. Locales not in the run are not checked: there is no pulled state for them. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 + src/core/json-summary.ts | 7 +- src/core/push.ts | 76 +++++++++++- src/core/state.ts | 61 ++++++++++ src/core/tests/json-summary.test.ts | 3 +- src/core/tests/push-auto-publish.test.ts | 89 ++++++++++++++ src/core/tests/push.test.ts | 60 +++++++++- .../pushers/page-pusher/process-sitemap.ts | 8 +- .../refresh-other-locale-versions.ts | 24 ++++ .../page-pusher/tests/process-sitemap.test.ts | 31 ++++- .../refresh-other-locale-versions.test.ts | 112 ++++++++++++++++-- 11 files changed, 459 insertions(+), 14 deletions(-) create mode 100644 src/core/tests/push-auto-publish.test.ts diff --git a/README.md b/README.md index 05a4ecb..ab351a0 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ The `--autoPublish` flag lets you automatically publish synced content and/or pa 4. Publishing is done per-locale to match the workflow API requirements 5. After publishing, reference mappings are updated to reflect the new published versions +A page's components are shared by all of its locales, so a save that adds, removes or reorders components in one locale also puts the page in staging in its other locales. When the page was published in another locale of the run before the save, and is published in the source, `pages` and `both` republish it in that locale too. With `content` or no `--autoPublish`, the sync lists those pages under "Pages Left in Staging by Another Locale's Save" (and in `--jsonSummary` under `autoPublish.warnings`, type `staging`). Locales not included in the run are not checked. + #### Examples ```bash diff --git a/src/core/json-summary.ts b/src/core/json-summary.ts index 801b95b..9e62f3f 100644 --- a/src/core/json-summary.ts +++ b/src/core/json-summary.ts @@ -301,9 +301,12 @@ export function buildJsonSummary(input: BuildJsonSummaryInput): JsonSummary { ); // Same split the console summary uses (PROD-2311): "publish"/"fatal" block the run, - // "mapping"/"refresh" are post-publish bookkeeping and deliberately do not. + // "mapping"/"refresh" are post-publish bookkeeping and deliberately do not, nor does "staging" + // (PROD-2782: a page another locale's save left in staging, not republished). const autoPublishErrors = input.autoPublishErrors.filter((e) => e.type === "publish" || e.type === "fatal"); - const autoPublishWarnings = input.autoPublishErrors.filter((e) => e.type === "mapping" || e.type === "refresh"); + const autoPublishWarnings = input.autoPublishErrors.filter( + (e) => e.type === "mapping" || e.type === "refresh" || e.type === "staging" + ); return { schemaVersion: JSON_SUMMARY_SCHEMA_VERSION, diff --git a/src/core/push.ts b/src/core/push.ts index 1b00072..1143765 100644 --- a/src/core/push.ts +++ b/src/core/push.ts @@ -6,6 +6,8 @@ import { clearFailedContentRegistry, clearHeldBackTemplateRegistry, clearEffectiveTargetModels, + clearOtherLocaleStagingRegistry, + getOtherLocaleStaging, getPageCmsLink, getContentCmsLink, } from "./state"; @@ -21,6 +23,48 @@ import { buildJsonSummary, writeJsonSummary, JsonSummaryDetail, JsonSummaryPhase import type { FailureDetail } from "../types/sourceData"; import { foreignCdnReport } from "../lib/content/foreign-cdn-report"; import { resetForeignCdnDetectorCache } from "../lib/pushers/content-pusher/util/report-foreign-cdn-references"; +import type { OtherLocaleStagingEntry } from "./state"; + +/** + * PROD-2782: whether an --autoPublish mode publishes pages ("pages" or "both"). + */ +export function autoPublishIncludesPages(autoPublishMode: string | undefined): boolean { + return autoPublishMode === "pages" || autoPublishMode === "both"; +} + +/** + * PROD-2782: add the pages a save in one locale put in staging in another locale to that locale's + * page publish list. Duplicates are left in; executeAutoPublish de-duplicates per locale. + */ +export function mergeOtherLocaleStaging( + pageIdsByLocale: Map, + stagingByLocale: Map +): void { + stagingByLocale.forEach((entries, locale) => { + const existing = pageIdsByLocale.get(locale) || []; + pageIdsByLocale.set(locale, [...existing, ...entries.map((e) => e.targetPageID)]); + }); +} + +/** + * PROD-2782: non-blocking "staging" notices for pages a save in one locale put in staging in + * another locale, used when auto-publish does not publish pages and so cannot republish them. + */ +export function otherLocaleStagingWarnings( + stagingByLocale: Map +): Array<{ locale: string; type: string; error: string }> { + const warnings: Array<{ locale: string; type: string; error: string }> = []; + stagingByLocale.forEach((entries, locale) => { + for (const { targetPageID, pageName, savedLocale } of entries) { + warnings.push({ + locale, + type: "staging", + error: `page "${pageName}" (target page ${targetPageID}) was put in staging by the ${savedLocale} save and was not republished (auto-publish does not include pages)`, + }); + } + }); + return warnings; +} /** * PROD-2310: Decide whether a set of auto-publish errors should fail the sync exit code. @@ -61,6 +105,7 @@ export class Push { clearHeldBackTemplateRegistry(); // PROD-2542: models saved by a previous run must not stand in for this run's target models clearEffectiveTargetModels(); + clearOtherLocaleStagingRegistry(); // Preflight (PROD-2203): start each run with a clean report of planned actions preflightReport.reset(); @@ -210,6 +255,12 @@ export class Push { autoPublishErrors = await this.executeAutoPublish(results, autoPublish); } + // PROD-2782: pages a save in one locale put in staging in another locale are republished by + // auto-publish when it covers pages. Otherwise, say which ones were left in staging. + if (!autoPublishIncludesPages(autoPublish) || !isSync || state.preflight) { + autoPublishErrors.push(...otherLocaleStagingWarnings(getOtherLocaleStaging())); + } + // Split auto-publish outcomes: genuine publish failures (blocking) vs // post-publish mapping/refresh bookkeeping notices (non-blocking). PROD-2311 const genuinePublishErrors = autoPublishErrors.filter( @@ -218,6 +269,7 @@ export class Push { const mappingWarnings = autoPublishErrors.filter( ({ type }) => type === "mapping" || type === "refresh" ); + const stagingWarnings = autoPublishErrors.filter(({ type }) => type === "staging"); // PROD-2310: Fold auto-publish failures into the returned success signal so the // entry point can exit non-zero. "mapping"/"refresh" entries are post-publish @@ -322,6 +374,16 @@ export class Push { }); } + // PROD-2782: pages left in staging in a locale by another locale's save (no auto-publish of + // pages). Non-blocking: nothing was lost, the page needs publishing in that locale. + if (stagingWarnings.length > 0) { + console.log(ansiColors.yellow(`\n Pages Left in Staging by Another Locale's Save (non-blocking):`)); + stagingWarnings.forEach(({ locale, error }) => { + const localeDisplay = locale ? `[${locale}]` : ""; + console.log(ansiColors.yellow(` • ${localeDisplay} ${error}`)); + }); + } + // Show log file paths at the very end if (logFilePaths.length > 0) { console.log(ansiColors.cyan("\n📄 Log Files:")); @@ -494,7 +556,14 @@ export class Push { // Determine what to publish based on mode const publishContent = autoPublishMode === "content" || autoPublishMode === "both"; - const publishPages = autoPublishMode === "pages" || autoPublishMode === "both"; + const publishPages = autoPublishIncludesPages(autoPublishMode); + + // PROD-2782: a page saved in one locale may have been put in staging in another locale that + // this run did not save it in. Republish it there too. + const otherLocaleStaging = getOtherLocaleStaging(); + if (publishPages && otherLocaleStaging.size > 0) { + mergeOtherLocaleStaging(pageIdsByLocale, otherLocaleStaging); + } // Get all locales that have items to publish const allLocales = new Set(); @@ -527,6 +596,11 @@ export class Push { console.log(ansiColors.gray(`Locales to publish: ${Array.from(allLocales).join(", ")}`)); console.log(ansiColors.gray(`Total content items: ${totalContent}`)); console.log(ansiColors.gray(`Total pages: ${totalPages}`)); + if (publishPages && otherLocaleStaging.size > 0) { + let count = 0; + otherLocaleStaging.forEach((entries) => (count += entries.length)); + console.log(ansiColors.gray(` incl. ${count} page(s) put in staging by a save in another locale`)); + } // Import workflow dependencies const { processBatches } = await import("../lib/workflows/process-batches"); diff --git a/src/core/state.ts b/src/core/state.ts index 6367c63..4c20d3d 100644 --- a/src/core/state.ts +++ b/src/core/state.ts @@ -120,6 +120,20 @@ export interface State { // Held-back template registry - source template names whose mapped target template is gone and // was not recreated (no --overwrite). Used by page pusher to skip pages that depend on them. heldBackTemplateRegistry: Map; + + // Other-locale staging registry (PROD-2782) - pages that a save in one locale put in staging in + // another locale, keyed by that other locale, then by target page ID. Auto-publish republishes + // them; without auto-publish they are reported as warnings. + otherLocaleStagingRegistry: Map>; + // "locale:targetPageID" for pages whose own save in that locale failed this run - never republished + otherLocaleStagingBlocked: Set; +} + +export interface OtherLocaleStagingEntry { + targetPageID: number; + pageName: string; + /** The locale whose save put this page in staging. */ + savedLocale: string; } // Global state - populated from argv and referenced throughout the app @@ -199,6 +213,10 @@ export const state: State = { // Held-back template registry - templates whose mapped target is gone and was not recreated heldBackTemplateRegistry: new Map(), + + // Other-locale staging registry - pages a save in one locale put in staging in another (PROD-2782) + otherLocaleStagingRegistry: new Map(), + otherLocaleStagingBlocked: new Set(), }; /** @@ -848,6 +866,49 @@ export function clearHeldBackTemplateRegistry(): void { state.heldBackTemplateRegistry.clear(); } +/** + * Record a page that a save in another locale put in staging in `locale` (PROD-2782) + * @param locale - The locale the page went to staging in + * @param entry - The target page and the locale whose save caused it + */ +export function registerOtherLocaleStaging(locale: string, entry: OtherLocaleStagingEntry): void { + if (state.otherLocaleStagingBlocked.has(`${locale}:${entry.targetPageID}`)) return; + let pages = state.otherLocaleStagingRegistry.get(locale); + if (!pages) { + pages = new Map(); + state.otherLocaleStagingRegistry.set(locale, pages); + } + pages.set(entry.targetPageID, entry); +} + +/** + * Never republish this page in `locale` this run - its own save in that locale failed. Drops it if + * it was already recorded, and keeps a later save in another locale from recording it again. + */ +export function blockOtherLocaleStaging(locale: string, targetPageID: number): void { + state.otherLocaleStagingRegistry.get(locale)?.delete(targetPageID); + state.otherLocaleStagingBlocked.add(`${locale}:${targetPageID}`); +} + +/** + * The pages recorded for each locale, as arrays + */ +export function getOtherLocaleStaging(): Map { + const result = new Map(); + state.otherLocaleStagingRegistry.forEach((pages, locale) => { + if (pages.size > 0) result.set(locale, Array.from(pages.values())); + }); + return result; +} + +/** + * Clear the other-locale staging registry (should be called at start of each sync) + */ +export function clearOtherLocaleStagingRegistry(): void { + state.otherLocaleStagingRegistry.clear(); + state.otherLocaleStagingBlocked.clear(); +} + /** * Get the CMS app URL based on environment * - dev/local/preprod: app-qa.publishwithagility.com diff --git a/src/core/tests/json-summary.test.ts b/src/core/tests/json-summary.test.ts index 9034769..c43f391 100644 --- a/src/core/tests/json-summary.test.ts +++ b/src/core/tests/json-summary.test.ts @@ -125,12 +125,13 @@ describe("buildJsonSummary — auto-publish classification", () => { { type: "fatal", error: "crashed" }, { type: "mapping", error: "mapping refresh skipped" }, { type: "refresh", error: "target refresh failed" }, + { type: "staging", error: "left in staging by another locale's save", locale: "pt-br" }, ], }) ); expect(summary.autoPublish.errors.map((e) => e.type)).toEqual(["publish", "fatal"]); - expect(summary.autoPublish.warnings.map((e) => e.type)).toEqual(["mapping", "refresh"]); + expect(summary.autoPublish.warnings.map((e) => e.type)).toEqual(["mapping", "refresh", "staging"]); }); it("yields empty groups when auto-publish reported nothing", () => { diff --git a/src/core/tests/push-auto-publish.test.ts b/src/core/tests/push-auto-publish.test.ts new file mode 100644 index 0000000..8f6e0fb --- /dev/null +++ b/src/core/tests/push-auto-publish.test.ts @@ -0,0 +1,89 @@ +/** + * PROD-2782: executeAutoPublish republishes pages that a save in one locale put in staging in + * another locale, alongside the pages the run saved. + */ +import { Push } from "../push"; +import { resetState, setState, registerOtherLocaleStaging, clearOtherLocaleStagingRegistry } from "../state"; + +jest.mock("../../lib/workflows/process-batches", () => ({ + processBatches: jest.fn(async (ids: number[]) => ({ processedIds: ids, failed: 0 })), +})); +jest.mock("../../lib/mappers/mapping-version-updater", () => ({ + updateMappingsAfterPublish: jest.fn(async () => ({ result: { errors: [] } })), +})); +jest.mock("../../lib/shared/get-fetch-api-status", () => ({ + waitForFetchApiSync: jest.fn(async () => undefined), +})); +jest.mock("../pull", () => ({ + Pull: jest.fn().mockImplementation(() => ({ pullInstances: jest.fn(async () => ({ success: true })) })), +})); + +import { processBatches } from "../../lib/workflows/process-batches"; +import { updateMappingsAfterPublish } from "../../lib/mappers/mapping-version-updater"; + +const mockProcessBatches = processBatches as jest.Mock; +const mockUpdateMappings = updateMappingsAfterPublish as jest.Mock; + +beforeEach(() => { + resetState(); + clearOtherLocaleStagingRegistry(); + setState({ sourceGuid: "src", targetGuid: "tgt" }); + mockProcessBatches.mockClear(); + mockUpdateMappings.mockClear(); + jest.spyOn(console, "log").mockImplementation(() => {}); + jest.spyOn(console, "warn").mockImplementation(() => {}); + jest.spyOn(console, "error").mockImplementation(() => {}); +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +/** The run saved page 98 in pt-br only; the pt-br save put page 98 in staging in en-us. */ +function results(): any[] { + return [ + { + publishableContentIdsByLocale: new Map(), + publishablePageIdsByLocale: new Map([["pt-br", [98, 40]]]), + }, + ]; +} + +function pageCalls(): Array<[number[], string]> { + return mockProcessBatches.mock.calls + .filter((c: any[]) => c[1] === "pages") + .map((c: any[]) => [c[0], c[2]] as [number[], string]); +} + +describe("executeAutoPublish — pages another locale's save put in staging (PROD-2782)", () => { + it("republishes the page in the locale it went to staging in, and updates its mapping", async () => { + registerOtherLocaleStaging("en-us", { targetPageID: 98, pageName: "how-it-works", savedLocale: "pt-br" }); + + const errors = await (new Push() as any).executeAutoPublish(results(), "both"); + + expect(errors).toEqual([]); + expect(pageCalls()).toEqual( + expect.arrayContaining([ + [[98, 40], "pt-br"], + [[98], "en-us"], + ]) + ); + expect(mockUpdateMappings).toHaveBeenCalledWith([], [98], "src", "tgt", "en-us"); + }); + + it("publishes a page once when the run also saved it in that locale", async () => { + registerOtherLocaleStaging("pt-br", { targetPageID: 40, pageName: "promotions", savedLocale: "en-us" }); + + await (new Push() as any).executeAutoPublish(results(), "pages"); + + expect(pageCalls()).toEqual([[[98, 40], "pt-br"]]); + }); + + it("does not republish pages when auto-publish is content-only", async () => { + registerOtherLocaleStaging("en-us", { targetPageID: 98, pageName: "how-it-works", savedLocale: "pt-br" }); + + await (new Push() as any).executeAutoPublish(results(), "content"); + + expect(pageCalls()).toEqual([]); + }); +}); diff --git a/src/core/tests/push.test.ts b/src/core/tests/push.test.ts index 8a61459..a4526a6 100644 --- a/src/core/tests/push.test.ts +++ b/src/core/tests/push.test.ts @@ -1,4 +1,11 @@ -import { Push, hasBlockingAutoPublishErrors, formatFailureName } from "../push"; +import { + Push, + hasBlockingAutoPublishErrors, + formatFailureName, + autoPublishIncludesPages, + mergeOtherLocaleStaging, + otherLocaleStagingWarnings, +} from "../push"; import { resetState, setState } from "../state"; import { Pull } from "../pull"; import { Pushers } from "../../lib/pushers/orchestrate-pushers"; @@ -78,6 +85,57 @@ describe("hasBlockingAutoPublishErrors", () => { ]) ).toBe(true); }); + + it("does not treat a page left in staging by another locale's save as blocking (PROD-2782)", () => { + expect(hasBlockingAutoPublishErrors([{ locale: "en-us", type: "staging", error: "left in staging" }])).toBe(false); + }); +}); + +// ─── PROD-2782: pages another locale's save put in staging ────────────────── + +describe("autoPublishIncludesPages (PROD-2782)", () => { + it("is true for pages and both, false otherwise", () => { + expect(autoPublishIncludesPages("pages")).toBe(true); + expect(autoPublishIncludesPages("both")).toBe(true); + expect(autoPublishIncludesPages("content")).toBe(false); + expect(autoPublishIncludesPages("")).toBe(false); + expect(autoPublishIncludesPages(undefined)).toBe(false); + }); +}); + +describe("mergeOtherLocaleStaging (PROD-2782)", () => { + it("adds each locale's pages to that locale's publish list, creating it if needed", () => { + const pageIdsByLocale = new Map([["en-us", [10, 98]]]); + mergeOtherLocaleStaging( + pageIdsByLocale, + new Map([ + ["en-us", [{ targetPageID: 98, pageName: "a", savedLocale: "pt-br" }]], + ["pt-br", [{ targetPageID: 40, pageName: "promotions", savedLocale: "en-us" }]], + ]) + ); + expect(pageIdsByLocale.get("en-us")).toEqual([10, 98, 98]); + expect(pageIdsByLocale.get("pt-br")).toEqual([40]); + }); +}); + +describe("otherLocaleStagingWarnings (PROD-2782)", () => { + it("emits one non-blocking staging notice per page, in the page's locale", () => { + const warnings = otherLocaleStagingWarnings( + new Map([["pt-br", [{ targetPageID: 40, pageName: "promotions", savedLocale: "en-us" }]]]) + ); + expect(warnings).toEqual([ + { + locale: "pt-br", + type: "staging", + error: + 'page "promotions" (target page 40) was put in staging by the en-us save and was not republished (auto-publish does not include pages)', + }, + ]); + }); + + it("is empty when nothing was recorded", () => { + expect(otherLocaleStagingWarnings(new Map())).toEqual([]); + }); }); // ─── --jsonSummary: the run is written as a machine-readable artifact ───────── diff --git a/src/lib/pushers/page-pusher/process-sitemap.ts b/src/lib/pushers/page-pusher/process-sitemap.ts index 10b7ce2..f478bf1 100644 --- a/src/lib/pushers/page-pusher/process-sitemap.ts +++ b/src/lib/pushers/page-pusher/process-sitemap.ts @@ -1,6 +1,6 @@ import * as mgmtApi from "@agility/management-sdk"; import ansiColors from "ansi-colors"; -import { state, getApiClient } from "../../../core/state"; +import { state, getApiClient, blockOtherLocaleStaging } from "../../../core/state"; import { FailureDetail, PusherResult } from "../../../types/sourceData"; import { SitemapHierarchy } from "./sitemap-hierarchy"; import { PageMapper } from "../../mappers/page-mapper"; @@ -201,6 +201,12 @@ export async function processSitemap({ } else { // pageRes.status is "failure" returnData.failed++; + // PROD-2782: a save in another locale may queue this page for republishing in this locale + // (before or after this point in the run). Its own save here failed, so leave it in staging. + const failedMapping = pageMapper.getPageMappingByPageID(sourcePage.pageID, "source"); + if (failedMapping) { + blockOtherLocaleStaging(locale, failedMapping.targetPageID); + } returnData.failureDetails.push({ name: sourcePage.name || `Page ${sourcePage.pageID}`, error: pageRes.error || "Unknown error", diff --git a/src/lib/pushers/page-pusher/refresh-other-locale-versions.ts b/src/lib/pushers/page-pusher/refresh-other-locale-versions.ts index dc3db58..5e97d0a 100644 --- a/src/lib/pushers/page-pusher/refresh-other-locale-versions.ts +++ b/src/lib/pushers/page-pusher/refresh-other-locale-versions.ts @@ -1,5 +1,9 @@ import * as mgmtApi from "@agility/management-sdk"; import { PageMapper } from "../../mappers/page-mapper"; +import { registerOtherLocaleStaging } from "core/state"; + +const STAGING_STATE = 1; +const PUBLISHED_STATE = 2; interface Props { targetPageID: number; @@ -25,6 +29,13 @@ interface Props { * Mapping files are oriented through PageMapper/fileOperations, so under reverse-sync this moves * the original source's recorded version, which is the side reverse-sync wrote to. * + * PROD-2782: a save that changes the page's components (add, remove, reorder) also puts the page in + * staging in every other locale, because the zones are shared and the server cuts a staging + * version for each language. Auto-publish only republishes pages saved in that locale, so the same + * re-read records the page for republishing when it was Published on the target before the run, + * is Staging now, and is Published on the source. A page left in staging on the source stays in + * staging on the target. + * * Returns the locales whose mapping was refreshed. Never throws — a failed re-read only leaves * the old behaviour (a spurious conflict on the next run) for that locale. */ @@ -57,6 +68,19 @@ export async function refreshOtherLocaleVersions({ const currentVersion = currentPage?.properties?.versionID; if (!currentVersion || currentVersion <= record.targetVersionID) continue; + const sourcePage = otherMapper.getMappedEntity(record, "source"); + if ( + pulledTargetPage.properties?.state === PUBLISHED_STATE && + currentPage.properties?.state === STAGING_STATE && + sourcePage?.properties?.state === PUBLISHED_STATE + ) { + registerOtherLocaleStaging(otherLocale, { + targetPageID, + pageName: currentPage.name || pulledTargetPage.name || `Page ${targetPageID}`, + savedLocale, + }); + } + const result = otherMapper.updateTargetVersionID(targetPageID, currentVersion); if (result.success) refreshed.push(otherLocale); } catch { diff --git a/src/lib/pushers/page-pusher/tests/process-sitemap.test.ts b/src/lib/pushers/page-pusher/tests/process-sitemap.test.ts index 674859b..9a9734b 100644 --- a/src/lib/pushers/page-pusher/tests/process-sitemap.test.ts +++ b/src/lib/pushers/page-pusher/tests/process-sitemap.test.ts @@ -1,7 +1,13 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -import { resetState, setState } from "core/state"; +import { + resetState, + setState, + registerOtherLocaleStaging, + getOtherLocaleStaging, + clearOtherLocaleStagingRegistry, +} from "core/state"; import { processSitemap, resetProcessedPageIDs } from "../process-sitemap"; import { SitemapNode } from "types/syncAnalysis"; @@ -26,6 +32,7 @@ afterAll(() => { beforeEach(() => { resetState(); + clearOtherLocaleStagingRegistry(); setState({ rootPath: tmpDir, sourceGuid: "src", targetGuid: "tgt" }); resetProcessedPageIDs(); mockProcessPage.mockClear(); @@ -250,6 +257,28 @@ describe("processSitemap — failed page processing", () => { const result = await processSitemap(makeProps({ sitemapNodes: nodes, sourcePages: pages })); expect(result.failureDetails[0].name).toContain("Page 5"); }); + + it("does not republish a page whose own save failed, even if another locale's save queued it (PROD-2782)", async () => { + // the pt-br save earlier in the run put target page 999 in staging in en-us + registerOtherLocaleStaging("en-us", { targetPageID: 999, pageName: "Page 5", savedLocale: "pt-br" }); + mockProcessPage.mockResolvedValue({ status: "failure", error: "boom" }); + + await processSitemap(makeProps({ sitemapNodes: [makeNode(5)], sourcePages: [makePage(5)] })); + + expect(getOtherLocaleStaging().size).toBe(0); + // and a later save in another locale cannot queue it again + registerOtherLocaleStaging("en-us", { targetPageID: 999, pageName: "Page 5", savedLocale: "fr-ca" }); + expect(getOtherLocaleStaging().size).toBe(0); + }); + + it("keeps a queued page when its own locale skipped it as up to date (PROD-2782)", async () => { + registerOtherLocaleStaging("en-us", { targetPageID: 999, pageName: "Page 5", savedLocale: "pt-br" }); + mockProcessPage.mockResolvedValue({ status: "skip" }); + + await processSitemap(makeProps({ sitemapNodes: [makeNode(5)], sourcePages: [makePage(5)] })); + + expect(getOtherLocaleStaging().get("en-us")).toHaveLength(1); + }); }); // ─── duplicate pageID prevention ────────────────────────────────────────────── diff --git a/src/lib/pushers/page-pusher/tests/refresh-other-locale-versions.test.ts b/src/lib/pushers/page-pusher/tests/refresh-other-locale-versions.test.ts index 110f5db..b9ef93f 100644 --- a/src/lib/pushers/page-pusher/tests/refresh-other-locale-versions.test.ts +++ b/src/lib/pushers/page-pusher/tests/refresh-other-locale-versions.test.ts @@ -1,7 +1,13 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -import { resetState, setState } from "core/state"; +import { + resetState, + setState, + getOtherLocaleStaging, + clearOtherLocaleStagingRegistry, + blockOtherLocaleStaging, +} from "core/state"; import { PageMapper } from "lib/mappers/page-mapper"; import { refreshOtherLocaleVersions } from "../refresh-other-locale-versions"; @@ -17,6 +23,7 @@ afterAll(() => { beforeEach(() => { resetState(); + clearOtherLocaleStagingRegistry(); setState({ rootPath: tmpDir }); jest.spyOn(console, "log").mockImplementation(() => {}); jest.spyOn(console, "warn").mockImplementation(() => {}); @@ -39,15 +46,25 @@ function nextPair() { TGT = `tgt-rl-${testCounter}`; } -function makePage(pageID: number, versionID: number): any { - return { pageID, name: "how-it-works", templateName: "Main", properties: { versionID } }; +const STAGING = 1; +const PUBLISHED = 2; + +function makePage(pageID: number, versionID: number, state?: number): any { + return { pageID, name: "how-it-works", templateName: "Main", properties: { versionID, state } }; } /** Write the pulled target page file for a locale (what the run read at start). */ -function writePulledTargetPage(locale: string, pageID: number, versionID: number) { +function writePulledTargetPage(locale: string, pageID: number, versionID: number, state?: number) { const dir = path.join(tmpDir, TGT, locale, "page"); fs.mkdirSync(dir, { recursive: true }); - fs.writeFileSync(path.join(dir, `${pageID}.json`), JSON.stringify(makePage(pageID, versionID))); + fs.writeFileSync(path.join(dir, `${pageID}.json`), JSON.stringify(makePage(pageID, versionID, state))); +} + +/** Write the pulled source page file for a locale. */ +function writePulledSourcePage(locale: string, pageID: number, versionID: number, state?: number) { + const dir = path.join(tmpDir, SRC, locale, "page"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, `${pageID}.json`), JSON.stringify(makePage(pageID, versionID, state))); } /** Seed a locale's mapping record: source page 176 v{sourceVersion} -> target page 98 v{targetVersion}. */ @@ -59,10 +76,12 @@ function recordFor(locale: string) { return new PageMapper(SRC, TGT, locale).getPageMappingByPageID(98, "target")!; } -function makeApiClient(versionsByLocale: Record): any { +function makeApiClient(versionsByLocale: Record, statesByLocale: Record = {}): any { return { pageMethods: { - getPage: jest.fn(async (pageID: number, _guid: string, locale: string) => makePage(pageID, versionsByLocale[locale])), + getPage: jest.fn(async (pageID: number, _guid: string, locale: string) => + makePage(pageID, versionsByLocale[locale], statesByLocale[locale]) + ), }, }; } @@ -207,3 +226,82 @@ describe("refreshOtherLocaleVersions (PROD-2628)", () => { expect(recordFor("en-us").targetVersionID).toBe(1349); }); }); + +// ─── PROD-2782: other locale left in staging ────────────────────────────────── + +describe("refreshOtherLocaleVersions records pages put in staging in another locale (PROD-2782)", () => { + /** + * en-us was in sync and Published (target page 98 v1349, source page 176 Published). The pt-br + * save just cut en-us version 1362 in state {currentState}. + */ + async function runAfterSave(opts: { + pulledTargetState?: number; + currentState?: number; + sourceState?: number; + pulledTargetVersion?: number; + }) { + nextPair(); + seedMapping("en-us", 4071, 1349); + writePulledTargetPage("en-us", 98, opts.pulledTargetVersion ?? 1349, opts.pulledTargetState ?? PUBLISHED); + writePulledSourcePage("en-us", 176, 4071, opts.sourceState ?? PUBLISHED); + seedMapping("pt-br", 5000, 1362); + const apiClient = makeApiClient({ "en-us": 1362 }, { "en-us": opts.currentState ?? STAGING }); + + const refreshed = await refreshOtherLocaleVersions({ + targetPageID: 98, + savedLocale: "pt-br", + locales: ["en-us", "pt-br"], + sourceGuid: SRC, + targetGuid: TGT, + apiClient, + }); + return { refreshed, apiClient }; + } + + it("records the page when it was Published on the target, is Staging now, and is Published on the source", async () => { + const { refreshed } = await runAfterSave({}); + + expect(refreshed).toEqual(["en-us"]); + expect(getOtherLocaleStaging()).toEqual( + new Map([["en-us", [{ targetPageID: 98, pageName: "how-it-works", savedLocale: "pt-br" }]]]) + ); + // the saved locale itself is never recorded + expect(getOtherLocaleStaging().has("pt-br")).toBe(false); + }); + + it("does not record a page that was already in staging on the target before the run", async () => { + await runAfterSave({ pulledTargetState: STAGING }); + expect(getOtherLocaleStaging().size).toBe(0); + }); + + it("does not record a page the save left Published (a field-only save)", async () => { + await runAfterSave({ currentState: PUBLISHED }); + expect(getOtherLocaleStaging().size).toBe(0); + }); + + it("does not record a page that is in staging on the source - the target mirrors it", async () => { + await runAfterSave({ sourceState: STAGING }); + expect(getOtherLocaleStaging().size).toBe(0); + }); + + it("does not record a page with an independent target edit (it still conflicts)", async () => { + // pulled en-us page is ahead of the mapping's 1349 + const { apiClient } = await runAfterSave({ pulledTargetVersion: 1355 }); + expect(apiClient.pageMethods.getPage).not.toHaveBeenCalled(); + expect(getOtherLocaleStaging().size).toBe(0); + }); + + it("does not record a page whose own save in that locale failed earlier in the run", async () => { + blockOtherLocaleStaging("en-us", 98); + await runAfterSave({}); + expect(getOtherLocaleStaging().size).toBe(0); + }); + + it("drops a recorded page when its own save in that locale fails later in the run", async () => { + await runAfterSave({}); + expect(getOtherLocaleStaging().get("en-us")).toHaveLength(1); + + blockOtherLocaleStaging("en-us", 98); + expect(getOtherLocaleStaging().size).toBe(0); + }); +});