From 3efc30fba8b81e4057301f33dd1839cf99704239 Mon Sep 17 00:00:00 2001 From: Essential Randomness Date: Mon, 10 Aug 2026 03:34:19 -0700 Subject: [PATCH] finally give atproto-loader caches the love they deserve --- .changeset/sleepy-loaders-cache.md | 19 + .gitignore | 3 +- astro-atproto-loader/README.md | 63 +- .../public-hydrated-record-cache.test.ts | 288 ++++ .../cache/source-cache-retry.test.ts | 178 ++ .../__tests__/cache/ttl-cache.test.ts | 235 +++ astro-atproto-loader/__tests__/index.test.ts | 1531 ----------------- .../live-loader/cache-and-errors.test.ts | 283 +++ .../__tests__/live-loader/collection.test.ts | 402 +++++ .../__tests__/live-loader/entry.test.ts | 212 +++ .../__tests__/live-loader/pagination.test.ts | 314 ++++ .../live-loader/per-source-swr.test.ts | 400 +++++ .../__tests__/msw/handlers.ts | 67 +- astro-atproto-loader/__tests__/msw/install.ts | 58 + .../__tests__/msw/track-requests.ts | 41 + astro-atproto-loader/__tests__/setup.ts | 22 +- .../__tests__/static-loader.test.ts | 351 ++++ astro-atproto-loader/package.json | 2 +- astro-atproto-loader/src/cache/index.ts | 105 ++ .../src/cache/source-caches.ts | 160 ++ astro-atproto-loader/src/cache/swr.ts | 93 + astro-atproto-loader/src/cache/ttl.ts | 143 ++ astro-atproto-loader/src/client/identity.ts | 58 +- astro-atproto-loader/src/client/records.ts | 10 +- astro-atproto-loader/src/index.ts | 1 + astro-atproto-loader/src/loaders/live.ts | 294 +--- astro-atproto-loader/src/loaders/static.ts | 59 +- .../src/pipeline/fetch-record.ts | 103 +- astro-atproto-loader/src/pipeline/join.ts | 118 ++ astro-atproto-loader/src/pipeline/run.ts | 119 +- astro-atproto-loader/src/pipeline/single.ts | 81 +- astro-atproto-loader/src/pipeline/source.ts | 13 +- astro-atproto-loader/src/types.ts | 72 +- astro-atproto-loader/src/utils.ts | 51 +- package-lock.json | 2 +- 35 files changed, 3864 insertions(+), 2087 deletions(-) create mode 100644 .changeset/sleepy-loaders-cache.md create mode 100644 astro-atproto-loader/__tests__/cache/public-hydrated-record-cache.test.ts create mode 100644 astro-atproto-loader/__tests__/cache/source-cache-retry.test.ts create mode 100644 astro-atproto-loader/__tests__/cache/ttl-cache.test.ts delete mode 100644 astro-atproto-loader/__tests__/index.test.ts create mode 100644 astro-atproto-loader/__tests__/live-loader/cache-and-errors.test.ts create mode 100644 astro-atproto-loader/__tests__/live-loader/collection.test.ts create mode 100644 astro-atproto-loader/__tests__/live-loader/entry.test.ts create mode 100644 astro-atproto-loader/__tests__/live-loader/pagination.test.ts create mode 100644 astro-atproto-loader/__tests__/live-loader/per-source-swr.test.ts create mode 100644 astro-atproto-loader/__tests__/msw/install.ts create mode 100644 astro-atproto-loader/__tests__/msw/track-requests.ts create mode 100644 astro-atproto-loader/__tests__/static-loader.test.ts create mode 100644 astro-atproto-loader/src/cache/index.ts create mode 100644 astro-atproto-loader/src/cache/source-caches.ts create mode 100644 astro-atproto-loader/src/cache/swr.ts create mode 100644 astro-atproto-loader/src/cache/ttl.ts create mode 100644 astro-atproto-loader/src/pipeline/join.ts diff --git a/.changeset/sleepy-loaders-cache.md b/.changeset/sleepy-loaders-cache.md new file mode 100644 index 0000000..71702de --- /dev/null +++ b/.changeset/sleepy-loaders-cache.md @@ -0,0 +1,19 @@ +--- +"@fujocoded/astro-atproto-loader": patch +--- + +Overhaul caching and error handling in the live loader: + +- **Fix:** the live loader cached the whole collection as one snapshot, so when + a single source failed during a refresh its records silently vanished. Each + source now keeps its own stale-while-revalidate cache: a failed refresh serves + that source's last good records (and logs the failure) instead of dropping + them. +- Cold-start failures (a source erroring before it has anything cached) are no + longer swallowed silently: the new `onInitialLoadError` option defaults to + `"empty"` (preserves existing behavior) but can be set to `"throw"` to surface + them as loader errors. This will change in a future release, so it should be + set to "empty" explicitly if the behavior should be preserved longer term. +- `fetchRecord` hydration now shares a process-wide cache across loader + instances, with a fixed policy: 5 minute TTL, 5 second retry for transient + failures, 20,000 record cap. diff --git a/.gitignore b/.gitignore index bef610f..76595f8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules/ dist/ test-results/ .vscode/ -.astro \ No newline at end of file +.astro +.turbo/ \ No newline at end of file diff --git a/astro-atproto-loader/README.md b/astro-atproto-loader/README.md index 17b6eb7..ae291c6 100644 --- a/astro-atproto-loader/README.md +++ b/astro-atproto-loader/README.md @@ -280,8 +280,21 @@ defineAtProtoLiveCollection({ Every `filter` and `transform` callback receives `fetchRecord({ atUri, parse? })`, which fetches a single record from any public PDS by its `AtUri` (the `at://...` address that uniquely identifies a record on the network). When more -than one callback asks for the _same_ URI in the same cycle (for example a -`subject` URI shared across many records), they share a single network call. +than one callback asks for the _same_ URI (for example a `subject` URI shared +across many records or loader instances), they share the default process-wide +cache and any concurrent request shares a single network call. + +> [!NOTE] +> +> Hydrated records have a fixed **cache policy:** +> +> - Successful records remain cached for a five minutes +> - Transient failures (network errors, 5xx) can retry after five seconds +> - Definitive failures (the record doesn't exist, or its value can never parse) +> are held the full five minutes like successes (since retrying can't change +> the answer in the short term) +> - The cache retains at most 20,000 records: since records are small this isn't +> a "huge" cache, but the limit still keeps it from growing forever A successful call resolves to `{ value, repo }`. `value` is the record body (or whatever your `parse` callback returned). `repo` is the fetched record's @@ -379,7 +392,8 @@ The defaults are picked so each loader behaves sensibly out of the box: - **Live loader:** `sources: [...]` defaults to `"skip"` so one flaky PDS doesn't take down your whole live collection. `source: {...}` defaults to - `"throw"`, because there's no alternate source to fall back to + `"throw"`, since skipping the only source you have wouldn't leave much of a + collection. The error handling on cold start falls back to [`onInitialLoadError`](#live-only-options). - **Static loader:** defaults to `"throw"` everywhere, so a broken source fails the build instead of quietly publishing partial content. Pass `onSourceError: "skip"` if you'd rather ship the rest of the data anyway @@ -391,14 +405,18 @@ onSourceError: (error, source) => source.repo === "critical.test" ? "throw" : "skip", ``` -> [!NOTE] +> [!IMPORTANT] +> +> **In the live loader, source errors only surface while a source is cold.** +> Each source (including a single `source: {...}`) keeps a +> stale-while-revalidate cache of its records: once it has fetched +> successfully at least once, a failed refresh serves its last good records +> instead of erroring. The failure is still logged. This also applies to +> `"skip"`: when _every_ source fails, cold sources throw an `AggregateError` +> so the failure isn't swallowed silently, but warm ones serve stale data. > -> When `onSourceError` is `"throw"`, the first source error fails the whole -> load right away. When you're skipping errors and _every_ source ends up -> failing, the pipeline throws an `AggregateError` so the failure isn't -> swallowed silently. In a live loader, the cache holds onto its last good -> snapshot when a refresh throws, so a transient outage won't blank out -> your page. +> The static loader has no record cache, so `"throw"` always fails the build, +> and `"skip"` with all sources failing always raises the `AggregateError`. ## `groupBy`: merging records from multiple sources @@ -475,8 +493,29 @@ yourself. - `queryFilter`, optional. A request-time filter for `getLiveCollection("collection", filter)`. It receives `{ entry, filter }`. (This was `loadCollectionFilter` in v0.1) -- `cacheTtl`, optional. Cache lifetime in milliseconds. Defaults to `300000` - (5 minutes) +- `onInitialLoadError`, optional. What to do if a cold source read fails before + that source has any successfully cached records. The read includes `filter`. + Defaults to `"empty"`, which treats that failure as an empty collection / + missing entry. Pass `"throw"` to surface it to Astro as a loader error + instead. `groupBy` and `transform` run after the source read and their + failures always return loader errors; this option does not suppress them. + + > [!NOTE] + > + > With `"empty"`, a page can't currently tell a genuinely empty + > collection from a failed cold fetch. If that distinction matters to you + > right now, use `"throw"` and handle the error. + + > [!WARNING] + > + > The default is `"empty"` because that matches the behavior of earlier + > versions, but it may become `"throw"` in a future release. If you're relying + > on `"empty"`, set it explicitly. + +- `cacheTtl`, optional. How long each source's cached records stay fresh before a + background refresh begins. Defaults to `300000` (5 minutes). This does not + control hydrated records fetched through `fetchRecord`; those use the + independent fixed policy described above. # Support Us diff --git a/astro-atproto-loader/__tests__/cache/public-hydrated-record-cache.test.ts b/astro-atproto-loader/__tests__/cache/public-hydrated-record-cache.test.ts new file mode 100644 index 0000000..4061dd4 --- /dev/null +++ b/astro-atproto-loader/__tests__/cache/public-hydrated-record-cache.test.ts @@ -0,0 +1,288 @@ +import { FAKE_CID, useMockAtprotoRepo } from "@fujocoded/msw-atproto"; +import { z } from "astro/zod"; +import { http, HttpResponse } from "msw"; +import { afterEach, expect, test, vi } from "vitest"; + +import { server } from "../msw/server.ts"; +import { trackXrpcRequests } from "../msw/track-requests.ts"; +import { + createAtProtoCache, + HYDRATED_RECORD_CACHE_TTL, + HYDRATED_RECORD_NOT_FOUND_TTL, + HYDRATED_RECORD_RETRY_TTL, + type AtProtoCache, +} from "../../src/cache/index.ts"; +import { defineAtProtoLiveCollection } from "../../src/index.ts"; + +vi.mock("astro/content/config", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + defineLiveCollection: vi.fn((config) => config), + }; +}); + +const DID = "did:plc:public-cache"; +const PDS = "https://public-cache-pds.example.test"; +const SOURCE_COLLECTION = "site.standard.document"; +const HYDRATED_COLLECTION = "app.example.hydrated"; +const HYDRATED_URI = `at://${DID}/${HYDRATED_COLLECTION}/shared`; +const GET_RECORD = "com.atproto.repo.getRecord"; + +const installSource = () => { + return useMockAtprotoRepo(server, { + did: DID, + pds: PDS, + records: { + [SOURCE_COLLECTION]: [ + { rkey: "source", value: { target: HYDRATED_URI } }, + ], + [HYDRATED_COLLECTION]: [{ rkey: "shared", value: { version: 1 } }], + }, + }); +}; + +const createPublicLoader = (cache?: AtProtoCache) => { + return defineAtProtoLiveCollection({ + ...(cache ? { cache } : {}), + source: { repo: DID, collection: SOURCE_COLLECTION }, + outputSchema: z.object({ hydrated: z.unknown() }), + transform: async ({ rkey, fetchRecord }) => ({ + id: rkey, + data: { hydrated: await fetchRecord({ atUri: HYDRATED_URI }) }, + }), + }).loader; +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +test("shares public fetchRecord results across loaders", async () => { + let now = 0; + vi.spyOn(Date, "now").mockImplementation(() => now); + installSource(); + const calls = trackXrpcRequests(server); + + const firstLoader = createPublicLoader(); + const secondLoader = createPublicLoader(); + + const first = await firstLoader.loadCollection({}); + now = HYDRATED_RECORD_CACHE_TTL - 1; + const shared = await secondLoader.loadCollection({}); + + expect(first).toMatchObject({ + entries: [{ data: { hydrated: { value: { version: 1 } } } }], + }); + expect(shared).toMatchObject({ + entries: [{ data: { hydrated: { value: { version: 1 } } } }], + }); + expect(calls.count(PDS, GET_RECORD)).toBe(1); +}); + +test("expires hydrated records after five minutes", async () => { + const cache = createAtProtoCache(); + let now = 0; + vi.spyOn(Date, "now").mockImplementation(() => now); + const repo = installSource(); + const calls = trackXrpcRequests(server); + const loader = createPublicLoader(cache); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: { value: { version: 1 } } } }], + }); + + repo.seed(HYDRATED_COLLECTION, [{ rkey: "shared", value: { version: 2 } }]); + now = HYDRATED_RECORD_CACHE_TTL - 1; + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: { value: { version: 1 } } } }], + }); + expect(calls.count(PDS, GET_RECORD)).toBe(1); + + now = HYDRATED_RECORD_CACHE_TTL; + const expired = await loader.loadCollection({}); + + expect(expired).toMatchObject({ + entries: [{ data: { hydrated: { value: { version: 2 } } } }], + }); + expect(calls.count(PDS, GET_RECORD)).toBe(2); +}); + +test("keeps explicitly separate loader caches isolated", async () => { + let now = 0; + vi.spyOn(Date, "now").mockImplementation(() => now); + const repo = installSource(); + + const firstLoader = createPublicLoader(createAtProtoCache()); + const secondLoader = createPublicLoader(createAtProtoCache()); + + await expect(firstLoader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: { value: { version: 1 } } } }], + }); + + repo.seed(HYDRATED_COLLECTION, [{ rkey: "shared", value: { version: 2 } }]); + now = 1; + + await expect(secondLoader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: { value: { version: 2 } } } }], + }); +}); + +test("retries public fetchRecord failures after the fixed five-second floor", async () => { + const cache = createAtProtoCache(); + let now = 0; + vi.spyOn(Date, "now").mockImplementation(() => now); + vi.spyOn(console, "warn").mockImplementation(() => {}); + const repo = installSource(); + repo.seed(HYDRATED_COLLECTION, [ + { rkey: "shared", value: { recovered: true } }, + ]); + repo.failOnce.getRecord({ + collection: HYDRATED_COLLECTION, + rkey: "shared", + status: 503, + }); + + const firstLoader = createPublicLoader(cache); + const secondLoader = createPublicLoader(cache); + + await expect(firstLoader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: null } }], + }); + now = HYDRATED_RECORD_RETRY_TTL - 1; + await expect(secondLoader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: null } }], + }); + + now = HYDRATED_RECORD_RETRY_TTL; + await expect(secondLoader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: { value: { recovered: true } } } }], + }); +}); + +test("holds record-not-found failures for the full five-minute TTL", async () => { + const cache = createAtProtoCache(); + let now = 0; + vi.spyOn(Date, "now").mockImplementation(() => now); + vi.spyOn(console, "warn").mockImplementation(() => {}); + installSource(); + + let getRecordCalls = 0; + server.use( + http.get(`${PDS}/xrpc/com.atproto.repo.getRecord`, () => { + getRecordCalls += 1; + return HttpResponse.json( + { error: "RecordNotFound", message: "Could not locate record" }, + { status: 400 }, + ); + }), + ); + + const loader = createPublicLoader(cache); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: null } }], + }); + expect(getRecordCalls).toBe(1); + + // Past the transient retry floor: a missing record must stay cached. + now = HYDRATED_RECORD_RETRY_TTL; + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: null } }], + }); + expect(getRecordCalls).toBe(1); + + now = HYDRATED_RECORD_NOT_FOUND_TTL; + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: null } }], + }); + expect(getRecordCalls).toBe(2); +}); + +test("holds non-object record values for the full five-minute TTL", async () => { + const cache = createAtProtoCache(); + let now = 0; + vi.spyOn(Date, "now").mockImplementation(() => now); + vi.spyOn(console, "warn").mockImplementation(() => {}); + installSource(); + + let getRecordCalls = 0; + server.use( + http.get(`${PDS}/xrpc/com.atproto.repo.getRecord`, () => { + getRecordCalls += 1; + // An array passes the lexicon's `unknown` validation (it's an object to + // `typeof`) but can never be a usable record value. Scalars don't reach + // the loader at all: the XRPC client rejects them as invalid responses, + // which stay transient. + return HttpResponse.json({ + uri: HYDRATED_URI, + cid: FAKE_CID, + value: [], + }); + }), + ); + + const loader = createPublicLoader(cache); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: null } }], + }); + + now = HYDRATED_RECORD_RETRY_TTL; + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ data: { hydrated: null } }], + }); + expect(getRecordCalls).toBe(1); +}); + +test("deduplicates concurrent same-URI hydration through public loaders", async () => { + const cache = createAtProtoCache(); + let getRecordCalls = 0; + let releaseHydration: (() => void) | undefined; + installSource(); + server.use( + http.get(`${PDS}/xrpc/com.atproto.repo.getRecord`, async () => { + getRecordCalls += 1; + await new Promise((resolve) => { + releaseHydration = resolve; + }); + return HttpResponse.json({ + uri: HYDRATED_URI, + cid: FAKE_CID, + value: { shared: true }, + }); + }), + ); + + const firstLoader = createPublicLoader(cache); + const secondLoader = createPublicLoader(cache); + const pending = Promise.all([ + firstLoader.loadCollection({}), + secondLoader.loadCollection({}), + ]); + + await vi.waitFor(() => expect(getRecordCalls).toBe(1)); + releaseHydration?.(); + + await expect(pending).resolves.toEqual([ + expect.objectContaining({ + entries: [ + expect.objectContaining({ + data: { + hydrated: expect.objectContaining({ value: { shared: true } }), + }, + }), + ], + }), + expect.objectContaining({ + entries: [ + expect.objectContaining({ + data: { + hydrated: expect.objectContaining({ value: { shared: true } }), + }, + }), + ], + }), + ]); + expect(getRecordCalls).toBe(1); +}); diff --git a/astro-atproto-loader/__tests__/cache/source-cache-retry.test.ts b/astro-atproto-loader/__tests__/cache/source-cache-retry.test.ts new file mode 100644 index 0000000..f0fa561 --- /dev/null +++ b/astro-atproto-loader/__tests__/cache/source-cache-retry.test.ts @@ -0,0 +1,178 @@ +import { useMockAtprotoRepo } from "@fujocoded/msw-atproto"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + createSourceCaches, + SOURCE_RETRY_TTL_MS, +} from "../../src/cache/source-caches.ts"; +import { + createAtProtoCache, + type AtProtoCache, +} from "../../src/cache/index.ts"; +import { createFetchRecord } from "../../src/pipeline/fetch-record.ts"; +import { server } from "../msw/server.ts"; +import { trackXrpcRequests } from "../msw/track-requests.ts"; + +const COLLECTION = "site.standard.document"; +const LIST_RECORDS = "com.atproto.repo.listRecords"; +const DID = "did:plc:cacherepo"; +const PDS = "https://cache-pds.example.test"; + +const UNAVAILABLE = { + status: 503, + error: "TemporarilyUnavailable", + message: "TemporarilyUnavailable", +}; + +const installRepo = () => + useMockAtprotoRepo(server, { + did: DID, + pds: PDS, + records: { [COLLECTION]: [{ rkey: "doc", value: { title: "warm" } }] }, + }); + +let caches: AtProtoCache; + +const readTitles = async (read: () => Promise<{ value: unknown }[][]>) => { + const sourceRecords = await read(); + return sourceRecords + .flat() + .map((args) => (args.value as { title: string }).title); +}; + +describe("source cache retry behavior", () => { + beforeEach(() => { + caches = createAtProtoCache(); + }); + + afterEach(async () => { + // Settle before restoring mocks so a late background refresh reports to + // this test's silenced console, not the next test's spy. + await caches.whenIdle(); + vi.restoreAllMocks(); + }); + + it("serves stale records while throttling failed refreshes and recovering", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const repo = installRepo(); + const calls = trackXrpcRequests(server); + + let now = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + + const read = createSourceCaches({ + sources: [{ repo: DID, collection: COLLECTION }], + callbacks: {}, + fetchRecord: createFetchRecord(caches), + cacheTtl: 100, + onSourceError: "throw", + onInitialLoadError: "throw", + caches, + }); + + await expect(readTitles(read)).resolves.toEqual(["warm"]); + expect(calls.count(PDS, LIST_RECORDS)).toBe(1); + + // Stale read behind a failing refresh keeps serving the warm records. + repo.failOnce.listRecords(UNAVAILABLE); + repo.seed(COLLECTION, [{ rkey: "doc", value: { title: "recovered" } }]); + now = 1_101; + await expect(readTitles(read)).resolves.toEqual(["warm"]); + await vi.waitFor(() => expect(warn).toHaveBeenCalledTimes(1)); + expect(calls.count(PDS, LIST_RECORDS)).toBe(2); + + // Inside the retry floor no new refresh is attempted. + await expect(readTitles(read)).resolves.toEqual(["warm"]); + now = 1_101 + SOURCE_RETRY_TTL_MS - 1; + await expect(readTitles(read)).resolves.toEqual(["warm"]); + await caches.whenIdle(); + expect(calls.count(PDS, LIST_RECORDS)).toBe(2); + + // Past the floor the refresh retries and the next read sees fresh data. + now = 1_101 + SOURCE_RETRY_TTL_MS; + await expect(readTitles(read)).resolves.toEqual(["warm"]); + await vi.waitFor(async () => { + await expect(readTitles(read)).resolves.toEqual(["recovered"]); + }); + expect(calls.count(PDS, LIST_RECORDS)).toBe(3); + }); + + it("does not apply the retry floor before the first successful load", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + const repo = installRepo(); + const calls = trackXrpcRequests(server); + repo.failOnce.listRecords(UNAVAILABLE); + repo.failOnce.listRecords(UNAVAILABLE); + + const read = createSourceCaches({ + sources: [{ repo: DID, collection: COLLECTION }], + callbacks: {}, + fetchRecord: createFetchRecord(caches), + cacheTtl: 100, + onSourceError: "throw", + onInitialLoadError: "throw", + caches, + }); + + await expect(read()).rejects.toMatchObject({ + message: expect.stringContaining("TemporarilyUnavailable"), + }); + await expect(read()).rejects.toMatchObject({ + message: expect.stringContaining("TemporarilyUnavailable"), + }); + expect(calls.count(PDS, LIST_RECORDS)).toBe(2); + }); + + it("degrades a cold failure to an empty read under the 'empty' policy", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const repo = installRepo(); + repo.failOnce.listRecords(UNAVAILABLE); + + const read = createSourceCaches({ + sources: [{ repo: DID, collection: COLLECTION }], + callbacks: {}, + fetchRecord: createFetchRecord(caches), + cacheTtl: 100, + onSourceError: "throw", + onInitialLoadError: "empty", + caches, + }); + + await expect(read()).resolves.toEqual([]); + expect(error).toHaveBeenCalledWith( + expect.stringContaining("refresh failed:"), + expect.anything(), + ); + }); + + it("resolves whenIdle immediately when nothing is tracked", async () => { + await expect(createAtProtoCache().whenIdle()).resolves.toBeUndefined(); + }); + + it("waits out tracked refreshes, including failed and chained ones", async () => { + const cache = createAtProtoCache(); + let release!: () => void; + const refresh = new Promise((resolve) => { + release = resolve; + }); + cache.onRefresh( + refresh.then(() => { + // A settling refresh may start more work; idle must cover the cascade. + cache.onRefresh(Promise.reject(new Error("chained"))); + }), + ); + + let idle = false; + const wait = cache.whenIdle().then(() => { + idle = true; + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(idle).toBe(false); + + release(); + await wait; + expect(idle).toBe(true); + }); +}); diff --git a/astro-atproto-loader/__tests__/cache/ttl-cache.test.ts b/astro-atproto-loader/__tests__/cache/ttl-cache.test.ts new file mode 100644 index 0000000..e7d954a --- /dev/null +++ b/astro-atproto-loader/__tests__/cache/ttl-cache.test.ts @@ -0,0 +1,235 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { TtlCache } from "../../src/cache/ttl.ts"; + +describe("TtlCache", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(0); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it.each([-1, Number.NaN])("rejects invalid success TTL %s", (successTtl) => { + expect(() => new TtlCache({ successTtl, failureTtl: 100 })).toThrowError( + "successTtl must be a non-negative number", + ); + }); + + it.each([-1, Number.NaN])("rejects invalid failure TTL %s", (failureTtl) => { + expect(() => new TtlCache({ successTtl: 1_000, failureTtl })).toThrowError( + "failureTtl must be a non-negative number", + ); + }); + + it.each([-1, 1.5, Number.MAX_SAFE_INTEGER + 1])( + "rejects invalid max-entry policy %s", + (maxEntries) => { + expect( + () => + new TtlCache({ + successTtl: 1_000, + failureTtl: 100, + maxEntries, + }), + ).toThrowError("maxEntries must be a non-negative safe integer"); + }, + ); + + it("serves a successful entry until its stamped expiry, then refetches", async () => { + const cache = new TtlCache({ + successTtl: 1_000, + failureTtl: 100, + }); + const load = vi + .fn<() => Promise>() + .mockResolvedValueOnce("first") + .mockResolvedValueOnce("second"); + + await expect(cache.get("key", load)).resolves.toBe("first"); + + vi.advanceTimersByTime(999); + await expect(cache.get("key", load)).resolves.toBe("first"); + expect(load).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(1); + await expect(cache.get("key", load)).resolves.toBe("second"); + expect(load).toHaveBeenCalledTimes(2); + }); + + it("throttles a failed lookup until the failure retry floor elapses", async () => { + const cache = new TtlCache({ + successTtl: 1_000, + failureTtl: 100, + }); + const failure = new Error("temporarily unavailable"); + const load = vi + .fn<() => Promise>() + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce("recovered"); + + await expect(cache.get("key", load)).rejects.toBe(failure); + + vi.advanceTimersByTime(99); + await expect(cache.get("key", load)).rejects.toBe(failure); + expect(load).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(1); + await expect(cache.get("key", load)).resolves.toBe("recovered"); + expect(load).toHaveBeenCalledTimes(2); + }); + + it("lets the failure TTL vary by rejection error", async () => { + const cache = new TtlCache({ + successTtl: 1_000, + failureTtl: (error) => (error instanceof RangeError ? 500 : 100), + }); + const definitive = new RangeError("permanently gone"); + const transient = new Error("temporarily unavailable"); + const load = vi + .fn<() => Promise>() + .mockRejectedValueOnce(definitive) + .mockRejectedValueOnce(transient) + .mockResolvedValue("recovered"); + + await expect(cache.get("definitive", load)).rejects.toBe(definitive); + await expect(cache.get("transient", load)).rejects.toBe(transient); + + vi.advanceTimersByTime(100); + await expect(cache.get("definitive", load)).rejects.toBe(definitive); + await expect(cache.get("transient", load)).resolves.toBe("recovered"); + expect(load).toHaveBeenCalledTimes(3); + + vi.advanceTimersByTime(400); + await expect(cache.get("definitive", load)).resolves.toBe("recovered"); + expect(load).toHaveBeenCalledTimes(4); + }); + + it("uses one construction-owned policy for every key", async () => { + const cache = new TtlCache({ + successTtl: 100, + failureTtl: 10, + }); + const load = vi.fn(async (key: string) => key); + + await cache.get("first", () => load("first")); + await cache.get("second", () => load("second")); + + vi.advanceTimersByTime(99); + await cache.get("first", () => load("first")); + await cache.get("second", () => load("second")); + expect(load).toHaveBeenCalledTimes(2); + + vi.advanceTimersByTime(1); + await cache.get("first", () => load("first")); + await cache.get("second", () => load("second")); + expect(load).toHaveBeenCalledTimes(4); + }); + + it("deduplicates concurrent reads for the same key", async () => { + const cache = new TtlCache({ + successTtl: 1_000, + failureTtl: 100, + }); + let resolveLoad: ((value: string) => void) | undefined; + const load = vi.fn( + () => + new Promise((resolve) => { + resolveLoad = resolve; + }), + ); + + const first = cache.get("key", load); + const second = cache.get("key", load); + await Promise.resolve(); + + expect(load).toHaveBeenCalledTimes(1); + resolveLoad?.("value"); + await expect(Promise.all([first, second])).resolves.toEqual([ + "value", + "value", + ]); + }); + + it("does not let an in-flight pre-reset generation corrupt its replacement", async () => { + const cache = new TtlCache({ + successTtl: 1_000, + failureTtl: 100, + }); + let resolveOld: ((value: string) => void) | undefined; + const oldLoad = vi.fn( + () => + new Promise((resolve) => { + resolveOld = resolve; + }), + ); + const replacementLoad = vi.fn(async () => "replacement"); + const unexpectedReload = vi.fn(async () => "unexpected"); + + const oldRequest = cache.get("key", oldLoad); + await Promise.resolve(); + expect(oldLoad).toHaveBeenCalledTimes(1); + + cache.reset(); + await expect(cache.get("key", replacementLoad)).resolves.toBe( + "replacement", + ); + + resolveOld?.("old"); + await expect(oldRequest).resolves.toBe("old"); + await expect(cache.get("key", unexpectedReload)).resolves.toBe( + "replacement", + ); + + expect(replacementLoad).toHaveBeenCalledTimes(1); + expect(unexpectedReload).not.toHaveBeenCalled(); + }); +}); + +describe("TtlCache LRU bound", () => { + it("never retains more entries than its configured ceiling", async () => { + const cache = new TtlCache({ + successTtl: 1_000, + failureTtl: 100, + maxEntries: 2, + }); + const load = vi.fn(async (key: string) => key); + const get = (key: string) => cache.get(key, () => load(key)); + + await get("first"); + await get("second"); + await get("third"); + await get("second"); + await get("third"); + await get("first"); + + expect(load).toHaveBeenCalledTimes(4); + }); + + it("refreshes recency on a cache hit and evicts the least recently read entry", async () => { + const cache = new TtlCache({ + successTtl: 1_000, + failureTtl: 100, + maxEntries: 2, + }); + const loads = new Map(); + const get = (key: string) => + cache.get(key, async () => { + loads.set(key, (loads.get(key) ?? 0) + 1); + return key; + }); + + await get("first"); + await get("second"); + await get("first"); + await get("third"); + + await get("first"); + await get("second"); + + expect(loads.get("first")).toBe(1); + expect(loads.get("second")).toBe(2); + expect(loads.get("third")).toBe(1); + }); +}); diff --git a/astro-atproto-loader/__tests__/index.test.ts b/astro-atproto-loader/__tests__/index.test.ts deleted file mode 100644 index 7bafcf7..0000000 --- a/astro-atproto-loader/__tests__/index.test.ts +++ /dev/null @@ -1,1531 +0,0 @@ -import { http, HttpResponse } from "msw"; -import { beforeEach, describe, expect, test, vi } from "vitest"; - -import { server } from "./msw/server.ts"; -import { - FAKE_CID, - failingGetRecord, - mockGetRecord, - mockListRecords, - mockRepoIdentity, - type FakeRecord, -} from "./msw/handlers.ts"; - -const PDS = "https://pds.example.test"; - -const importLoader = async () => { - vi.resetModules(); - const live = await import("../src/loaders/live.ts"); - const staticLoader = await import("../src/loaders/static.ts"); - return { - atProtoLiveLoader: live.atProtoLiveLoader, - atProtoStaticLoader: staticLoader.atProtoStaticLoader, - }; -}; - -beforeEach(() => { - vi.restoreAllMocks(); -}); - -describe("atProtoLiveLoader", () => { - test("loads a collection, applies the object callback signature, and resolves handles", async () => { - server.use( - ...mockRepoIdentity({ - did: "did:plc:resolved-handle", - pds: PDS, - handle: "events.example.com", - }), - mockListRecords({ - pds: PDS, - repo: "events.example.com", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { - did: "did:plc:resolved-handle", - rkey: "first", - value: { title: "Opening", published: true }, - }, - { - did: "did:plc:resolved-handle", - rkey: "second", - value: { title: "Draft", published: false }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const filterSpy = vi.fn( - ({ value }: { value: Record }) => - value.published === true, - ); - const transformSpy = vi.fn( - ({ - value, - rkey, - repo, - }: { - value: Record; - rkey: string; - repo: { did: string; handle?: string }; - }) => ({ - id: rkey, - data: { - did: repo.did, - title: String(value.title), - }, - }), - ); - - const loader = atProtoLiveLoader({ - source: { - repo: "events.example.com", - collection: "community.lexicon.calendar.event", - }, - filter: filterSpy, - transform: transformSpy, - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { - id: "first", - data: { - did: "did:plc:resolved-handle", - title: "Opening", - }, - }, - ]); - expect(filterSpy).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - value: { title: "Opening", published: true }, - repo: { - did: "did:plc:resolved-handle", - handle: "events.example.com", - pds: PDS, - }, - collection: "community.lexicon.calendar.event", - rkey: "first", - }), - ); - expect(transformSpy).toHaveBeenCalledTimes(1); - }); - - test("sends the listRecords XRPC query with the configured limit and no initial cursor", async () => { - const cursorCalls: Array = []; - - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { - did: "did:plc:testrepo", - rkey: "only", - value: { title: "Only" }, - }, - ], - ], - onCall: (cursor) => cursorCalls.push(cursor), - }), - ); - - const requestLog: Array<{ limit: string | null; cursor: string | null }> = - []; - server.events.on("request:start", ({ request }) => { - const url = new URL(request.url); - if (url.pathname === "/xrpc/com.atproto.repo.listRecords") { - requestLog.push({ - limit: url.searchParams.get("limit"), - cursor: url.searchParams.get("cursor"), - }); - } - }); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - }, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { title: String(value.title) }, - }), - }); - - await loader.loadCollection({}); - - expect(requestLog).toEqual([{ limit: "100", cursor: null }]); - expect(cursorCalls).toEqual([null]); - }); - - test("stops after `source.limit` entries and skips remaining pages", async () => { - const cursorCalls: Array = []; - - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { did: "did:plc:testrepo", rkey: "one", value: { title: "A" } }, - { did: "did:plc:testrepo", rkey: "two", value: { title: "B" } }, - { did: "did:plc:testrepo", rkey: "three", value: { title: "C" } }, - ], - [{ did: "did:plc:testrepo", rkey: "four", value: { title: "D" } }], - ], - onCall: (cursor) => cursorCalls.push(cursor), - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - limit: 2, - }, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { id: "one", data: { title: "A" } }, - { id: "two", data: { title: "B" } }, - ]); - expect(cursorCalls).toEqual([null]); - }); - - test("caps the XRPC listRecords page size at `source.limit`", async () => { - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [{ did: "did:plc:testrepo", rkey: "only", value: { title: "Only" } }], - ], - }), - ); - - const requestLog: Array<{ limit: string | null }> = []; - server.events.on("request:start", ({ request }) => { - const url = new URL(request.url); - if (url.pathname === "/xrpc/com.atproto.repo.listRecords") { - requestLog.push({ limit: url.searchParams.get("limit") }); - } - }); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - limit: 5, - }, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { title: String(value.title) }, - }), - }); - - await loader.loadCollection({}); - - expect(requestLog).toEqual([{ limit: "5" }]); - }); - - test("counts only post-filter entries against `source.limit`", async () => { - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { - did: "did:plc:testrepo", - rkey: "one", - value: { title: "A", published: false }, - }, - { - did: "did:plc:testrepo", - rkey: "two", - value: { title: "B", published: true }, - }, - { - did: "did:plc:testrepo", - rkey: "three", - value: { title: "C", published: false }, - }, - { - did: "did:plc:testrepo", - rkey: "four", - value: { title: "D", published: true }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - limit: 2, - }, - filter: ({ value }) => value.published === true, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { id: "two", data: { title: "B" } }, - { id: "four", data: { title: "D" } }, - ]); - }); - - test("deduplicates collection entries by id and keeps the newest one", async () => { - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { - did: "did:plc:testrepo", - rkey: "early", - value: { slug: "session-1", title: "First title" }, - }, - { - did: "did:plc:testrepo", - rkey: "later", - value: { slug: "session-1", title: "Updated title" }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - }, - transform: ({ value }) => ({ - id: String(value.slug), - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { - id: "session-1", - data: { title: "Updated title" }, - }, - ]); - }); - - test("supports request-time collection filtering", async () => { - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { - did: "did:plc:testrepo", - rkey: "one", - value: { track: "main", title: "Main stage" }, - }, - { - did: "did:plc:testrepo", - rkey: "two", - value: { track: "hallway", title: "Hallway track" }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader< - readonly [{ repo: string; collection: string }], - { title: string; track: string }, - { track: string } - >({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - }, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { - title: String(value.title), - track: String(value.track), - }, - }), - queryFilter: ({ entry, filter }) => entry.data.track === filter.track, - }); - - const result = await loader.loadCollection({ - filter: { track: "hallway" }, - }); - - expect("entries" in result && result.entries).toEqual([ - { - id: "two", - data: { title: "Hallway track", track: "hallway" }, - }, - ]); - }); - - test("supports a dedicated single-source `source` option", async () => { - server.use( - ...mockRepoIdentity({ - did: "did:plc:source-option", - pds: PDS, - handle: "source.example.com", - }), - mockListRecords({ - pds: PDS, - repo: "source.example.com", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:source-option", - rkey: "doc-1", - value: { title: "From source" }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "source.example.com", - collection: "site.standard.document", - }, - transform: ({ value, rkey, repo, collection }) => ({ - id: `${repo.did}/${rkey}`, - data: { - title: String(value.title), - repo: repo.handle ?? repo.did, - collection, - }, - }), - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { - id: "did:plc:source-option/doc-1", - data: { - title: "From source", - repo: "source.example.com", - collection: "site.standard.document", - }, - }, - ]); - }); - - test("defaults to passthrough entries for a single source when transform is omitted", async () => { - server.use( - ...mockRepoIdentity({ - did: "did:plc:passthrough-live", - pds: PDS, - handle: "passthrough.example.com", - }), - mockListRecords({ - pds: PDS, - repo: "passthrough.example.com", - collection: "place.stream.livestream", - pages: [ - [ - { - did: "did:plc:passthrough-live", - rkey: "stream-1", - value: { - title: "Coworking stream", - createdAt: "2026-04-04T00:30:21Z", - }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader< - readonly [{ repo: string; collection: string }], - { - title: string; - createdAt: string; - } - >({ - source: { - repo: "passthrough.example.com", - collection: "place.stream.livestream", - }, - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { - id: "stream-1", - data: { - title: "Coworking stream", - createdAt: "2026-04-04T00:30:21Z", - }, - }, - ]); - }); - - test("supports multiple sources under one loader", async () => { - const bobatanPds = "https://bobatan-pds.example.test"; - const bobPds = "https://bob-pds.example.test"; - - server.use( - ...mockRepoIdentity({ - did: "did:plc:bobatan", - pds: bobatanPds, - handle: "bobatan.fujocoded.dev", - }), - ...mockRepoIdentity({ - did: "did:plc:bob", - pds: bobPds, - handle: "bob.example.com", - }), - mockListRecords({ - pds: bobatanPds, - repo: "bobatan.fujocoded.dev", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bobatan", - rkey: "alpha", - value: { title: "Bobatan doc" }, - }, - ], - ], - }), - mockListRecords({ - pds: bobPds, - repo: "bob.example.com", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bob", - rkey: "beta", - value: { title: "Bob doc" }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - sources: [ - { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, - { repo: "bob.example.com", collection: "site.standard.document" }, - ], - transform: ({ value, rkey, repo, collection }) => ({ - id: `${repo.did}/${collection}/${rkey}`, - data: { - title: String(value.title), - repo: repo.handle ?? repo.did, - }, - }), - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { - id: "did:plc:bobatan/site.standard.document/alpha", - data: { title: "Bobatan doc", repo: "bobatan.fujocoded.dev" }, - }, - { - id: "did:plc:bob/site.standard.document/beta", - data: { title: "Bob doc", repo: "bob.example.com" }, - }, - ]); - }); - - test("namespaces ids by did/collection when multiple sources omit transform", async () => { - const bobatanPds = "https://bobatan-pds.example.test"; - const bobPds = "https://bob-pds.example.test"; - - server.use( - ...mockRepoIdentity({ - did: "did:plc:bobatan", - pds: bobatanPds, - handle: "bobatan.fujocoded.dev", - }), - ...mockRepoIdentity({ - did: "did:plc:bob", - pds: bobPds, - handle: "bob.example.com", - }), - mockListRecords({ - pds: bobatanPds, - repo: "bobatan.fujocoded.dev", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bobatan", - rkey: "shared", - value: { title: "Bobatan doc" }, - }, - ], - ], - }), - mockListRecords({ - pds: bobPds, - repo: "bob.example.com", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bob", - rkey: "shared", - value: { title: "Bob doc" }, - }, - ], - ], - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - sources: [ - { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, - { repo: "bob.example.com", collection: "site.standard.document" }, - ], - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { - id: "did:plc:bobatan/site.standard.document/shared", - data: { title: "Bobatan doc" }, - }, - { - id: "did:plc:bob/site.standard.document/shared", - data: { title: "Bob doc" }, - }, - ]); - }); - - test("loads a single entry directly by rkey and supports custom ids", async () => { - const record: FakeRecord = { - did: "did:plc:testrepo", - rkey: "record-123", - value: { slug: "opening-keynote", title: "Opening keynote" }, - }; - - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockGetRecord({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - record, - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - }, - transform: ({ value }) => ({ - id: String(value.slug), - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadEntry({ - filter: { id: "opening-keynote", rkey: "record-123" }, - }); - - expect(result).toEqual({ - id: "opening-keynote", - data: { title: "Opening keynote" }, - }); - }); - - test("defaults single-record lookups to rkey ids when transform is omitted", async () => { - const record: FakeRecord = { - did: "did:plc:passthrough-live", - rkey: "stream-1", - value: { - title: "Coworking stream", - createdAt: "2026-04-04T00:30:21Z", - }, - }; - - server.use( - ...mockRepoIdentity({ did: "did:plc:passthrough-live", pds: PDS }), - mockGetRecord({ - pds: PDS, - repo: "did:plc:passthrough-live", - collection: "place.stream.livestream", - record, - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader< - readonly [{ repo: string; collection: string }], - { - title: string; - createdAt: string; - } - >({ - source: { - repo: "did:plc:passthrough-live", - collection: "place.stream.livestream", - }, - }); - - const result = await loader.loadEntry({ - filter: { id: "stream-1" }, - }); - - expect(result).toEqual({ - id: "stream-1", - data: { - title: "Coworking stream", - createdAt: "2026-04-04T00:30:21Z", - }, - }); - }); - - test("can disambiguate direct single-record loads across multiple sources", async () => { - const bobatanPds = "https://bobatan-pds.example.test"; - const bobPds = "https://bob-pds.example.test"; - - let bobatanGetRecordCalls = 0; - let bobGetRecordCalls = 0; - - server.use( - ...mockRepoIdentity({ - did: "did:plc:bobatan", - pds: bobatanPds, - handle: "bobatan.fujocoded.dev", - }), - ...mockRepoIdentity({ - did: "did:plc:bob", - pds: bobPds, - handle: "bob.example.com", - }), - http.get(`${bobatanPds}/xrpc/com.atproto.repo.getRecord`, () => { - bobatanGetRecordCalls += 1; - return new HttpResponse(JSON.stringify({ error: "UnexpectedCall" }), { - status: 500, - headers: { "content-type": "application/json" }, - }); - }), - http.get(`${bobPds}/xrpc/com.atproto.repo.getRecord`, ({ request }) => { - bobGetRecordCalls += 1; - const url = new URL(request.url); - expect(url.searchParams.get("repo")).toBe("bob.example.com"); - expect(url.searchParams.get("collection")).toBe( - "site.standard.document", - ); - expect(url.searchParams.get("rkey")).toBe("shared-rkey"); - return HttpResponse.json({ - uri: "at://did:plc:bob/site.standard.document/shared-rkey", - cid: FAKE_CID, - value: { slug: "bob/shared-rkey", title: "Bob shared doc" }, - }); - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - sources: [ - { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, - { repo: "bob.example.com", collection: "site.standard.document" }, - ], - transform: ({ value }) => ({ - id: String(value.slug), - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadEntry({ - filter: { - id: "bob/shared-rkey", - rkey: "shared-rkey", - repo: "bob.example.com", - collection: "site.standard.document", - }, - }); - - expect(result).toEqual({ - id: "bob/shared-rkey", - data: { title: "Bob shared doc" }, - }); - expect(bobatanGetRecordCalls).toBe(0); - expect(bobGetRecordCalls).toBe(1); - }); - - test("returns stale cached entries while a background refresh is in flight", async () => { - let callCount = 0; - let resolveSecondFetch: (() => void) | undefined; - - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - http.get(`${PDS}/xrpc/com.atproto.repo.listRecords`, async () => { - callCount += 1; - if (callCount === 1) { - return HttpResponse.json({ - records: [ - { - uri: "at://did:plc:testrepo/community.lexicon.calendar.event/first", - cid: FAKE_CID, - value: { title: "Initial title" }, - }, - ], - }); - } - await new Promise((resolve) => { - resolveSecondFetch = resolve; - }); - return HttpResponse.json({ - records: [ - { - uri: "at://did:plc:testrepo/community.lexicon.calendar.event/first", - cid: FAKE_CID, - value: { title: "Refreshed title" }, - }, - ], - }); - }), - ); - - let now = 1_000; - vi.spyOn(Date, "now").mockImplementation(() => now); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - }, - cacheTtl: 1, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { title: String(value.title) }, - }), - }); - - const first = await loader.loadCollection({}); - now = 1_005; - const stale = await loader.loadCollection({}); - - expect("entries" in first && first.entries?.[0]?.data.title).toBe( - "Initial title", - ); - expect("entries" in stale && stale.entries?.[0]?.data.title).toBe( - "Initial title", - ); - - await vi.waitFor(() => { - expect(resolveSecondFetch).toBeDefined(); - }); - resolveSecondFetch?.(); - - await vi.waitFor(async () => { - const refreshed = await loader.loadCollection({}); - expect("entries" in refreshed && refreshed.entries?.[0]?.data.title).toBe( - "Refreshed title", - ); - }); - }); - - test("falls back to the cached collection if direct single-record loading fails", async () => { - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { - did: "did:plc:testrepo", - rkey: "record-123", - value: { slug: "opening-keynote", title: "Opening keynote" }, - }, - ], - ], - }), - failingGetRecord(PDS), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - }, - transform: ({ value }) => ({ - id: String(value.slug), - data: { title: String(value.title) }, - }), - }); - - await loader.loadCollection({}); - const result = await loader.loadEntry({ - filter: { id: "opening-keynote", rkey: "record-123" }, - }); - - expect(result).toEqual({ - id: "opening-keynote", - data: { title: "Opening keynote" }, - }); - }); - - test("follows the cursor across multiple pages of listRecords", async () => { - const observedCursors: Array = []; - - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - mockListRecords({ - pds: PDS, - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - pages: [ - [ - { - did: "did:plc:testrepo", - rkey: "one", - value: { title: "Page one entry" }, - }, - { - did: "did:plc:testrepo", - rkey: "two", - value: { title: "Page one entry two" }, - }, - ], - [ - { - did: "did:plc:testrepo", - rkey: "three", - value: { title: "Page two entry" }, - }, - ], - ], - onCall: (cursor) => observedCursors.push(cursor), - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - limit: "all", - }, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { id: "one", data: { title: "Page one entry" } }, - { id: "two", data: { title: "Page one entry two" } }, - { id: "three", data: { title: "Page two entry" } }, - ]); - expect(observedCursors).toEqual([null, "1"]); - }); - - test("terminates the cursor loop once the PDS omits a next cursor", async () => { - let callCount = 0; - - server.use( - ...mockRepoIdentity({ did: "did:plc:testrepo", pds: PDS }), - http.get(`${PDS}/xrpc/com.atproto.repo.listRecords`, () => { - callCount += 1; - return HttpResponse.json({ - records: - callCount === 1 - ? [ - { - uri: "at://did:plc:testrepo/community.lexicon.calendar.event/only", - cid: FAKE_CID, - value: { title: "Only" }, - }, - ] - : [], - // cursor intentionally omitted on every page - }); - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - source: { - repo: "did:plc:testrepo", - collection: "community.lexicon.calendar.event", - }, - transform: ({ value, rkey }) => ({ - id: rkey, - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadCollection({}); - - expect(callCount).toBe(1); - expect("entries" in result && result.entries).toEqual([ - { id: "only", data: { title: "Only" } }, - ]); - }); - - test("paginates each source independently when combined with multi-source", async () => { - const bobatanPds = "https://bobatan-pds.example.test"; - const bobPds = "https://bob-pds.example.test"; - - const bobatanCursors: Array = []; - const bobCursors: Array = []; - - server.use( - ...mockRepoIdentity({ did: "did:plc:bobatan", pds: bobatanPds }), - ...mockRepoIdentity({ did: "did:plc:bob", pds: bobPds }), - mockListRecords({ - pds: bobatanPds, - repo: "did:plc:bobatan", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bobatan", - rkey: "alpha", - value: { title: "Bobatan page one" }, - }, - ], - [ - { - did: "did:plc:bobatan", - rkey: "alpha-two", - value: { title: "Bobatan page two" }, - }, - ], - ], - onCall: (cursor) => bobatanCursors.push(cursor), - }), - mockListRecords({ - pds: bobPds, - repo: "did:plc:bob", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bob", - rkey: "beta", - value: { title: "Bob only page" }, - }, - ], - ], - onCall: (cursor) => bobCursors.push(cursor), - }), - ); - - const { atProtoLiveLoader } = await importLoader(); - - const loader = atProtoLiveLoader({ - sources: [ - { - repo: "did:plc:bobatan", - collection: "site.standard.document", - limit: "all", - }, - { - repo: "did:plc:bob", - collection: "site.standard.document", - limit: "all", - }, - ], - transform: ({ value, repo, rkey }) => ({ - id: `${repo.did}/${rkey}`, - data: { title: String(value.title) }, - }), - }); - - const result = await loader.loadCollection({}); - - expect("entries" in result && result.entries).toEqual([ - { id: "did:plc:bobatan/alpha", data: { title: "Bobatan page one" } }, - { id: "did:plc:bobatan/alpha-two", data: { title: "Bobatan page two" } }, - { id: "did:plc:bob/beta", data: { title: "Bob only page" } }, - ]); - expect(bobatanCursors).toEqual([null, "1"]); - expect(bobCursors).toEqual([null]); - }); -}); - -describe("atProtoStaticLoader", () => { - test("loads a single source into the Astro data store", async () => { - server.use( - ...mockRepoIdentity({ - did: "did:plc:staticrepo", - pds: PDS, - handle: "static.example.com", - }), - mockListRecords({ - pds: PDS, - repo: "static.example.com", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:staticrepo", - rkey: "doc-1", - value: { title: "Static doc", body: "Hello from Astro" }, - }, - ], - ], - }), - ); - - const { atProtoStaticLoader } = await importLoader(); - - const store = { clear: vi.fn(), set: vi.fn() }; - const parseData = vi.fn(async ({ data }) => data); - - const loader = atProtoStaticLoader({ - source: { - repo: "static.example.com", - collection: "site.standard.document", - }, - transform: ({ value, rkey, repo }) => ({ - id: `${repo.did}/${rkey}`, - data: { title: String(value.title), repo: repo.handle ?? repo.did }, - body: String(value.body), - }), - }); - - await loader.load({ - store, - parseData, - } as unknown as Parameters[0]); - - expect(store.clear).toHaveBeenCalledTimes(1); - expect(parseData).toHaveBeenCalledWith({ - id: "did:plc:staticrepo/doc-1", - data: { title: "Static doc", repo: "static.example.com" }, - filePath: undefined, - }); - expect(store.set).toHaveBeenCalledWith({ - id: "did:plc:staticrepo/doc-1", - data: { title: "Static doc", repo: "static.example.com" }, - body: "Hello from Astro", - filePath: undefined, - }); - }); - - test("defaults to passthrough entries for a single source when transform is omitted", async () => { - server.use( - ...mockRepoIdentity({ - did: "did:plc:passthrough-static", - pds: PDS, - handle: "passthrough.example.com", - }), - mockListRecords({ - pds: PDS, - repo: "passthrough.example.com", - collection: "place.stream.livestream", - pages: [ - [ - { - did: "did:plc:passthrough-static", - rkey: "stream-1", - value: { - title: "Coworking stream", - createdAt: "2026-04-04T00:30:21Z", - }, - }, - ], - ], - }), - ); - - const { atProtoStaticLoader } = await importLoader(); - - const store = { clear: vi.fn(), set: vi.fn() }; - const parseData = vi.fn(async ({ data }) => data); - - const loader = atProtoStaticLoader< - readonly [{ repo: string; collection: string }], - { - title: string; - createdAt: string; - } - >({ - source: { - repo: "passthrough.example.com", - collection: "place.stream.livestream", - }, - }); - - await loader.load({ - store, - parseData, - } as unknown as Parameters[0]); - - expect(parseData).toHaveBeenCalledWith({ - id: "stream-1", - data: { - title: "Coworking stream", - createdAt: "2026-04-04T00:30:21Z", - }, - filePath: undefined, - }); - expect(store.set).toHaveBeenCalledWith({ - id: "stream-1", - data: { - title: "Coworking stream", - createdAt: "2026-04-04T00:30:21Z", - }, - body: undefined, - filePath: undefined, - }); - }); - - test("preserves Date values returned by parseData", async () => { - server.use( - ...mockRepoIdentity({ - did: "did:plc:dates-static", - pds: PDS, - handle: "dates.example.com", - }), - mockListRecords({ - pds: PDS, - repo: "dates.example.com", - collection: "place.stream.livestream", - pages: [ - [ - { - did: "did:plc:dates-static", - rkey: "stream-1", - value: { - title: "Morning stream", - createdAt: "2026-04-04T00:30:21Z", - }, - }, - ], - ], - }), - ); - - const { atProtoStaticLoader } = await importLoader(); - - const createdAt = new Date("2026-04-04T00:30:21Z"); - const store = { clear: vi.fn(), set: vi.fn() }; - const parseData = vi.fn(async ({ data }) => ({ - ...data, - createdAt, - })); - - const loader = atProtoStaticLoader< - readonly [{ repo: string; collection: string }], - { - title: string; - createdAt: Date; - } - >({ - source: { - repo: "dates.example.com", - collection: "place.stream.livestream", - }, - }); - - await loader.load({ - store, - parseData, - } as unknown as Parameters[0]); - - expect(store.set).toHaveBeenCalledWith({ - id: "stream-1", - data: { - title: "Morning stream", - createdAt, - }, - body: undefined, - filePath: undefined, - }); - expect(store.set.mock.calls[0]?.[0].data.createdAt).toBeInstanceOf(Date); - }); - - test("surfaces schema parse failures from parseData", async () => { - server.use( - ...mockRepoIdentity({ - did: "did:plc:staticrepo", - pds: PDS, - handle: "static.example.com", - }), - mockListRecords({ - pds: PDS, - repo: "static.example.com", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:staticrepo", - rkey: "doc-1", - value: { title: "Static doc" }, - }, - ], - ], - }), - ); - - const { atProtoStaticLoader } = await importLoader(); - - const store = { clear: vi.fn(), set: vi.fn() }; - const parseError = new Error( - "Schema parse failed for did:plc:staticrepo/doc-1", - ); - const parseData = vi.fn(async () => { - throw parseError; - }); - - const loader = atProtoStaticLoader({ - source: { - repo: "static.example.com", - collection: "site.standard.document", - }, - transform: ({ value, rkey, repo }) => ({ - id: `${repo.did}/${rkey}`, - data: { title: String(value.title), repo: repo.handle ?? repo.did }, - }), - }); - - await expect( - loader.load({ - store, - parseData, - } as unknown as Parameters[0]), - ).rejects.toThrow("Schema parse failed for did:plc:staticrepo/doc-1"); - - expect(store.clear).toHaveBeenCalledTimes(1); - expect(parseData).toHaveBeenCalledTimes(1); - expect(store.set).not.toHaveBeenCalled(); - }); - - test("supports multiple sources and deduplicates by transformed id", async () => { - const bobatanPds = "https://bobatan-pds.example.test"; - const bobPds = "https://bob-pds.example.test"; - - server.use( - ...mockRepoIdentity({ - did: "did:plc:bobatan", - pds: bobatanPds, - handle: "bobatan.fujocoded.dev", - }), - ...mockRepoIdentity({ - did: "did:plc:bob", - pds: bobPds, - handle: "bob.example.com", - }), - mockListRecords({ - pds: bobatanPds, - repo: "bobatan.fujocoded.dev", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bobatan", - rkey: "alpha", - value: { slug: "shared-post", title: "Older title" }, - }, - ], - ], - }), - mockListRecords({ - pds: bobPds, - repo: "bob.example.com", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bob", - rkey: "beta", - value: { slug: "shared-post", title: "Newer title" }, - }, - ], - ], - }), - ); - - const { atProtoStaticLoader } = await importLoader(); - - const store = { clear: vi.fn(), set: vi.fn() }; - const parseData = vi.fn(async ({ data }) => data); - - const loader = atProtoStaticLoader({ - sources: [ - { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, - { repo: "bob.example.com", collection: "site.standard.document" }, - ], - transform: ({ value, repo }) => ({ - id: String(value.slug), - data: { title: String(value.title), repo: repo.handle ?? repo.did }, - }), - }); - - await loader.load({ - store, - parseData, - } as unknown as Parameters[0]); - - expect(store.clear).toHaveBeenCalledTimes(1); - expect(store.set).toHaveBeenCalledTimes(1); - expect(store.set).toHaveBeenCalledWith({ - id: "shared-post", - data: { title: "Newer title", repo: "bob.example.com" }, - body: undefined, - filePath: undefined, - }); - }); - - test("namespaces ids by did/collection when multiple static sources omit transform", async () => { - const bobatanPds = "https://bobatan-pds.example.test"; - const bobPds = "https://bob-pds.example.test"; - - server.use( - ...mockRepoIdentity({ - did: "did:plc:bobatan", - pds: bobatanPds, - handle: "bobatan.fujocoded.dev", - }), - ...mockRepoIdentity({ - did: "did:plc:bob", - pds: bobPds, - handle: "bob.example.com", - }), - mockListRecords({ - pds: bobatanPds, - repo: "bobatan.fujocoded.dev", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bobatan", - rkey: "shared", - value: { title: "Bobatan doc" }, - }, - ], - ], - }), - mockListRecords({ - pds: bobPds, - repo: "bob.example.com", - collection: "site.standard.document", - pages: [ - [ - { - did: "did:plc:bob", - rkey: "shared", - value: { title: "Bob doc" }, - }, - ], - ], - }), - ); - - const { atProtoStaticLoader } = await importLoader(); - - const store = { clear: vi.fn(), set: vi.fn() }; - const parseData = vi.fn(async ({ data }) => data); - - const loader = atProtoStaticLoader({ - sources: [ - { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, - { repo: "bob.example.com", collection: "site.standard.document" }, - ], - }); - - await loader.load({ - store, - parseData, - } as unknown as Parameters[0]); - - expect(store.set).toHaveBeenCalledTimes(2); - expect(store.set).toHaveBeenNthCalledWith(1, { - id: "did:plc:bobatan/site.standard.document/shared", - data: { title: "Bobatan doc" }, - body: undefined, - filePath: undefined, - }); - expect(store.set).toHaveBeenNthCalledWith(2, { - id: "did:plc:bob/site.standard.document/shared", - data: { title: "Bob doc" }, - body: undefined, - filePath: undefined, - }); - }); -}); diff --git a/astro-atproto-loader/__tests__/live-loader/cache-and-errors.test.ts b/astro-atproto-loader/__tests__/live-loader/cache-and-errors.test.ts new file mode 100644 index 0000000..dfd2438 --- /dev/null +++ b/astro-atproto-loader/__tests__/live-loader/cache-and-errors.test.ts @@ -0,0 +1,283 @@ +import { + createMockRepoIdentity, + FAKE_CID, + useMockAtprotoRepo, +} from "@fujocoded/msw-atproto"; +import { http, HttpResponse } from "msw"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +import { createAtProtoCache } from "../../src/cache/index.ts"; +import { atProtoLiveLoader } from "../../src/loaders/live.ts"; +import { server } from "../msw/server.ts"; +import { PDS } from "../msw/install.ts"; + +const TEST_REPO = "did:plc:testrepo"; +const CALENDAR_COLLECTION = "community.lexicon.calendar.event"; + +const installCalendarRepo = () => + useMockAtprotoRepo(server, { + did: TEST_REPO, + pds: PDS, + records: { + [CALENDAR_COLLECTION]: [ + { rkey: "first", value: { title: "Initial title" } }, + ], + }, + }); + +// Unwraps a loadCollection result so a loader error fails the test with the +// actual error instead of an `expected false to equal [...]` diff. +const entriesOf = (result: object) => { + if ("error" in result && result.error instanceof Error) { + throw new Error(`loader returned an error: ${result.error.message}`, { + cause: result.error, + }); + } + return "entries" in result ? result.entries : undefined; +}; + +// Type-safe checker for tests that expect a loader error; the `in` check +// narrows the `{ error } | LiveDataCollection` union, which plain +// `result.error` cannot. +const errorOf = (result: object) => + "error" in result && result.error instanceof Error ? result.error : undefined; + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("atProtoLiveLoader", () => { + test("returns stale cached entries while a background refresh is in flight", async () => { + let callCount = 0; + let resolveSecondFetch: (() => void) | undefined; + + server.use( + ...createMockRepoIdentity({ + did: "did:plc:testrepo", + pds: PDS, + }).handlers(), + http.get(`${PDS}/xrpc/com.atproto.repo.listRecords`, async () => { + callCount += 1; + if (callCount === 1) { + return HttpResponse.json({ + records: [ + { + uri: "at://did:plc:testrepo/community.lexicon.calendar.event/first", + cid: FAKE_CID, + value: { title: "Initial title" }, + }, + ], + }); + } + await new Promise((resolve) => { + resolveSecondFetch = resolve; + }); + return HttpResponse.json({ + records: [ + { + uri: "at://did:plc:testrepo/community.lexicon.calendar.event/first", + cid: FAKE_CID, + value: { title: "Refreshed title" }, + }, + ], + }); + }), + ); + + let now = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + }, + cacheTtl: 1, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const first = await loader.loadCollection({}); + now = 1_005; + const stale = await loader.loadCollection({}); + + expect(entriesOf(first)).toMatchObject([ + { data: { title: "Initial title" } }, + ]); + expect(entriesOf(stale)).toMatchObject([ + { data: { title: "Initial title" } }, + ]); + + await vi.waitFor(() => { + expect(resolveSecondFetch).toBeDefined(); + }); + resolveSecondFetch?.(); + + await vi.waitFor(async () => { + const refreshed = await loader.loadCollection({}); + expect(entriesOf(refreshed)).toMatchObject([ + { data: { title: "Refreshed title" } }, + ]); + }); + }); + + test("returns a loader error when a warm-cache transform fails", async () => { + installCalendarRepo(); + + let shouldThrow = false; + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: TEST_REPO, + collection: CALENDAR_COLLECTION, + }, + transform: ({ value, rkey }) => { + if (shouldThrow) throw new Error("warm transform failed"); + return { id: rkey, data: { title: String(value.title) } }; + }, + }); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ id: "first", data: { title: "Initial title" } }], + }); + shouldThrow = true; + + const result = await loader.loadCollection({}); + + expect(errorOf(result)?.cause).toEqual(new Error("warm transform failed")); + }); + + test("returns a loader error for a cold transform failure under the default empty policy", async () => { + installCalendarRepo(); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: TEST_REPO, + collection: CALENDAR_COLLECTION, + }, + transform: () => { + throw new Error("cold transform failed"); + }, + }); + + const result = await loader.loadCollection({}); + + expect(errorOf(result)?.cause).toEqual(new Error("cold transform failed")); + }); + + test("returns a loader error when groupBy fails under the default empty policy", async () => { + installCalendarRepo(); + + const transform = vi.fn(() => ({ + id: "unused", + data: { title: "unused" }, + })); + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: TEST_REPO, + collection: CALENDAR_COLLECTION, + }, + groupBy: () => { + throw new Error("groupBy failed"); + }, + transform, + }); + + const result = await loader.loadCollection({}); + + expect(errorOf(result)?.cause).toEqual(new Error("groupBy failed")); + expect(transform).not.toHaveBeenCalled(); + }); + + test("surfaces the initial collection fetch error when onInitialLoadError is 'throw'", async () => { + const repo = installCalendarRepo(); + repo.failOnce.listRecords({ + collection: CALENDAR_COLLECTION, + status: 503, + error: "TemporarilyUnavailable", + }); + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: TEST_REPO, + collection: CALENDAR_COLLECTION, + }, + onInitialLoadError: "throw", + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(errorOf(result)?.message).toBe( + "Failed to load the AtProto record from collection community.lexicon.calendar.event", + ); + }); + + test("degrades a cold-start collection failure to an empty collection by default", async () => { + const repo = installCalendarRepo(); + repo.failOnce.listRecords({ + collection: CALENDAR_COLLECTION, + status: 503, + error: "TemporarilyUnavailable", + }); + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: TEST_REPO, + collection: CALENDAR_COLLECTION, + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(result).toMatchObject({ entries: [] }); + }); + + test("degrades a cold-start entry failure to a missing entry by default", async () => { + const repo = installCalendarRepo(); + repo.failOnce.getRecord({ + collection: CALENDAR_COLLECTION, + rkey: "record-123", + status: 503, + error: "TemporarilyUnavailable", + }); + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: TEST_REPO, + collection: CALENDAR_COLLECTION, + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadEntry({ + filter: { id: "opening-keynote", rkey: "record-123" }, + }); + + expect(result).toBeUndefined(); + }); +}); diff --git a/astro-atproto-loader/__tests__/live-loader/collection.test.ts b/astro-atproto-loader/__tests__/live-loader/collection.test.ts new file mode 100644 index 0000000..6508c47 --- /dev/null +++ b/astro-atproto-loader/__tests__/live-loader/collection.test.ts @@ -0,0 +1,402 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +import { createAtProtoCache } from "../../src/cache/index.ts"; +import { atProtoLiveLoader } from "../../src/loaders/live.ts"; +import { installScriptedRepo, PDS } from "../msw/install.ts"; + +// Unwraps a loadCollection result so a loader error fails the test with the +// actual error instead of an `expected false to equal [...]` diff. +const entriesOf = (result: object) => { + if ("error" in result && result.error instanceof Error) { + throw new Error(`loader returned an error: ${result.error.message}`, { + cause: result.error, + }); + } + return "entries" in result ? result.entries : undefined; +}; + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("atProtoLiveLoader", () => { + test("loads a collection, applies the object callback signature, and resolves handles", async () => { + installScriptedRepo({ + did: "did:plc:resolved-handle", + handle: "events.example.com", + collection: "community.lexicon.calendar.event", + pages: [ + [ + { rkey: "first", value: { title: "Opening", published: true } }, + { rkey: "second", value: { title: "Draft", published: false } }, + ], + ], + }); + + const filterSpy = vi.fn( + ({ value }: { value: Record }) => + value.published === true, + ); + const transformSpy = vi.fn( + ({ + value, + rkey, + repo, + }: { + value: Record; + rkey: string; + repo: { did: string; handle?: string }; + }) => ({ + id: rkey, + data: { + did: repo.did, + title: String(value.title), + }, + }), + ); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "events.example.com", + collection: "community.lexicon.calendar.event", + }, + filter: filterSpy, + transform: transformSpy, + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { + id: "first", + data: { + did: "did:plc:resolved-handle", + title: "Opening", + }, + }, + ]); + expect(filterSpy).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + value: { title: "Opening", published: true }, + repo: { + did: "did:plc:resolved-handle", + handle: "events.example.com", + pds: PDS, + }, + collection: "community.lexicon.calendar.event", + rkey: "first", + }), + ); + expect(transformSpy).toHaveBeenCalledTimes(1); + }); + + test("deduplicates collection entries by id and keeps the newest one", async () => { + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [ + [ + { + rkey: "early", + value: { slug: "session-1", title: "First title" }, + }, + { + rkey: "later", + value: { slug: "session-1", title: "Updated title" }, + }, + ], + ], + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + }, + transform: ({ value }) => ({ + id: String(value.slug), + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { + id: "session-1", + data: { title: "Updated title" }, + }, + ]); + }); + + test("supports request-time collection filtering", async () => { + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [ + [ + { rkey: "one", value: { track: "main", title: "Main stage" } }, + { + rkey: "two", + value: { track: "hallway", title: "Hallway track" }, + }, + ], + ], + }); + + const loader = atProtoLiveLoader< + readonly [{ repo: string; collection: string }], + { title: string; track: string }, + { track: string } + >({ + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { + title: String(value.title), + track: String(value.track), + }, + }), + queryFilter: ({ entry, filter }) => entry.data.track === filter.track, + }); + + const result = await loader.loadCollection({ + filter: { track: "hallway" }, + }); + + expect(entriesOf(result)).toEqual([ + { + id: "two", + data: { title: "Hallway track", track: "hallway" }, + }, + ]); + }); + + test("supports a dedicated single-source `source` option", async () => { + installScriptedRepo({ + did: "did:plc:source-option", + handle: "source.example.com", + collection: "site.standard.document", + pages: [[{ rkey: "doc-1", value: { title: "From source" } }]], + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "source.example.com", + collection: "site.standard.document", + }, + transform: ({ value, rkey, repo, collection }) => ({ + id: `${repo.did}/${rkey}`, + data: { + title: String(value.title), + repo: repo.handle ?? repo.did, + collection, + }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { + id: "did:plc:source-option/doc-1", + data: { + title: "From source", + repo: "source.example.com", + collection: "site.standard.document", + }, + }, + ]); + }); + + test("defaults to passthrough entries for a single source when transform is omitted", async () => { + installScriptedRepo({ + did: "did:plc:passthrough-live", + handle: "passthrough.example.com", + collection: "place.stream.livestream", + pages: [ + [ + { + rkey: "stream-1", + value: { + title: "Coworking stream", + createdAt: "2026-04-04T00:30:21Z", + }, + }, + ], + ], + }); + + const loader = atProtoLiveLoader< + readonly [{ repo: string; collection: string }], + { + title: string; + createdAt: string; + } + >({ + source: { + repo: "passthrough.example.com", + collection: "place.stream.livestream", + }, + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { + id: "stream-1", + data: { + title: "Coworking stream", + createdAt: "2026-04-04T00:30:21Z", + }, + }, + ]); + }); + + test("supports multiple sources under one loader", async () => { + const bobatanPds = "https://bobatan-pds.fujocoded.test"; + const bobatanAltPds = "https://bobatan-alt-pds.fujocoded.test"; + + installScriptedRepo({ + did: "did:plc:bobatan", + handle: "bobatan.fujocoded.dev", + pds: bobatanPds, + collection: "site.standard.document", + pages: [[{ rkey: "alpha", value: { title: "Bobatan doc" } }]], + }); + installScriptedRepo({ + did: "did:plc:bobatan-alt", + handle: "bobatan-alt.fujocoded.dev", + pds: bobatanAltPds, + collection: "site.standard.document", + pages: [[{ rkey: "beta", value: { title: "Alt doc" } }]], + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + sources: [ + { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, + { + repo: "bobatan-alt.fujocoded.dev", + collection: "site.standard.document", + }, + ], + transform: ({ value, rkey, repo, collection }) => ({ + id: `${repo.did}/${collection}/${rkey}`, + data: { + title: String(value.title), + repo: repo.handle ?? repo.did, + }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { + id: "did:plc:bobatan/site.standard.document/alpha", + data: { title: "Bobatan doc", repo: "bobatan.fujocoded.dev" }, + }, + { + id: "did:plc:bobatan-alt/site.standard.document/beta", + data: { title: "Alt doc", repo: "bobatan-alt.fujocoded.dev" }, + }, + ]); + }); + + test("namespaces ids by did/collection when multiple sources omit transform", async () => { + const bobatanPds = "https://bobatan-pds.fujocoded.test"; + const bobatanAltPds = "https://bobatan-alt-pds.fujocoded.test"; + + installScriptedRepo({ + did: "did:plc:bobatan", + handle: "bobatan.fujocoded.dev", + pds: bobatanPds, + collection: "site.standard.document", + pages: [[{ rkey: "shared", value: { title: "Bobatan doc" } }]], + }); + installScriptedRepo({ + did: "did:plc:bobatan-alt", + handle: "bobatan-alt.fujocoded.dev", + pds: bobatanAltPds, + collection: "site.standard.document", + pages: [[{ rkey: "shared", value: { title: "Alt doc" } }]], + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + sources: [ + { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, + { + repo: "bobatan-alt.fujocoded.dev", + collection: "site.standard.document", + }, + ], + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { + id: "did:plc:bobatan/site.standard.document/shared", + data: { title: "Bobatan doc" }, + }, + { + id: "did:plc:bobatan-alt/site.standard.document/shared", + data: { title: "Alt doc" }, + }, + ]); + }); + + test("groups records across sources and passes each group to transform", async () => { + const bobatanPds = "https://bobatan-pds.fujocoded.test"; + const bobatanAltPds = "https://bobatan-alt-pds.fujocoded.test"; + + installScriptedRepo({ + did: "did:plc:bobatan", + pds: bobatanPds, + collection: "site.standard.document", + pages: [ + [ + { rkey: "a", value: { slug: "shared", title: "Bobatan version" } }, + { rkey: "b", value: { slug: "solo", title: "Only entry" } }, + ], + ], + }); + installScriptedRepo({ + did: "did:plc:bobatan-alt", + pds: bobatanAltPds, + collection: "site.standard.document", + pages: [[{ rkey: "c", value: { slug: "shared", title: "Alt version" } }]], + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + sources: [ + { repo: "did:plc:bobatan", collection: "site.standard.document" }, + { repo: "did:plc:bobatan-alt", collection: "site.standard.document" }, + ], + groupBy: ({ value }) => String(value.slug), + transform: ({ key, records }) => ({ + id: key, + data: { + titles: records.map((record) => String(record.value.title)), + }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { id: "shared", data: { titles: ["Bobatan version", "Alt version"] } }, + { id: "solo", data: { titles: ["Only entry"] } }, + ]); + }); +}); diff --git a/astro-atproto-loader/__tests__/live-loader/entry.test.ts b/astro-atproto-loader/__tests__/live-loader/entry.test.ts new file mode 100644 index 0000000..5062d9f --- /dev/null +++ b/astro-atproto-loader/__tests__/live-loader/entry.test.ts @@ -0,0 +1,212 @@ +import { createMockRepoIdentity, FAKE_CID } from "@fujocoded/msw-atproto"; +import { http, HttpResponse } from "msw"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +import { createAtProtoCache } from "../../src/cache/index.ts"; +import { atProtoLiveLoader } from "../../src/loaders/live.ts"; +import { server } from "../msw/server.ts"; +import { failingGetRecord } from "../msw/handlers.ts"; +import { + installScriptedRecord, + installScriptedRepo, + PDS, +} from "../msw/install.ts"; + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("atProtoLiveLoader", () => { + test("loads a single entry directly by rkey and supports custom ids", async () => { + installScriptedRecord({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + record: { + rkey: "record-123", + value: { slug: "opening-keynote", title: "Opening keynote" }, + }, + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + }, + transform: ({ value }) => ({ + id: String(value.slug), + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadEntry({ + filter: { id: "opening-keynote", rkey: "record-123" }, + }); + + expect(result).toEqual({ + id: "opening-keynote", + data: { title: "Opening keynote" }, + }); + }); + + test("defaults single-record lookups to rkey ids when transform is omitted", async () => { + installScriptedRecord({ + did: "did:plc:passthrough-live", + collection: "place.stream.livestream", + record: { + rkey: "stream-1", + value: { + title: "Coworking stream", + createdAt: "2026-04-04T00:30:21Z", + }, + }, + }); + + const loader = atProtoLiveLoader< + readonly [{ repo: string; collection: string }], + { + title: string; + createdAt: string; + } + >({ + source: { + repo: "did:plc:passthrough-live", + collection: "place.stream.livestream", + }, + }); + + const result = await loader.loadEntry({ + filter: { id: "stream-1" }, + }); + + expect(result).toEqual({ + id: "stream-1", + data: { + title: "Coworking stream", + createdAt: "2026-04-04T00:30:21Z", + }, + }); + }); + + test("can disambiguate direct single-record loads across multiple sources", async () => { + const bobatanPds = "https://bobatan-pds.fujocoded.test"; + const bobatanAltPds = "https://bobatan-alt-pds.fujocoded.test"; + + let bobatanGetRecordCalls = 0; + const bobatanAltGetRecordParams: Array<{ + repo: string | null; + collection: string | null; + rkey: string | null; + }> = []; + + server.use( + ...createMockRepoIdentity({ + did: "did:plc:bobatan", + pds: bobatanPds, + handle: "bobatan.fujocoded.dev", + }).handlers(), + ...createMockRepoIdentity({ + did: "did:plc:bobatan-alt", + pds: bobatanAltPds, + handle: "bobatan-alt.fujocoded.dev", + }).handlers(), + http.get(`${bobatanPds}/xrpc/com.atproto.repo.getRecord`, () => { + bobatanGetRecordCalls += 1; + return new HttpResponse(JSON.stringify({ error: "UnexpectedCall" }), { + status: 500, + headers: { "content-type": "application/json" }, + }); + }), + http.get( + `${bobatanAltPds}/xrpc/com.atproto.repo.getRecord`, + ({ request }) => { + const url = new URL(request.url); + bobatanAltGetRecordParams.push({ + repo: url.searchParams.get("repo"), + collection: url.searchParams.get("collection"), + rkey: url.searchParams.get("rkey"), + }); + return HttpResponse.json({ + uri: "at://did:plc:bobatan-alt/site.standard.document/shared-rkey", + cid: FAKE_CID, + value: { slug: "alt/shared-rkey", title: "Alt shared doc" }, + }); + }, + ), + ); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + sources: [ + { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, + { + repo: "bobatan-alt.fujocoded.dev", + collection: "site.standard.document", + }, + ], + transform: ({ value }) => ({ + id: String(value.slug), + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadEntry({ + filter: { + id: "alt/shared-rkey", + rkey: "shared-rkey", + repo: "bobatan-alt.fujocoded.dev", + collection: "site.standard.document", + }, + }); + + expect(result).toEqual({ + id: "alt/shared-rkey", + data: { title: "Alt shared doc" }, + }); + expect(bobatanGetRecordCalls).toBe(0); + expect(bobatanAltGetRecordParams).toEqual([ + { + repo: "bobatan-alt.fujocoded.dev", + collection: "site.standard.document", + rkey: "shared-rkey", + }, + ]); + }); + test("falls back to the cached collection if direct single-record loading fails", async () => { + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [ + [ + { + rkey: "record-123", + value: { slug: "opening-keynote", title: "Opening keynote" }, + }, + ], + ], + }); + server.use(failingGetRecord(PDS)); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + }, + transform: ({ value }) => ({ + id: String(value.slug), + data: { title: String(value.title) }, + }), + }); + + await loader.loadCollection({}); + const result = await loader.loadEntry({ + filter: { id: "opening-keynote", rkey: "record-123" }, + }); + + expect(result).toEqual({ + id: "opening-keynote", + data: { title: "Opening keynote" }, + }); + }); +}); diff --git a/astro-atproto-loader/__tests__/live-loader/pagination.test.ts b/astro-atproto-loader/__tests__/live-loader/pagination.test.ts new file mode 100644 index 0000000..2a989d8 --- /dev/null +++ b/astro-atproto-loader/__tests__/live-loader/pagination.test.ts @@ -0,0 +1,314 @@ +import { createMockRepoIdentity, FAKE_CID } from "@fujocoded/msw-atproto"; +import { http, HttpResponse } from "msw"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +import { createAtProtoCache } from "../../src/cache/index.ts"; +import { atProtoLiveLoader } from "../../src/loaders/live.ts"; +import { server } from "../msw/server.ts"; +import { installScriptedRepo, PDS } from "../msw/install.ts"; + +// Unwraps a loadCollection result so a loader error fails the test with the +// actual error instead of an `expected false to equal [...]` diff. +const entriesOf = (result: object) => { + if ("error" in result && result.error instanceof Error) { + throw new Error(`loader returned an error: ${result.error.message}`, { + cause: result.error, + }); + } + return "entries" in result ? result.entries : undefined; +}; + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("atProtoLiveLoader", () => { + test("sends the listRecords XRPC query with the configured limit and no initial cursor", async () => { + const cursorCalls: Array = []; + + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [[{ rkey: "only", value: { title: "Only" } }]], + onCall: (cursor) => cursorCalls.push(cursor), + }); + + const requestLog: Array<{ limit: string | null; cursor: string | null }> = + []; + server.events.on("request:start", ({ request }) => { + const url = new URL(request.url); + if (url.pathname === "/xrpc/com.atproto.repo.listRecords") { + requestLog.push({ + limit: url.searchParams.get("limit"), + cursor: url.searchParams.get("cursor"), + }); + } + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + await loader.loadCollection({}); + + expect(requestLog).toEqual([{ limit: "100", cursor: null }]); + expect(cursorCalls).toEqual([null]); + }); + + test("stops after `source.limit` entries and skips remaining pages", async () => { + const cursorCalls: Array = []; + + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [ + [ + { rkey: "one", value: { title: "A" } }, + { rkey: "two", value: { title: "B" } }, + { rkey: "three", value: { title: "C" } }, + ], + [{ rkey: "four", value: { title: "D" } }], + ], + onCall: (cursor) => cursorCalls.push(cursor), + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + limit: 2, + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { id: "one", data: { title: "A" } }, + { id: "two", data: { title: "B" } }, + ]); + expect(cursorCalls).toEqual([null]); + }); + + test("caps the XRPC listRecords page size at `source.limit`", async () => { + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [[{ rkey: "only", value: { title: "Only" } }]], + }); + + const requestLog: Array<{ limit: string | null }> = []; + server.events.on("request:start", ({ request }) => { + const url = new URL(request.url); + if (url.pathname === "/xrpc/com.atproto.repo.listRecords") { + requestLog.push({ limit: url.searchParams.get("limit") }); + } + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + limit: 5, + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + await loader.loadCollection({}); + + expect(requestLog).toEqual([{ limit: "5" }]); + }); + + test("counts only post-filter entries against `source.limit`", async () => { + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [ + [ + { rkey: "one", value: { title: "A", published: false } }, + { rkey: "two", value: { title: "B", published: true } }, + { rkey: "three", value: { title: "C", published: false } }, + { rkey: "four", value: { title: "D", published: true } }, + ], + ], + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + limit: 2, + }, + filter: ({ value }) => value.published === true, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { id: "two", data: { title: "B" } }, + { id: "four", data: { title: "D" } }, + ]); + }); + + test("follows the cursor across multiple pages of listRecords", async () => { + const observedCursors: Array = []; + + installScriptedRepo({ + did: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + pages: [ + [ + { rkey: "one", value: { title: "Page one entry" } }, + { rkey: "two", value: { title: "Page one entry two" } }, + ], + [{ rkey: "three", value: { title: "Page two entry" } }], + ], + onCall: (cursor) => observedCursors.push(cursor), + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + limit: "all", + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { id: "one", data: { title: "Page one entry" } }, + { id: "two", data: { title: "Page one entry two" } }, + { id: "three", data: { title: "Page two entry" } }, + ]); + expect(observedCursors).toEqual([null, "1"]); + }); + + test("terminates the cursor loop once the PDS omits a next cursor", async () => { + let callCount = 0; + + server.use( + ...createMockRepoIdentity({ + did: "did:plc:testrepo", + pds: PDS, + }).handlers(), + http.get(`${PDS}/xrpc/com.atproto.repo.listRecords`, () => { + callCount += 1; + return HttpResponse.json({ + records: + callCount === 1 + ? [ + { + uri: "at://did:plc:testrepo/community.lexicon.calendar.event/only", + cid: FAKE_CID, + value: { title: "Only" }, + }, + ] + : [], + // cursor intentionally omitted on every page + }); + }), + ); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + source: { + repo: "did:plc:testrepo", + collection: "community.lexicon.calendar.event", + }, + transform: ({ value, rkey }) => ({ + id: rkey, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(callCount).toBe(1); + expect(entriesOf(result)).toEqual([ + { id: "only", data: { title: "Only" } }, + ]); + }); + + test("paginates each source independently when combined with multi-source", async () => { + const bobatanPds = "https://bobatan-pds.fujocoded.test"; + const bobatanAltPds = "https://bobatan-alt-pds.fujocoded.test"; + + const bobatanCursors: Array = []; + const bobatanAltCursors: Array = []; + + installScriptedRepo({ + did: "did:plc:bobatan", + pds: bobatanPds, + collection: "site.standard.document", + pages: [ + [{ rkey: "alpha", value: { title: "Bobatan page one" } }], + [{ rkey: "alpha-two", value: { title: "Bobatan page two" } }], + ], + onCall: (cursor) => bobatanCursors.push(cursor), + }); + installScriptedRepo({ + did: "did:plc:bobatan-alt", + pds: bobatanAltPds, + collection: "site.standard.document", + pages: [[{ rkey: "beta", value: { title: "Alt only page" } }]], + onCall: (cursor) => bobatanAltCursors.push(cursor), + }); + + const loader = atProtoLiveLoader({ + cache: createAtProtoCache(), + sources: [ + { + repo: "did:plc:bobatan", + collection: "site.standard.document", + limit: "all", + }, + { + repo: "did:plc:bobatan-alt", + collection: "site.standard.document", + limit: "all", + }, + ], + transform: ({ value, repo, rkey }) => ({ + id: `${repo.did}/${rkey}`, + data: { title: String(value.title) }, + }), + }); + + const result = await loader.loadCollection({}); + + expect(entriesOf(result)).toEqual([ + { id: "did:plc:bobatan/alpha", data: { title: "Bobatan page one" } }, + { id: "did:plc:bobatan/alpha-two", data: { title: "Bobatan page two" } }, + { id: "did:plc:bobatan-alt/beta", data: { title: "Alt only page" } }, + ]); + expect(bobatanCursors).toEqual([null, "1"]); + expect(bobatanAltCursors).toEqual([null]); + }); +}); diff --git a/astro-atproto-loader/__tests__/live-loader/per-source-swr.test.ts b/astro-atproto-loader/__tests__/live-loader/per-source-swr.test.ts new file mode 100644 index 0000000..1b55369 --- /dev/null +++ b/astro-atproto-loader/__tests__/live-loader/per-source-swr.test.ts @@ -0,0 +1,400 @@ +import { useMockAtprotoRepo } from "@fujocoded/msw-atproto"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { SOURCE_RETRY_TTL_MS } from "../../src/cache/source-caches.ts"; +import { + createAtProtoCache, + type AtProtoCache, +} from "../../src/cache/index.ts"; +import { atProtoLiveLoader } from "../../src/loaders/live.ts"; +import { server } from "../msw/server.ts"; +import { trackXrpcRequests } from "../msw/track-requests.ts"; + +const COLLECTION = "site.standard.document"; +const LIST_RECORDS = "com.atproto.repo.listRecords"; +const MAIN_DID = "did:plc:bobatan"; +const MAIN_PDS = "https://bobatan-pds.fujocoded.test"; +const ALT_DID = "did:plc:bobatan-alt"; +const ALT_PDS = "https://bobatan-alt-pds.fujocoded.test"; + +const UNAVAILABLE = { + status: 503, + error: "TemporarilyUnavailable", + message: "TemporarilyUnavailable", +}; + +const titled = (rkey: string, title: string) => [{ rkey, value: { title } }]; + +const installSources = () => ({ + main: useMockAtprotoRepo(server, { + did: MAIN_DID, + pds: MAIN_PDS, + records: { [COLLECTION]: titled("main", "Main warm") }, + }), + alt: useMockAtprotoRepo(server, { + did: ALT_DID, + pds: ALT_PDS, + records: { [COLLECTION]: titled("alt", "Alt warm") }, + }), +}); + +const byDidAndRkey = ({ + repo, + rkey, + value, +}: { + repo: { did: string }; + rkey: string; + value: Record; +}) => ({ + id: `${repo.did}/${rkey}`, + data: { title: String(value.title) }, +}); + +let cache: AtProtoCache; + +beforeEach(() => { + cache = createAtProtoCache(); +}); + +afterEach(async () => { + // Settle before restoring mocks so a late background refresh reports to + // this test's silenced console, not the next test's spy. + await cache.whenIdle(); + vi.restoreAllMocks(); +}); + +test("keeps a failed source's warm records beside a healthy source's refresh", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { main, alt } = installSources(); + const calls = trackXrpcRequests(server); + + let now = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + + const loader = atProtoLiveLoader({ + cache, + sources: [ + { repo: MAIN_DID, collection: COLLECTION }, + { repo: ALT_DID, collection: COLLECTION }, + ], + cacheTtl: 1, + transform: byDidAndRkey, + }); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [ + { id: `${MAIN_DID}/main`, data: { title: "Main warm" } }, + { id: `${ALT_DID}/alt`, data: { title: "Alt warm" } }, + ], + }); + + main.failOnce.listRecords(UNAVAILABLE); + alt.seed(COLLECTION, titled("alt", "Alt fresh")); + now = 1_002; + await loader.loadCollection({}); + + await vi.waitFor(async () => { + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [ + { id: `${MAIN_DID}/main`, data: { title: "Main warm" } }, + { id: `${ALT_DID}/alt`, data: { title: "Alt fresh" } }, + ], + }); + }); + // The message format itself is pinned in the dedicated operator-report + // test below; here we only care that the failure was reported once. + await vi.waitFor(() => expect(warn).toHaveBeenCalledTimes(1)); + expect(calls.count(MAIN_PDS, LIST_RECORDS)).toBe(2); + expect(calls.count(ALT_PDS, LIST_RECORDS)).toBe(2); +}); + +test("reports one underlying error through the real source-reader handler", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const main = useMockAtprotoRepo(server, { + did: MAIN_DID, + pds: MAIN_PDS, + records: { [COLLECTION]: titled("main", "Main warm") }, + }); + const calls = trackXrpcRequests(server); + + let now = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + + const loader = atProtoLiveLoader({ + cache, + source: { repo: MAIN_DID, collection: COLLECTION }, + cacheTtl: 1, + }); + + await loader.loadCollection({}); + main.failOnce.listRecords(UNAVAILABLE); + now = 1_002; + await loader.loadCollection({}); + + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledWith( + expect.stringContaining( + `[atproto-loader] source ${MAIN_DID}/${COLLECTION} refresh failed:`, + ), + expect.objectContaining({ + message: expect.stringContaining("TemporarilyUnavailable"), + }), + ); + }); + expect(warn.mock.calls[0]?.[1]).toBeInstanceOf(Error); + // One failed refresh attempt must produce exactly one operator report. + expect(warn).toHaveBeenCalledTimes(1); + expect(calls.count(MAIN_PDS, LIST_RECORDS)).toBe(2); +}); + +test("omits only a source that fails before it has warm records", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { main } = installSources(); + const calls = trackXrpcRequests(server); + main.failOnce.listRecords(UNAVAILABLE); + + const loader = atProtoLiveLoader({ + cache, + sources: [ + { repo: MAIN_DID, collection: COLLECTION }, + { repo: ALT_DID, collection: COLLECTION }, + ], + cacheTtl: 1, + transform: byDidAndRkey, + }); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ id: `${ALT_DID}/alt`, data: { title: "Alt warm" } }], + }); + expect(calls.count(MAIN_PDS, LIST_RECORDS)).toBe(1); + expect(calls.count(ALT_PDS, LIST_RECORDS)).toBe(1); +}); + +test("passes each source failure to an onSourceError callback with its source", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { main } = installSources(); + main.failOnce.listRecords(UNAVAILABLE); + const onSourceError = vi.fn(() => "skip" as const); + + const loader = atProtoLiveLoader({ + cache, + sources: [ + { repo: MAIN_DID, collection: COLLECTION }, + { repo: ALT_DID, collection: COLLECTION }, + ], + onSourceError, + transform: byDidAndRkey, + }); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [{ id: `${ALT_DID}/alt`, data: { title: "Alt warm" } }], + }); + expect(onSourceError).toHaveBeenCalledTimes(1); + expect(onSourceError).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining("TemporarilyUnavailable"), + }), + expect.objectContaining({ repo: MAIN_DID, collection: COLLECTION }), + ); +}); + +test("keeps each source on its own refresh schedule", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { main, alt } = installSources(); + const calls = trackXrpcRequests(server); + + let now = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + + const loader = atProtoLiveLoader({ + cache, + sources: [ + { repo: MAIN_DID, collection: COLLECTION }, + { repo: ALT_DID, collection: COLLECTION }, + ], + cacheTtl: 10, + transform: byDidAndRkey, + }); + + await loader.loadCollection({}); + + // Past both TTLs: main refreshes; alt's refresh fails, starting its + // private retry floor. Wait for both outcomes to land before touching the + // clock again — a refresh completing after a clock jump would stamp its + // cache time with the jumped value. + main.seed(COLLECTION, titled("main", "Main refreshed")); + alt.failOnce.listRecords(UNAVAILABLE); + now = 1_011; + await loader.loadCollection({}); + await vi.waitFor(async () => { + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [ + { id: `${MAIN_DID}/main`, data: { title: "Main refreshed" } }, + { id: `${ALT_DID}/alt`, data: { title: "Alt warm" } }, + ], + }); + }); + await vi.waitFor(() => expect(warn).toHaveBeenCalledTimes(1)); + + // Main is stale again and refetches; alt is still inside its retry + // floor and must not. + alt.seed(COLLECTION, titled("alt", "Alt recovered")); + now = 1_011 + SOURCE_RETRY_TTL_MS - 1; + await loader.loadCollection({}); + await vi.waitFor(() => expect(calls.count(MAIN_PDS, LIST_RECORDS)).toBe(3)); + await cache.whenIdle(); + expect(calls.count(ALT_PDS, LIST_RECORDS)).toBe(2); + + // One tick later the floor has elapsed: alt retries and recovers while + // main stays fresh. + now = 1_011 + SOURCE_RETRY_TTL_MS; + await loader.loadCollection({}); + await vi.waitFor(async () => { + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [ + { id: `${MAIN_DID}/main`, data: { title: "Main refreshed" } }, + { id: `${ALT_DID}/alt`, data: { title: "Alt recovered" } }, + ], + }); + }); + await cache.whenIdle(); + expect(calls.count(MAIN_PDS, LIST_RECORDS)).toBe(3); + expect(calls.count(ALT_PDS, LIST_RECORDS)).toBe(3); +}); + +test("preserves the cold-start error when every source fails", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + const { main, alt } = installSources(); + const calls = trackXrpcRequests(server); + main.failOnce.listRecords({ + status: 503, + error: "MainSourceUnavailable", + message: "MainSourceUnavailable", + }); + alt.failOnce.listRecords({ + status: 503, + error: "AltSourceUnavailable", + message: "AltSourceUnavailable", + }); + + const loader = atProtoLiveLoader({ + cache, + sources: [ + { repo: MAIN_DID, collection: COLLECTION }, + { repo: ALT_DID, collection: COLLECTION }, + ], + onInitialLoadError: "throw", + transform: byDidAndRkey, + }); + + const result = await loader.loadCollection({}); + + expect(result).toHaveProperty("error"); + if (!("error" in result) || !result.error) { + throw new Error("Expected the live loader to return an error"); + } + expect(result.error.cause).toBeInstanceOf(AggregateError); + const aggregate = result.error.cause as AggregateError; + expect(aggregate.message).toBe("All AtProto sources failed"); + expect(aggregate.errors).toHaveLength(2); + expect( + aggregate.errors.map((error) => + error instanceof Error && "error" in error ? error.error : undefined, + ), + ).toEqual(["MainSourceUnavailable", "AltSourceUnavailable"]); + expect(calls.count(MAIN_PDS, LIST_RECORDS)).toBe(1); + expect(calls.count(ALT_PDS, LIST_RECORDS)).toBe(1); +}); + +test("serves every last-good record set when every warm source fails", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { main, alt } = installSources(); + const calls = trackXrpcRequests(server); + + let now = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => now); + + const loader = atProtoLiveLoader({ + cache, + sources: [ + { repo: MAIN_DID, collection: COLLECTION }, + { repo: ALT_DID, collection: COLLECTION }, + ], + cacheTtl: 1, + onInitialLoadError: "throw", + transform: byDidAndRkey, + }); + + await loader.loadCollection({}); + main.failOnce.listRecords(UNAVAILABLE); + alt.failOnce.listRecords(UNAVAILABLE); + now = 1_002; + await loader.loadCollection({}); + await vi.waitFor(() => { + expect(calls.count(MAIN_PDS, LIST_RECORDS)).toBe(2); + expect(calls.count(ALT_PDS, LIST_RECORDS)).toBe(2); + }); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [ + { id: `${MAIN_DID}/main`, data: { title: "Main warm" } }, + { id: `${ALT_DID}/alt`, data: { title: "Alt warm" } }, + ], + }); + // One report per failing source; the message format is pinned in the + // dedicated operator-report test above. + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledTimes(2); + }); +}); + +test("preserves source order and last-value dedupe when every source is healthy", async () => { + useMockAtprotoRepo(server, { + did: MAIN_DID, + pds: MAIN_PDS, + records: { + [COLLECTION]: [ + { rkey: "first", value: { slug: "first", title: "First" } }, + { + rkey: "old-shared", + value: { slug: "shared", title: "Old shared" }, + }, + ], + }, + }); + useMockAtprotoRepo(server, { + did: ALT_DID, + pds: ALT_PDS, + records: { + [COLLECTION]: [ + { rkey: "second", value: { slug: "second", title: "Second" } }, + { + rkey: "new-shared", + value: { slug: "shared", title: "New shared" }, + }, + ], + }, + }); + + const loader = atProtoLiveLoader({ + cache, + sources: [ + { repo: MAIN_DID, collection: COLLECTION }, + { repo: ALT_DID, collection: COLLECTION }, + ], + transform: ({ value }) => ({ + id: String(value.slug), + data: { title: String(value.title) }, + }), + }); + + await expect(loader.loadCollection({})).resolves.toMatchObject({ + entries: [ + { id: "first", data: { title: "First" } }, + { id: "shared", data: { title: "New shared" } }, + { id: "second", data: { title: "Second" } }, + ], + }); +}); diff --git a/astro-atproto-loader/__tests__/msw/handlers.ts b/astro-atproto-loader/__tests__/msw/handlers.ts index 0926d1c..ba43879 100644 --- a/astro-atproto-loader/__tests__/msw/handlers.ts +++ b/astro-atproto-loader/__tests__/msw/handlers.ts @@ -1,18 +1,11 @@ -import { P256Keypair } from "@atproto/crypto"; +import { FAKE_CID } from "@fujocoded/msw-atproto"; import { http, HttpResponse, type HttpHandler } from "msw"; -// Valid CIDv1 that passes `multiformats/cid`'s `CID.parse`. The lexicon -// validator rejects responses that lack a parseable cid, so every fake -// record we serve needs one. -export const FAKE_CID = - "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"; - -// `@atproto/identity` insists on a well-formed DID document — signingKey + -// handle + pds all required — even though the loader only cares about pds. -// Generate one real P-256 multibase key at module load and reuse it. -const keypair = await P256Keypair.create(); -const SIGNING_KEY_MULTIBASE = keypair.did().slice("did:key:".length); - +// Don't replace these with msw-atproto's stateful repo mock. That mock manages +// pagination internally, but tests here assert the wire protocol itself: which +// cursor each request carries, how many pages get fetched, what limit is sent. +// Scripting exact pages/cursors locally is the point. Identity mocking has no +// such needs, so it uses @fujocoded/msw-atproto directly. export type FakeRecord = { did: string; rkey: string; @@ -20,54 +13,6 @@ export type FakeRecord = { cid?: string; }; -export type RepoIdentity = { - did: string; - pds: string; - handle?: string; -}; - -export const mockRepoIdentity = ({ - did, - pds, - handle, -}: RepoIdentity): HttpHandler[] => { - const advertisedHandle = handle ?? `${did.split(":").pop()}.example.test`; - - const handlers: HttpHandler[] = [ - http.get(`https://plc.directory/${encodeURIComponent(did)}`, () => - HttpResponse.json({ - id: did, - alsoKnownAs: [`at://${advertisedHandle}`], - verificationMethod: [ - { - id: `${did}#atproto`, - type: "Multikey", - controller: did, - publicKeyMultibase: SIGNING_KEY_MULTIBASE, - }, - ], - service: [ - { - id: "#atproto_pds", - type: "AtprotoPersonalDataServer", - serviceEndpoint: pds, - }, - ], - }), - ), - ]; - - if (handle) { - handlers.push( - http.get(`https://${handle}/.well-known/atproto-did`, () => - HttpResponse.text(did), - ), - ); - } - - return handlers; -}; - export type MockListRecordsConfig = { pds: string; repo: string; diff --git a/astro-atproto-loader/__tests__/msw/install.ts b/astro-atproto-loader/__tests__/msw/install.ts new file mode 100644 index 0000000..5739eea --- /dev/null +++ b/astro-atproto-loader/__tests__/msw/install.ts @@ -0,0 +1,58 @@ +import { createMockRepoIdentity } from "@fujocoded/msw-atproto"; + +import { server } from "./server.ts"; +import { mockGetRecord, mockListRecords, type FakeRecord } from "./handlers.ts"; + +export const PDS = "https://pds.example.test"; + +type ScriptedRecord = Omit; + +type ScriptedRepoConfig = { + did: string; + handle?: string; + pds?: string; + collection: string; +}; + +// Composes identity resolution + scripted listRecords pages; the loader is +// configured with the handle when one exists, so that's what arrives as ?repo=. +export const installScriptedRepo = ({ + did, + handle, + pds = PDS, + collection, + pages, + onCall, +}: ScriptedRepoConfig & { + pages: ScriptedRecord[][]; + onCall?: (cursor: string | null) => void; +}) => { + server.use( + ...createMockRepoIdentity({ did, pds, handle }).handlers(), + mockListRecords({ + pds, + repo: handle ?? did, + collection, + onCall, + pages: pages.map((page) => page.map((record) => ({ did, ...record }))), + }), + ); +}; + +export const installScriptedRecord = ({ + did, + handle, + pds = PDS, + collection, + record, +}: ScriptedRepoConfig & { record: ScriptedRecord }) => { + server.use( + ...createMockRepoIdentity({ did, pds, handle }).handlers(), + mockGetRecord({ + pds, + repo: handle ?? did, + collection, + record: { did, ...record }, + }), + ); +}; diff --git a/astro-atproto-loader/__tests__/msw/track-requests.ts b/astro-atproto-loader/__tests__/msw/track-requests.ts new file mode 100644 index 0000000..b92aefd --- /dev/null +++ b/astro-atproto-loader/__tests__/msw/track-requests.ts @@ -0,0 +1,41 @@ +type RequestStartListener = (info: { request: Request }) => void; + +export type XrpcRequestTracker = { + /** Every XRPC request seen since tracking started, in arrival order. */ + requests: { origin: string; method: string; url: URL }[]; + /** How many requests hit `method` on the PDS at `pds` so far. */ + count: (pds: string, method: string) => number; +}; + +/** + * Records every XRPC request the MSW server intercepts, so tests can assert + * how often an endpoint was hit without hand-rolling counting handlers. + * + * Use this only where the request count IS the behavior under test (refresh + * throttling, request dedupe); prefer asserting on returned values otherwise. + * The listener registered here is cleaned up by setup.ts's + * `server.events.removeAllListeners()` between tests. + */ +export const trackXrpcRequests = (server: { + events: { on(event: "request:start", listener: RequestStartListener): void }; +}): XrpcRequestTracker => { + const requests: XrpcRequestTracker["requests"] = []; + + server.events.on("request:start", ({ request }) => { + const url = new URL(request.url); + const method = /^\/xrpc\/(.+)$/.exec(url.pathname)?.[1]; + if (method) { + requests.push({ origin: url.origin, method, url }); + } + }); + + return { + requests, + count: (pds, method) => { + const origin = new URL(pds).origin; + return requests.filter( + (entry) => entry.origin === origin && entry.method === method, + ).length; + }, + }; +}; diff --git a/astro-atproto-loader/__tests__/setup.ts b/astro-atproto-loader/__tests__/setup.ts index 0f465ac..083134a 100644 --- a/astro-atproto-loader/__tests__/setup.ts +++ b/astro-atproto-loader/__tests__/setup.ts @@ -1,22 +1,14 @@ import { afterAll, afterEach, beforeAll, vi } from "vitest"; import { server } from "./msw/server.ts"; -vi.mock("node:dns/promises", () => { - const fail = async () => { - throw Object.assign(new Error("ENODATA (test stub)"), { code: "ENODATA" }); - }; - return { - default: { - resolveTxt: fail, - lookup: fail, - Resolver: class { - setServers() {} - resolveTxt = fail; - }, - }, - }; +vi.mock("node:dns/promises", async (importActual) => { + const { createDnsMock } = await import("@fujocoded/msw-atproto"); + return createDnsMock(importActual); }); beforeAll(() => server.listen({ onUnhandledRequest: "error" })); -afterEach(() => server.resetHandlers()); +afterEach(() => { + server.resetHandlers(); + server.events.removeAllListeners(); +}); afterAll(() => server.close()); diff --git a/astro-atproto-loader/__tests__/static-loader.test.ts b/astro-atproto-loader/__tests__/static-loader.test.ts new file mode 100644 index 0000000..7620795 --- /dev/null +++ b/astro-atproto-loader/__tests__/static-loader.test.ts @@ -0,0 +1,351 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { LoaderContext } from "astro/loaders"; + +import { createAtProtoCache } from "../src/cache/index.ts"; +import { atProtoStaticLoader } from "../src/loaders/static.ts"; +import { installScriptedRepo } from "./msw/install.ts"; + +// Minimal LoaderContext stand-in: only `store` and `parseData` drive the +// static loader today; the rest are stubs so any new context usage fails +// loudly on a mock instead of an `undefined is not a function`. +const fakeLoaderContext = ({ + store, + parseData, +}: { + store: { clear: () => void; set: (entry: unknown) => unknown }; + parseData: (props: { + id: string; + data: Record; + filePath?: string; + }) => Promise>; +}): LoaderContext => + ({ + collection: "static-test-collection", + store, + parseData, + meta: new Map(), + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }, + generateDigest: vi.fn(() => "digest"), + config: {}, + }) as unknown as LoaderContext; + +const staticHarness = ( + parseImpl: (props: { + id: string; + data: Record; + filePath?: string; + }) => Promise> = async ({ data }) => data, +) => { + const store = { clear: vi.fn(), set: vi.fn() }; + const parseData = vi.fn(parseImpl); + return { store, parseData, context: fakeLoaderContext({ store, parseData }) }; +}; + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("atProtoStaticLoader", () => { + test("loads a single source into the Astro data store", async () => { + installScriptedRepo({ + did: "did:plc:staticrepo", + handle: "static.example.com", + collection: "site.standard.document", + pages: [ + [ + { + rkey: "doc-1", + value: { title: "Static doc", body: "Hello from Astro" }, + }, + ], + ], + }); + + const { store, parseData, context } = staticHarness(); + + const loader = atProtoStaticLoader({ + cache: createAtProtoCache(), + source: { + repo: "static.example.com", + collection: "site.standard.document", + }, + transform: ({ value, rkey, repo }) => ({ + id: `${repo.did}/${rkey}`, + data: { title: String(value.title), repo: repo.handle ?? repo.did }, + body: String(value.body), + }), + }); + + await loader.load(context); + + expect(store.clear).toHaveBeenCalledTimes(1); + expect(parseData).toHaveBeenCalledWith({ + id: "did:plc:staticrepo/doc-1", + data: { title: "Static doc", repo: "static.example.com" }, + filePath: undefined, + }); + expect(store.set).toHaveBeenCalledWith({ + id: "did:plc:staticrepo/doc-1", + data: { title: "Static doc", repo: "static.example.com" }, + body: "Hello from Astro", + filePath: undefined, + }); + }); + + test("defaults to passthrough entries for a single source when transform is omitted", async () => { + installScriptedRepo({ + did: "did:plc:passthrough-static", + handle: "passthrough.example.com", + collection: "place.stream.livestream", + pages: [ + [ + { + rkey: "stream-1", + value: { + title: "Coworking stream", + createdAt: "2026-04-04T00:30:21Z", + }, + }, + ], + ], + }); + + const { store, parseData, context } = staticHarness(); + + const loader = atProtoStaticLoader< + readonly [{ repo: string; collection: string }], + { + title: string; + createdAt: string; + } + >({ + source: { + repo: "passthrough.example.com", + collection: "place.stream.livestream", + }, + }); + + await loader.load(context); + + expect(parseData).toHaveBeenCalledWith({ + id: "stream-1", + data: { + title: "Coworking stream", + createdAt: "2026-04-04T00:30:21Z", + }, + filePath: undefined, + }); + expect(store.set).toHaveBeenCalledWith({ + id: "stream-1", + data: { + title: "Coworking stream", + createdAt: "2026-04-04T00:30:21Z", + }, + body: undefined, + filePath: undefined, + }); + }); + + test("preserves Date values returned by parseData", async () => { + installScriptedRepo({ + did: "did:plc:dates-static", + handle: "dates.example.com", + collection: "place.stream.livestream", + pages: [ + [ + { + rkey: "stream-1", + value: { + title: "Morning stream", + createdAt: "2026-04-04T00:30:21Z", + }, + }, + ], + ], + }); + + const createdAt = new Date("2026-04-04T00:30:21Z"); + const { store, context } = staticHarness(async ({ data }) => ({ + ...data, + createdAt, + })); + + const loader = atProtoStaticLoader< + readonly [{ repo: string; collection: string }], + { + title: string; + createdAt: Date; + } + >({ + source: { + repo: "dates.example.com", + collection: "place.stream.livestream", + }, + }); + + await loader.load(context); + + expect(store.set).toHaveBeenCalledWith({ + id: "stream-1", + data: { + title: "Morning stream", + createdAt, + }, + body: undefined, + filePath: undefined, + }); + expect(store.set.mock.calls[0]?.[0].data.createdAt).toBeInstanceOf(Date); + }); + + test("surfaces schema parse failures from parseData", async () => { + installScriptedRepo({ + did: "did:plc:staticrepo", + handle: "static.example.com", + collection: "site.standard.document", + pages: [[{ rkey: "doc-1", value: { title: "Static doc" } }]], + }); + + const parseError = new Error( + "Schema parse failed for did:plc:staticrepo/doc-1", + ); + const { store, parseData, context } = staticHarness(async () => { + throw parseError; + }); + + const loader = atProtoStaticLoader({ + cache: createAtProtoCache(), + source: { + repo: "static.example.com", + collection: "site.standard.document", + }, + transform: ({ value, rkey, repo }) => ({ + id: `${repo.did}/${rkey}`, + data: { title: String(value.title), repo: repo.handle ?? repo.did }, + }), + }); + + await expect(loader.load(context)).rejects.toThrow( + "Schema parse failed for did:plc:staticrepo/doc-1", + ); + + expect(store.clear).toHaveBeenCalledTimes(1); + expect(parseData).toHaveBeenCalledTimes(1); + expect(store.set).not.toHaveBeenCalled(); + }); + + test("supports multiple sources and deduplicates by transformed id", async () => { + const bobatanPds = "https://bobatan-pds.fujocoded.test"; + const bobatanAltPds = "https://bobatan-alt-pds.fujocoded.test"; + + installScriptedRepo({ + did: "did:plc:bobatan", + handle: "bobatan.fujocoded.dev", + pds: bobatanPds, + collection: "site.standard.document", + pages: [ + [ + { + rkey: "alpha", + value: { slug: "shared-post", title: "Older title" }, + }, + ], + ], + }); + installScriptedRepo({ + did: "did:plc:bobatan-alt", + handle: "bobatan-alt.fujocoded.dev", + pds: bobatanAltPds, + collection: "site.standard.document", + pages: [ + [ + { + rkey: "beta", + value: { slug: "shared-post", title: "Newer title" }, + }, + ], + ], + }); + + const { store, context } = staticHarness(); + + const loader = atProtoStaticLoader({ + cache: createAtProtoCache(), + sources: [ + { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, + { + repo: "bobatan-alt.fujocoded.dev", + collection: "site.standard.document", + }, + ], + transform: ({ value, repo }) => ({ + id: String(value.slug), + data: { title: String(value.title), repo: repo.handle ?? repo.did }, + }), + }); + + await loader.load(context); + + expect(store.clear).toHaveBeenCalledTimes(1); + expect(store.set).toHaveBeenCalledTimes(1); + expect(store.set).toHaveBeenCalledWith({ + id: "shared-post", + data: { title: "Newer title", repo: "bobatan-alt.fujocoded.dev" }, + body: undefined, + filePath: undefined, + }); + }); + + test("namespaces ids by did/collection when multiple static sources omit transform", async () => { + const bobatanPds = "https://bobatan-pds.fujocoded.test"; + const bobatanAltPds = "https://bobatan-alt-pds.fujocoded.test"; + + installScriptedRepo({ + did: "did:plc:bobatan", + handle: "bobatan.fujocoded.dev", + pds: bobatanPds, + collection: "site.standard.document", + pages: [[{ rkey: "shared", value: { title: "Bobatan doc" } }]], + }); + installScriptedRepo({ + did: "did:plc:bobatan-alt", + handle: "bobatan-alt.fujocoded.dev", + pds: bobatanAltPds, + collection: "site.standard.document", + pages: [[{ rkey: "shared", value: { title: "Alt doc" } }]], + }); + + const { store, context } = staticHarness(); + + const loader = atProtoStaticLoader({ + cache: createAtProtoCache(), + sources: [ + { repo: "bobatan.fujocoded.dev", collection: "site.standard.document" }, + { + repo: "bobatan-alt.fujocoded.dev", + collection: "site.standard.document", + }, + ], + }); + + await loader.load(context); + + expect(store.set).toHaveBeenCalledTimes(2); + expect(store.set).toHaveBeenNthCalledWith(1, { + id: "did:plc:bobatan/site.standard.document/shared", + data: { title: "Bobatan doc" }, + body: undefined, + filePath: undefined, + }); + expect(store.set).toHaveBeenNthCalledWith(2, { + id: "did:plc:bobatan-alt/site.standard.document/shared", + data: { title: "Alt doc" }, + body: undefined, + filePath: undefined, + }); + }); +}); diff --git a/astro-atproto-loader/package.json b/astro-atproto-loader/package.json index 867262f..ce138bf 100644 --- a/astro-atproto-loader/package.json +++ b/astro-atproto-loader/package.json @@ -54,7 +54,7 @@ "@atproto/syntax": "^0.4.2" }, "devDependencies": { - "@atproto/crypto": "^0.4.5", + "@fujocoded/msw-atproto": "^0.0.1", "astro": "^5.16.14", "msw": "^2.13.4", "tsdown": "^0.14.2", diff --git a/astro-atproto-loader/src/cache/index.ts b/astro-atproto-loader/src/cache/index.ts new file mode 100644 index 0000000..7b7200e --- /dev/null +++ b/astro-atproto-loader/src/cache/index.ts @@ -0,0 +1,105 @@ +import { IdResolver } from "@atproto/identity"; + +import { TtlCache } from "./ttl.ts"; +import type { AtProtoRecordRepo, RecordValue } from "../types.ts"; +import { isDefinitiveRecordFailure } from "../utils.ts"; + +export const IDENTITY_CACHE_TTL = 60 * 60 * 1_000; +export const IDENTITY_RETRY_TTL = 30 * 1_000; + +export const HYDRATED_RECORD_CACHE_TTL = 5 * 60_000; +/** + * Transient failures (network errors, 5xx) retry after this floor. + * Definitive ones (e.g. the PDS said the record doesn't exist, or its value can + * never parse) are held as long as successes, since retrying can't help. + */ +export const HYDRATED_RECORD_RETRY_TTL = 5_000; +export const HYDRATED_RECORD_NOT_FOUND_TTL = HYDRATED_RECORD_CACHE_TTL; + +/** + * A fixed 20,000-entry ceiling covers a large active set of hydrated + * references while bounding arbitrary record JSON to a predictable order of + * magnitude. There is no consumer evidence that this needs public tuning; if + * real workloads show cache churn, revisit the fixed value with that evidence. + */ +export const HYDRATED_RECORD_CACHE_MAX_ENTRIES = 20_000; + +/** + * The shared state a loader reads through: resolved identities and hydrated + * records, plus the `IdResolver` itself and its internal DID/handle caches. + */ +export interface AtProtoCache { + identity: TtlCache; + hydratedRecords: TtlCache< + string, + { value: RecordValue; repo: AtProtoRecordRepo } + >; + resolver: IdResolver; + /** + * Tells `whenIdle` about fire-and-forget work, like a background refresh + * nobody awaits. + * + * Call it for promises with no awaiter: since nothing keeps the process alive + * for them, they can be killed mid-flight by a serverless runtime. Pairing + * this with awaiting `whenIdle` before shutdown ensures that all work that + * was started has settled, so the process doesn't exit while a background + * refresh is still in progress. + * + * When called on awaited promises, this call just makes `whenIdle` stall on + * other callers' in-flight requests, which will be settled anyway on their + * own. + */ + onRefresh: (refresh: Promise) => void; + /** + * Resolves once every background data refresh has settled, including those + * started while waiting. Never rejects: failed refreshes already report + * through their own error handling. + * + * Await this before asserting on request counts in tests, or before letting a + * serverless runtime freeze the process. + */ + whenIdle: () => Promise; +} + +export const createAtProtoCache = (): AtProtoCache => { + const pendingRefreshes = new Set>(); + + return { + identity: new TtlCache({ + successTtl: IDENTITY_CACHE_TTL, + failureTtl: IDENTITY_RETRY_TTL, + }), + hydratedRecords: new TtlCache({ + successTtl: HYDRATED_RECORD_CACHE_TTL, + failureTtl: (error) => + isDefinitiveRecordFailure(error) + ? HYDRATED_RECORD_NOT_FOUND_TTL + : HYDRATED_RECORD_RETRY_TTL, + maxEntries: HYDRATED_RECORD_CACHE_MAX_ENTRIES, + }), + resolver: new IdResolver({}), + onRefresh: (refresh) => { + // A refresh's failure is already reported through its own onError + // handling before the promise reaches us, so all we track here is *when* + // it settles. Without the catch, we'd have an unhandled rejection if the + // refresh fails, which is not what we want. + const settled = refresh.catch(() => {}); + pendingRefreshes.add(settled); + void settled.finally(() => pendingRefreshes.delete(settled)); + }, + whenIdle: async () => { + //We must wait for all refreshes to settle, including any that + // start while waiting. + while (pendingRefreshes.size > 0) { + await Promise.allSettled(pendingRefreshes); + } + }, + }; +}; + +/** + * Loaders use this immutable default when callers do not provide their own + * cache. Sharing this instance makes caches span every default loader in the + * process. + */ +export const defaultAtProtoCache = createAtProtoCache(); diff --git a/astro-atproto-loader/src/cache/source-caches.ts b/astro-atproto-loader/src/cache/source-caches.ts new file mode 100644 index 0000000..ea9f2ac --- /dev/null +++ b/astro-atproto-loader/src/cache/source-caches.ts @@ -0,0 +1,160 @@ +import type { AtProtoCache } from "./index.ts"; +import type { + AtProtoLoaderSource, + AtProtoRecordCallbackArgs, + AtProtoRecordFilterOptions, + FetchRecord, + OnSourceError, +} from "../types.ts"; +import { fetchFromSource } from "../pipeline/source.ts"; +import { getCollectionsLabel } from "../utils.ts"; +import { createSwrCache } from "./swr.ts"; + +export type OnInitialLoadError = "throw" | "empty"; + +type SourceRecords = AtProtoRecordCallbackArgs[]; +type SourceErrorDecision = "skip" | "throw"; + +export const SOURCE_RETRY_TTL_MS = 5_000; + +class SourceFetchFailure { + constructor( + readonly sourceError: unknown, + readonly decision: SourceErrorDecision, + ) {} +} + +interface CreateSourceCacheArgs< + Sources extends readonly AtProtoLoaderSource[], +> { + source: AtProtoLoaderSource; + callbacks: AtProtoRecordFilterOptions; + fetchRecord: FetchRecord; + cacheTtl: number; + onSourceError: OnSourceError; + caches: AtProtoCache; +} + +const createSourceCache = < + Sources extends readonly AtProtoLoaderSource[], +>({ + source, + callbacks, + fetchRecord, + cacheTtl, + onSourceError, + caches, +}: CreateSourceCacheArgs): (() => Promise) => + createSwrCache({ + ttl: cacheTtl, + failureTtl: SOURCE_RETRY_TTL_MS, + onRefresh: (refresh) => caches.onRefresh(refresh), + fetch: async () => { + try { + return await fetchFromSource(source, callbacks, fetchRecord, caches); + } catch (error) { + throw new SourceFetchFailure( + error, + typeof onSourceError === "function" + ? onSourceError(error, source) + : onSourceError, + ); + } + }, + onError: (error) => { + const sourceError = + error instanceof SourceFetchFailure ? error.sourceError : error; + console.warn( + `[atproto-loader] source ${source.repo}/${source.collection} refresh failed:`, + sourceError, + ); + }, + }); + +export interface CreateSourceCachesArgs< + Sources extends readonly AtProtoLoaderSource[], +> { + sources: Sources; + callbacks: AtProtoRecordFilterOptions; + fetchRecord: FetchRecord; + cacheTtl: number; + onSourceError: OnSourceError; + onInitialLoadError: OnInitialLoadError; + caches: AtProtoCache; +} + +/** + * Cache each source's fetched records independently and read them as one + * ordered set. + * + * Readers stay in source declaration order. A warm source whose refresh fails + * keeps returning its stale records. Cold failures follow the error policies: + * `onSourceError` decides whether a single cold source is skipped or fatal, + * and `onInitialLoadError` decides whether a fatal cold read surfaces to the + * caller (`'throw'`) or degrades to an empty read (`'empty'`). Either way the + * failure is reported to the console. + */ +export const createSourceCaches = < + Sources extends readonly AtProtoLoaderSource[], +>({ + sources, + callbacks, + fetchRecord, + cacheTtl, + onSourceError, + onInitialLoadError, + caches, +}: CreateSourceCachesArgs) => { + const readers = sources.map((source) => + createSourceCache({ + source, + callbacks, + fetchRecord, + cacheTtl, + onSourceError, + caches, + }), + ); + + const readAllSources = async (): Promise => { + const results = await Promise.allSettled(readers.map((read) => read())); + const sourceRecords: SourceRecords[] = []; + const errors: unknown[] = []; + + for (const result of results) { + if (result.status === "fulfilled") { + sourceRecords.push(result.value); + continue; + } + + if (!(result.reason instanceof SourceFetchFailure)) { + throw result.reason; + } + if (result.reason.decision === "throw") { + throw result.reason.sourceError; + } + errors.push(result.reason.sourceError); + } + + if (errors.length > 0 && errors.length === results.length) { + throw new AggregateError(errors, "All AtProto sources failed"); + } + + return sourceRecords; + }; + + return async (): Promise => { + try { + return await readAllSources(); + } catch (error) { + console.error( + `[atproto-loader:${getCollectionsLabel(sources)}] refresh failed:`, + error, + ); + if (onInitialLoadError === "throw") { + throw error; + } + return []; + } + }; +}; diff --git a/astro-atproto-loader/src/cache/swr.ts b/astro-atproto-loader/src/cache/swr.ts new file mode 100644 index 0000000..22759fe --- /dev/null +++ b/astro-atproto-loader/src/cache/swr.ts @@ -0,0 +1,93 @@ +export interface SwrCacheOptions { + /** + * Fetches fresh data. Errors are caught and passed to `onError`; the cache + * keeps serving stale data until the next refresh succeeds. + */ + fetch: () => Promise; + /** + * Milliseconds after which cached data is considered stale. A read past the + * TTL still returns the cached value synchronously but triggers a background + * refresh. + */ + ttl: number; + /** + * Minimum milliseconds between refresh attempts after a failure. + */ + failureTtl: number; + onError: (error: unknown) => void; + /** + * Receives every refresh this cache starts, so the caller can wait for + * background work to settle. The first "cold" load is awaited by the caller + * and is deliberately not reported here. + */ + onRefresh: (refresh: Promise) => void; +} + +/** + * A small stale-while-revalidate cache for one source's records. + * + * The first read awaits that source's initial fetch. Later reads return its + * cached records immediately and start a background refresh when they are + * older than `ttl`. Concurrent refreshes for the source share one in-flight + * promise. A warm refresh failure preserves the source's last successful + * records, reports the error through `onError`, and observes `failureTtl` + * before trying again. A cold failure rejects; the caller decides what a + * failed first load means. + */ +export const createSwrCache = ({ + fetch, + ttl, + failureTtl, + onError, + onRefresh, +}: SwrCacheOptions) => { + let cached: { value: Snapshot } | undefined; + let cacheTime = 0; + let failureTime: number | undefined; + let refreshPromise: Promise | undefined; + + const triggerRefresh = () => { + if (refreshPromise) { + return refreshPromise; + } + + refreshPromise = (async () => { + try { + const value = await fetch(); + cached = { value }; + cacheTime = Date.now(); + failureTime = undefined; + return value; + } catch (error) { + failureTime = Date.now(); + onError(error); + if (!cached) { + throw error; + } + return cached.value; + } finally { + refreshPromise = undefined; + } + })(); + + return refreshPromise; + }; + + const read = async () => { + if (!cached) { + return triggerRefresh(); + } + + const now = Date.now(); + const retryFloorElapsed = + failureTime === undefined || now - failureTime >= failureTtl; + + if (now - cacheTime > ttl && retryFloorElapsed) { + onRefresh(triggerRefresh()); + } + + return cached.value; + }; + + return read; +}; diff --git a/astro-atproto-loader/src/cache/ttl.ts b/astro-atproto-loader/src/cache/ttl.ts new file mode 100644 index 0000000..c87f7f3 --- /dev/null +++ b/astro-atproto-loader/src/cache/ttl.ts @@ -0,0 +1,143 @@ +export interface TtlCacheOptions { + successTtl: number; + /** + * How long rejections stay cached before a retry is allowed. The function + * form receives the rejection error, so different failure kinds can get + * different TTLs. + */ + failureTtl: number | ((error: unknown) => number); + maxEntries?: number; +} + +type CacheEntry = + | { status: "success"; value: Value; expiresAt: number } + | { status: "failure"; error: unknown; expiresAt: number }; + +export class TtlCache { + private readonly entries = new Map>(); + private readonly inFlight = new Map>(); + private readonly successTtl: number; + private readonly failureTtl: number | ((error: unknown) => number); + private readonly maxEntries: number | undefined; + private generation = 0; + + constructor({ successTtl, failureTtl, maxEntries }: TtlCacheOptions) { + if (Number.isNaN(successTtl) || successTtl < 0) { + throw new RangeError("successTtl must be a non-negative number"); + } + if ( + typeof failureTtl === "number" && + (Number.isNaN(failureTtl) || failureTtl < 0) + ) { + throw new RangeError("failureTtl must be a non-negative number"); + } + if ( + maxEntries !== undefined && + (!Number.isSafeInteger(maxEntries) || maxEntries < 0) + ) { + throw new RangeError("maxEntries must be a non-negative safe integer"); + } + + this.successTtl = successTtl; + this.failureTtl = failureTtl; + this.maxEntries = maxEntries; + } + + get(key: Key, load: () => Promise): Promise { + const entry = this.entries.get(key); + + if (entry && entry.expiresAt > Date.now()) { + this.touch(key, entry); + return entry.status === "success" + ? Promise.resolve(entry.value) + : Promise.reject(entry.error); + } + + if (entry) { + this.entries.delete(key); + } + + const existingRequest = this.inFlight.get(key); + if (existingRequest) { + return existingRequest; + } + + const generation = this.generation; + const request = Promise.resolve().then(load); + const pending = request + .then( + (value) => { + if ( + this.generation === generation && + this.inFlight.get(key) === pending + ) { + this.setEntry(key, { + status: "success", + value, + expiresAt: Date.now() + this.successTtl, + }); + } + + return value; + }, + (error: unknown) => { + if ( + this.generation === generation && + this.inFlight.get(key) === pending + ) { + this.setEntry(key, { + status: "failure", + error, + expiresAt: Date.now() + this.failureTtlFor(error), + }); + } + + throw error; + }, + ) + .finally(() => { + // Delete the in-flight request only if it is still the same one + // we created, to avoid deleting a newer request that was started + // after this one. + if (this.inFlight.get(key) === pending) { + this.inFlight.delete(key); + } + }); + + this.inFlight.set(key, pending); + return pending; + } + + private failureTtlFor(error: unknown): number { + return typeof this.failureTtl === "function" + ? this.failureTtl(error) + : this.failureTtl; + } + + private touch(key: Key, entry: CacheEntry): void { + this.entries.delete(key); + this.entries.set(key, entry); + } + + private setEntry(key: Key, entry: CacheEntry): void { + this.touch(key, entry); + + if (this.maxEntries === undefined) { + return; + } + + while (this.entries.size > this.maxEntries) { + const oldest = this.entries.keys().next(); + if (oldest.done) { + return; + } + this.entries.delete(oldest.value); + } + } + + reset(): void { + this.generation += 1; + this.entries.clear(); + this.inFlight.clear(); + } +} diff --git a/astro-atproto-loader/src/client/identity.ts b/astro-atproto-loader/src/client/identity.ts index 1c923a4..f1a5e6b 100644 --- a/astro-atproto-loader/src/client/identity.ts +++ b/astro-atproto-loader/src/client/identity.ts @@ -1,22 +1,18 @@ import { AtpBaseClient } from "@atproto/api"; -import { IdResolver } from "@atproto/identity"; import type { DidString } from "@atproto/syntax"; -const identityResolver = new IdResolver({}); +import type { AtProtoCache } from "../cache/index.ts"; +import type { AtProtoRecordRepo } from "../types.ts"; -// Cache the in-flight identity lookup, keyed by the caller-provided repo -// string. Each lookup resolves the handle to a DID, then the DID to a PDS. -// Failed lookups are evicted so the next caller retries instead of reusing -// the rejected promise. -type AtprotoIdentity = { did: DidString; pds: string }; -const identityCache = new Map>(); - -export const getDid = async (repo: string): Promise => { +const getDid = async ( + repo: string, + caches: AtProtoCache, +): Promise => { if (repo.startsWith("did:")) { return repo as DidString; } - const did = await identityResolver.handle.resolve(repo); + const did = await caches.resolver.handle.resolve(repo); if (!did) { throw new Error(`Could not resolve a DID for ${repo}`); } @@ -24,34 +20,26 @@ export const getDid = async (repo: string): Promise => { return did as DidString; }; -const getIdentity = (repo: string): Promise => { - const cached = identityCache.get(repo); - if (cached) { - return cached; - } - - const request = (async () => { - const did = await getDid(repo); - const atprotoData = await identityResolver.did.resolveAtprotoData(did); +const getIdentity = ( + repo: string, + caches: AtProtoCache, +): Promise => + caches.identity.get(repo, async () => { + const did = await getDid(repo, caches); + const atprotoData = await caches.resolver.did.resolveAtprotoData(did); if (!atprotoData?.pds) { throw new Error(`Could not resolve a PDS for ${repo}`); } return { did, pds: atprotoData.pds }; - })(); - - identityCache.set(repo, request); - request.catch(() => { - // Kick out failed lookups so retries don't reuse the rejected promise. - if (identityCache.get(repo) === request) { - identityCache.delete(repo); - } }); - return request; -}; - -export const getPds = async (repo: string): Promise => - (await getIdentity(repo)).pds; +export const getPds = async ( + repo: string, + caches: AtProtoCache, +): Promise => (await getIdentity(repo, caches)).pds; -export const getClient = async (repo: string): Promise => - new AtpBaseClient((await getIdentity(repo)).pds); +export const getClient = async ( + repo: string, + caches: AtProtoCache, +): Promise => + new AtpBaseClient((await getIdentity(repo, caches)).pds); diff --git a/astro-atproto-loader/src/client/records.ts b/astro-atproto-loader/src/client/records.ts index 9798b45..fce221c 100644 --- a/astro-atproto-loader/src/client/records.ts +++ b/astro-atproto-loader/src/client/records.ts @@ -5,6 +5,7 @@ import { } from "@atproto/api"; import type { DidString, HandleString } from "@atproto/syntax"; +import type { AtProtoCache } from "../cache/index.ts"; import type { AtProtoLoaderSource, AtProtoRecordContext, @@ -18,6 +19,7 @@ export const isRecordValue = (value: unknown): value is RecordValue => export const toRecordContext = async ( source: AtProtoLoaderSource, record: { uri: string; cid?: string }, + caches: AtProtoCache, ): Promise => { const aturi = new AtUri(record.uri); if (!aturi.rkey) { @@ -30,7 +32,7 @@ export const toRecordContext = async ( const handle = source.repo.startsWith("did:") ? undefined : (source.repo as HandleString); - const pds = await getPds(source.repo); + const pds = await getPds(source.repo, caches); return { repo: { did: aturi.host as DidString, handle, pds }, @@ -44,8 +46,9 @@ export const toRecordContext = async ( export const listRecordsPage = async ( source: AtProtoLoaderSource, opts: { limit: number; cursor?: string }, + caches: AtProtoCache, ): Promise => { - const client = await getClient(source.repo); + const client = await getClient(source.repo, caches); const { data } = await client.com.atproto.repo.listRecords({ repo: source.repo, collection: source.collection, @@ -58,8 +61,9 @@ export const listRecordsPage = async ( export const getSingleRecord = async ( source: AtProtoLoaderSource, rkey: string, + caches: AtProtoCache, ): Promise => { - const client = await getClient(source.repo); + const client = await getClient(source.repo, caches); const { data } = await client.com.atproto.repo.getRecord({ repo: source.repo, collection: source.collection, diff --git a/astro-atproto-loader/src/index.ts b/astro-atproto-loader/src/index.ts index a2ab6c8..d7c426f 100644 --- a/astro-atproto-loader/src/index.ts +++ b/astro-atproto-loader/src/index.ts @@ -4,6 +4,7 @@ export { defineAtProtoLiveCollection } from "./loaders/live.ts"; export type { AtProtoLiveLoaderEntryFilter, AtProtoQueryFilterArgs, + OnInitialLoadError, } from "./loaders/live.ts"; export { defineAtProtoCollection } from "./loaders/static.ts"; diff --git a/astro-atproto-loader/src/loaders/live.ts b/astro-atproto-loader/src/loaders/live.ts index 6d9a19d..760f4a8 100644 --- a/astro-atproto-loader/src/loaders/live.ts +++ b/astro-atproto-loader/src/loaders/live.ts @@ -2,15 +2,21 @@ import type { LiveDataEntry } from "astro"; import { defineLiveCollection } from "astro/content/config"; import type { LiveLoader } from "astro/loaders"; -import { runPipeline } from "../pipeline/run.ts"; -import { runSingleFetch } from "../pipeline/single.ts"; +import { + createSourceCaches, + type OnInitialLoadError, +} from "../cache/source-caches.ts"; +import { defaultAtProtoCache, type AtProtoCache } from "../cache/index.ts"; +import { createFetchRecord } from "../pipeline/fetch-record.ts"; +import { joinSourceRecords } from "../pipeline/join.ts"; +import { findEntryViaFetch, type EntryLookup } from "../pipeline/single.ts"; import type { AtProtoLoaderSource, - AtProtoRecordCallbacks, AtProtoRecordFilterOptions, AtProtoRecordGroupBy, AtProtoRecordGroupTransform, AtProtoRecordTransform, + AtProtoTransformOptions, MaybePromise, OnSourceError, SchemaInput, @@ -20,10 +26,9 @@ import { type AtProtoSourceOptions, getCollectionsLabel, normalizeSources, - toNamespacedEntry, + resolveRecordCallbacks, toError, toSafePojo, - toRkeyEntry, } from "../utils.ts"; export interface AtProtoLiveLoaderEntryFilter { @@ -41,24 +46,19 @@ export interface AtProtoQueryFilterArgs< filter: QueryFilter; } -type AtProtoLiveTransformOptions< - Sources extends readonly AtProtoLoaderSource[], - Data extends Record, -> = - | { - groupBy?: never; - transform?: AtProtoRecordTransform>; - } - | { - groupBy: AtProtoRecordGroupBy; - transform: AtProtoRecordGroupTransform>; - }; +export type { OnInitialLoadError }; export type AtProtoLiveLoaderOptions< Sources extends readonly AtProtoLoaderSource[], Data extends Record, QueryFilter extends Record = never, > = AtProtoRecordFilterOptions & { + /** + * Cache shared by this loader. Omit it to use the process-wide default. + * Pass the same cache to several loaders to share identity and hydrated + * record results between them. + */ + cache?: AtProtoCache; /** * What to do when a source fails, according to what's passed: * - `sources: [...]` => defaults to `'skip'`, so one flaky PDS doesn't take @@ -76,93 +76,23 @@ export type AtProtoLiveLoaderOptions< args: AtProtoQueryFilterArgs, ) => MaybePromise; /** - * How long, in milliseconds, the cached collection is considered fresh - * before a background refresh is triggered. Defaults to five minutes. + * What to do if a cold source read fails (when the source does + * not yet have a successful snapshot). The read includes `filter`. + * `groupBy` and `transform` run afterward and always return loader errors. + * + * - `'empty'` => treat the first failed fetch as an empty collection / miss + * (default) + * - `'throw'` => surface the error to Astro */ - cacheTtl?: number; -} & AtProtoSourceOptions & - AtProtoLiveTransformOptions; - -interface SwrCacheOptions { + onInitialLoadError?: OnInitialLoadError; /** - * Fetches fresh data. Errors are caught and passed to `onError`; the cache - * keeps serving stale data until the next refresh succeeds. + * How long, in milliseconds, each source's cached records are considered + * fresh before a background refresh is triggered. Hydrated records use an + * independent fixed five-minute policy. Defaults to five minutes. */ - fetch: () => Promise; - /** - * Milliseconds after which cached data is considered stale. A read past the - * TTL still returns the cached value synchronously but triggers a background - * refresh. - */ - ttl: number; - /** - * Initial cache value returned before the first successful fetch. - */ - initial: Snapshot; - onError: (error: unknown) => void; -} - -/** - * A small stale-while-revalidate cache. - * - * The first read awaits the initial fetch. Later reads return the cached - * value immediately, and kick off a background refresh if the value is older - * than `ttl`. Concurrent refreshes share a single in-flight promise. If a - * refresh fails (for example because every source threw and the pipeline - * raised an `AggregateError`), the previous snapshot is preserved and the - * error is reported through `onError`. - */ -const createSwrCache = ({ - fetch, - ttl, - initial, - onError, -}: SwrCacheOptions) => { - let cached: Snapshot = initial; - let cacheTime = 0; - let refreshPromise: Promise | undefined; - - const triggerRefresh = () => { - if (refreshPromise) { - return refreshPromise; - } - - refreshPromise = (async () => { - try { - const value = await fetch(); - cached = value; - cacheTime = Date.now(); - return value; - } catch (error) { - onError(error); - return cached; - } finally { - refreshPromise = undefined; - } - })(); - - return refreshPromise; - }; - - return async () => { - if (cacheTime === 0) { - return triggerRefresh(); - } - - if (Date.now() - cacheTime > ttl) { - void triggerRefresh(); - } - - return cached; - }; -}; - -interface EntryLookup { - requestedId: string | undefined; - rkey: string | undefined; - repo: string | undefined; - collection: string | undefined; -} + cacheTtl?: number; +} & AtProtoSourceOptions & + AtProtoTransformOptions>; const getRequestedLookup = ( filter: AtProtoLiveLoaderEntryFilter | { id: string }, @@ -184,75 +114,6 @@ const findEntryInCache = >( (entry) => entry.id === requestedId || (rkey ? entry.id === rkey : false), ); -/** - * Try to resolve a single requested entry with direct `getRecord` calls, - * instead of waiting on the full collection refresh. - * - * The lookup goes through these steps: - * - * - Pick the sources whose `repo` and `collection` match the lookup - * - Fetch each by `rkey`, in parallel when several sources match - * - Return the first entry that lines up with the requested `id` - * - * Returns `undefined` in these cases: - * - * - No `rkey` was provided - * - No sources match the lookup - * - Nothing resolved to the requested `id` - * - * Callers should fall back to looking inside the cached collection. - */ -const findEntryViaFetch = async < - Sources extends readonly AtProtoLoaderSource[], - Data extends Record, ->( - sources: readonly AtProtoLoaderSource[], - callbacks: AtProtoRecordCallbacks>, - { requestedId, rkey, repo, collection }: EntryLookup, -): Promise | undefined> => { - if (!rkey) { - return undefined; - } - - const candidates = sources.filter( - (source) => - (!repo || source.repo === repo) && - (!collection || source.collection === collection), - ); - - const matchesRequestedId = (entry: LiveDataEntry) => - !requestedId || entry.id === requestedId; - - const flatten = (entry: LiveDataEntry): LiveDataEntry => ({ - ...entry, - data: toSafePojo(entry.data), - }); - - const [onlyCandidate] = candidates; - if (candidates.length === 1 && onlyCandidate) { - const entry = await runSingleFetch(onlyCandidate, callbacks, rkey); - if (!entry) return undefined; - return matchesRequestedId(entry) ? flatten(entry) : undefined; - } - - if (candidates.length > 1) { - const results = await Promise.allSettled( - candidates.map((source) => runSingleFetch(source, callbacks, rkey)), - ); - - const match = results.find( - (result): result is PromiseFulfilledResult> => - result.status === "fulfilled" && - result.value !== undefined && - matchesRequestedId(result.value), - ); - - return match ? flatten(match.value) : undefined; - } - - return undefined; -}; - export const atProtoLiveLoader = < const Sources extends readonly AtProtoLoaderSource[], Data extends Record, @@ -261,60 +122,47 @@ export const atProtoLiveLoader = < options: AtProtoLiveLoaderOptions, ): LiveLoader => { const sources = normalizeSources(options); - const { cacheTtl = 5 * 60_000 } = options; - const fallbackTransform = - sources.length > 1 ? toNamespacedEntry : toRkeyEntry; - const callbacks: AtProtoRecordCallbacks< - Sources, - LiveDataEntry - > = "groupBy" in options && options.groupBy - ? { - filter: options.filter, - groupBy: options.groupBy, - transform: options.transform, - } - : { - filter: options.filter, - transform: - options.transform ?? - (fallbackTransform as AtProtoRecordTransform< - Sources, - LiveDataEntry - >), - }; + const { cacheTtl = 5 * 60_000, onInitialLoadError = "empty" } = options; + const callbacks = resolveRecordCallbacks>( + sources, + options, + ); const onSourceError: OnSourceError = options.onSourceError ?? ("sources" in options && options.sources ? "skip" : "throw"); - const getEntries = createSwrCache[]>({ - ttl: cacheTtl, - initial: [], - fetch: async () => { - const entries = await runPipeline({ - sources, - callbacks, - onSourceError, - }); - return entries.map((entry) => ({ - ...entry, - data: toSafePojo(entry.data), - })); - }, - onError: (error) => { - console.error( - `[atproto-loader:${getCollectionsLabel(sources)}] refresh failed:`, - error, - ); - }, + const caches = options.cache ?? defaultAtProtoCache; + const fetchRecord = createFetchRecord(caches); + const readSources = createSourceCaches({ + sources, + callbacks, + fetchRecord, + cacheTtl, + onSourceError, + onInitialLoadError, + caches, }); + const getEntries = async (): Promise[]> => { + const entries = await joinSourceRecords({ + sourceRecords: await readSources(), + callbacks, + fetchRecord, + }); + return entries.map((entry) => ({ + ...entry, + data: toSafePojo(entry.data), + })); + }; + return { name: "atproto-loader", async loadCollection({ filter }) { try { const entries = await getEntries(); + if (!filter || !options.queryFilter) { return { entries }; } @@ -341,9 +189,14 @@ export const atProtoLiveLoader = < const lookup = getRequestedLookup(filter); try { - const direct = await findEntryViaFetch(sources, callbacks, lookup); + const direct = await findEntryViaFetch( + sources, + callbacks, + lookup, + caches, + ); if (direct) { - return direct; + return { ...direct, data: toSafePojo(direct.data) }; } const entries = await getEntries(); @@ -373,7 +226,22 @@ type LiveBaseConfig< QueryFilter extends Record, > = { outputSchema: Schema; + /** + * Cache shared by this collection's loader. Omit it to use the + * process-wide default. + */ + cache?: AtProtoCache; onSourceError?: OnSourceError; + /** + * What to do if a cold source read fails. The read includes + * `filter`; `groupBy` and `transform` failures always return loader errors. + */ + onInitialLoadError?: OnInitialLoadError; + /** + * How long each source's cached records stay fresh before a background refresh. + * Hydrated records use an independent fixed five-minute policy. Defaults to + * five minutes. + */ cacheTtl?: number; queryFilter?: ( args: AtProtoQueryFilterArgs, QueryFilter>, diff --git a/astro-atproto-loader/src/loaders/static.ts b/astro-atproto-loader/src/loaders/static.ts index 7dc431c..d7a6050 100644 --- a/astro-atproto-loader/src/loaders/static.ts +++ b/astro-atproto-loader/src/loaders/static.ts @@ -1,14 +1,15 @@ import { defineCollection } from "astro/content/config"; import type { Loader, LoaderContext } from "astro/loaders"; +import { defaultAtProtoCache, type AtProtoCache } from "../cache/index.ts"; import { runPipeline } from "../pipeline/run.ts"; import type { AtProtoLoaderSource, - AtProtoRecordCallbacks, AtProtoRecordFilterOptions, AtProtoRecordGroupBy, AtProtoRecordGroupTransform, AtProtoRecordTransform, + AtProtoTransformOptions, OnSourceError, SchemaInput, SchemaLike, @@ -16,9 +17,8 @@ import type { import { type AtProtoSourceOptions, normalizeSources, - toNamespacedEntry, + resolveRecordCallbacks, toSafePojo, - toRkeyEntry, } from "../utils.ts"; export interface AtProtoStaticDataEntry> { @@ -28,26 +28,17 @@ export interface AtProtoStaticDataEntry> { filePath?: string; } -type AtProtoStaticTransformOptions< - Sources extends readonly AtProtoLoaderSource[], - Data extends Record, -> = - | { - groupBy?: never; - transform?: AtProtoRecordTransform>; - } - | { - groupBy: AtProtoRecordGroupBy; - transform: AtProtoRecordGroupTransform< - Sources, - AtProtoStaticDataEntry - >; - }; - export type AtProtoStaticLoaderOptions< Sources extends readonly AtProtoLoaderSource[], Data extends Record, > = AtProtoRecordFilterOptions & { + /** + * Cache shared by this loader. Pass the same cache to several loaders to + * share identity and hydrated record results between them. + * + * Omit it to use the default. + */ + cache?: AtProtoCache; /** * What to do when a single source fails. Defaults to `'throw'` everywhere, * so a broken source fails the build instead of quietly publishing partial @@ -56,7 +47,7 @@ export type AtProtoStaticLoaderOptions< */ onSourceError?: OnSourceError; } & AtProtoSourceOptions & - AtProtoStaticTransformOptions; + AtProtoTransformOptions>; export const atProtoStaticLoader = < const Sources extends readonly AtProtoLoaderSource[], @@ -65,28 +56,14 @@ export const atProtoStaticLoader = < options: AtProtoStaticLoaderOptions, ): Loader => { const sources = normalizeSources(options); - const fallbackTransform = - sources.length > 1 ? toNamespacedEntry : toRkeyEntry; - const callbacks: AtProtoRecordCallbacks< + const callbacks = resolveRecordCallbacks< Sources, + Data, AtProtoStaticDataEntry - > = "groupBy" in options && options.groupBy - ? { - filter: options.filter, - groupBy: options.groupBy, - transform: options.transform, - } - : { - filter: options.filter, - transform: - options.transform ?? - (fallbackTransform as AtProtoRecordTransform< - Sources, - AtProtoStaticDataEntry - >), - }; + >(sources, options); const onSourceError: OnSourceError = options.onSourceError ?? "throw"; + const caches = options.cache ?? defaultAtProtoCache; return { name: "atproto-loader", @@ -96,6 +73,7 @@ export const atProtoStaticLoader = < sources, callbacks, onSourceError, + caches, }); context.store.clear(); @@ -123,6 +101,11 @@ type StaticBaseConfig< Schema extends SchemaLike, > = { outputSchema: Schema; + /** + * Cache shared by this collection's loader. Omit it to use the + * process-wide default. + */ + cache?: AtProtoCache; onSourceError?: OnSourceError; } & AtProtoRecordFilterOptions; diff --git a/astro-atproto-loader/src/pipeline/fetch-record.ts b/astro-atproto-loader/src/pipeline/fetch-record.ts index 3f14483..652336b 100644 --- a/astro-atproto-loader/src/pipeline/fetch-record.ts +++ b/astro-atproto-loader/src/pipeline/fetch-record.ts @@ -3,29 +3,68 @@ import type { DidString } from "@atproto/syntax"; import { getPds } from "../client/identity.ts"; import { getSingleRecord, isRecordValue } from "../client/records.ts"; +import type { AtProtoCache } from "../cache/index.ts"; import type { AtProtoRecordRepo, FetchRecord, RecordValue } from "../types.ts"; -import { getErrorMessage } from "../utils.ts"; - -type FetchedRecord = { value: RecordValue; repo: AtProtoRecordRepo }; +import { DefinitiveRecordError, getErrorMessage } from "../utils.ts"; /** - * Build a per-cycle `fetchRecord` helper. + * Build a `fetchRecord` helper. * - * Each instance keeps an in-memory cache that maps AT-URIs to in-flight - * fetch promises. Concurrent callers asking for the same URI within one - * pipeline cycle share a single network request. + * Each helper reads through a shared cache that maps AT-URIs to fetched + * records and in-flight fetch promises. Concurrent callers asking for the + * same URI share a single network request, and completed records are + * reused across pipeline cycles and collections. * * Each successful resolution carries the fetched record's owning DID and * PDS alongside its `value`, so callers can build blob URLs for the * hydrated record without re-resolving identity. * * All failures simply return `null`, but each one prints a distinct - * `console.warn` for debugging. + * `console.warn` for debugging. Internally the cache sees them as + * rejections so it can apply a shorter retry floor. */ -export const createFetchRecord = (): FetchRecord => { - const cache = new Map>(); +export const createFetchRecord = (caches: AtProtoCache): FetchRecord => { + const fetchBase = async ( + atUri: string, + parsed: AtUri, + ): Promise<{ value: RecordValue; repo: AtProtoRecordRepo }> => { + const [data, pds] = await Promise.all([ + getSingleRecord( + { repo: parsed.host, collection: parsed.collection }, + parsed.rkey, + caches, + ), + getPds(parsed.host, caches), + ]).catch((error: unknown) => { + console.warn( + `[atproto-loader] fetchRecord: getRecord failed for ${atUri}: ${getErrorMessage(error)}`, + ); + throw error; + }); + + if (!isRecordValue(data.value)) { + const error = new DefinitiveRecordError( + `Record value is not an object at ${atUri}`, + ); + console.warn( + `[atproto-loader] fetchRecord: record value is not an object at ${atUri}`, + ); + throw error; + } - const fetchBase = async (atUri: string): Promise => { + return { + value: data.value, + repo: { did: parsed.host as DidString, pds }, + }; + }; + + return async ({ + atUri, + parse, + }: { + atUri: string; + parse?: (value: unknown) => ParsedValue; + }): Promise<{ value: ParsedValue; repo: AtProtoRecordRepo } | null> => { let parsed: AtUri; try { parsed = new AtUri(atUri); @@ -43,47 +82,17 @@ export const createFetchRecord = (): FetchRecord => { return null; } + let fetched: { value: RecordValue; repo: AtProtoRecordRepo }; try { - const [data, pds] = await Promise.all([ - getSingleRecord( - { repo: parsed.host, collection: parsed.collection }, - parsed.rkey, - ), - getPds(parsed.host), - ]); - if (!isRecordValue(data.value)) { - console.warn( - `[atproto-loader] fetchRecord: record value is not an object at ${atUri}`, - ); - return null; - } - return { - value: data.value, - repo: { did: parsed.host as DidString, pds }, - }; - } catch (error) { - console.warn( - `[atproto-loader] fetchRecord: getRecord failed for ${atUri}: ${getErrorMessage(error)}`, + fetched = await caches.hydratedRecords.get(atUri, () => + fetchBase(atUri, parsed), ); + } catch { return null; } - }; - return async ({ - atUri, - parse, - }: { - atUri: string; - parse?: (value: unknown) => Parsed; - }): Promise<{ value: Parsed; repo: AtProtoRecordRepo } | null> => { - let pending = cache.get(atUri); - if (!pending) { - pending = fetchBase(atUri); - cache.set(atUri, pending); - } - const fetched = await pending; - if (fetched === null) return null; - if (!parse) return fetched as { value: Parsed; repo: AtProtoRecordRepo }; + if (!parse) + return fetched as { value: ParsedValue; repo: AtProtoRecordRepo }; try { return { value: parse(fetched.value), repo: fetched.repo }; } catch (error) { diff --git a/astro-atproto-loader/src/pipeline/join.ts b/astro-atproto-loader/src/pipeline/join.ts new file mode 100644 index 0000000..2d1cba3 --- /dev/null +++ b/astro-atproto-loader/src/pipeline/join.ts @@ -0,0 +1,118 @@ +import type { + ArgsUnion, + AtProtoLoaderSource, + AtProtoRecordCallbackArgs, + AtProtoRecordCallbacks, + AtProtoRecordGroupBy, + AtProtoRecordGroupTransformArgs, + FetchRecord, +} from "../types.ts"; + +const dedupeEntries = ( + entries: Entry[], +): Entry[] => { + const byId = new Map(); + for (const entry of entries) { + byId.set(entry.id, entry); + } + return [...byId.values()]; +}; + +const groupRecords = async < + Sources extends readonly AtProtoLoaderSource[], +>( + records: AtProtoRecordCallbackArgs[], + groupBy: AtProtoRecordGroupBy, +): Promise[]>> => { + const byKey = new Map[]>(); + + for (const args of records) { + const recordArgs = args as ArgsUnion; + const key = await groupBy(recordArgs); + if (typeof key !== "string") { + throw new Error( + `AtProto loader groupBy must return a string key for ${args.repo.handle ?? args.repo.did}/${args.collection}/${args.rkey}`, + ); + } + + const group = byKey.get(key) ?? []; + group.push(recordArgs); + byKey.set(key, group); + } + + return byKey; +}; + +export interface JoinSourceRecordsArgs< + Sources extends readonly AtProtoLoaderSource[], + Entry extends { id: string }, +> { + sourceRecords: AtProtoRecordCallbackArgs[][]; + callbacks: AtProtoRecordCallbacks; + fetchRecord: FetchRecord; +} + +/** + * Join each source's already-fetched records into entries. + * + * Sources are flattened in their supplied order. Records can then be grouped + * across sources before transforming, and entries are deduped by `id`. + */ +export const joinSourceRecords = async < + Sources extends readonly AtProtoLoaderSource[], + Entry extends { id: string }, +>({ + sourceRecords, + callbacks, + fetchRecord, +}: JoinSourceRecordsArgs): Promise => { + // Put all records in source order. + const merged = sourceRecords.flat(); + const entries: Entry[] = []; + + if (!callbacks.groupBy) { + // Turn each record into an entry. + let dropped = 0; + for (const args of merged) { + const entry = await callbacks.transform(args as ArgsUnion); + if (entry === null || entry === undefined) { + dropped++; + continue; + } + entries.push(entry); + } + if (dropped > 0) { + console.debug( + `[atproto-loader] transform dropped ${dropped}/${merged.length} entries`, + ); + } + + // Keep the last entry for each id. + return dedupeEntries(entries); + } + + // Gather related records before transforming. + const byKey = await groupRecords(merged, callbacks.groupBy); + let dropped = 0; + for (const [key, records] of byKey) { + const groupArgs: AtProtoRecordGroupTransformArgs = { + key, + records, + fetchRecord, + }; + const entry = await callbacks.transform(groupArgs); + if (entry === null || entry === undefined) { + dropped++; + continue; + } + entries.push(entry); + } + if (dropped > 0) { + console.debug( + `[atproto-loader] transform dropped ${dropped}/${byKey.size} groups`, + ); + } + + // Keep the last entry for each id. + return dedupeEntries(entries); +}; diff --git a/astro-atproto-loader/src/pipeline/run.ts b/astro-atproto-loader/src/pipeline/run.ts index 3bbee27..dc0225c 100644 --- a/astro-atproto-loader/src/pipeline/run.ts +++ b/astro-atproto-loader/src/pipeline/run.ts @@ -1,51 +1,15 @@ +import type { AtProtoCache } from "../cache/index.ts"; import type { - ArgsUnion, AtProtoLoaderSource, AtProtoRecordCallbackArgs, AtProtoRecordCallbacks, - AtProtoRecordGroupBy, - AtProtoRecordGroupTransformArgs, OnSourceError, } from "../types.ts"; import { getErrorMessage } from "../utils.ts"; import { createFetchRecord } from "./fetch-record.ts"; +import { joinSourceRecords } from "./join.ts"; import { fetchFromSource } from "./source.ts"; -const dedupeEntries = ( - entries: Entry[], -): Entry[] => { - const byId = new Map(); - for (const entry of entries) { - byId.set(entry.id, entry); - } - return [...byId.values()]; -}; - -const groupRecords = async < - Sources extends readonly AtProtoLoaderSource[], ->( - records: AtProtoRecordCallbackArgs[], - groupBy: AtProtoRecordGroupBy, -): Promise[]>> => { - const byKey = new Map[]>(); - - for (const args of records) { - const recordArgs = args as ArgsUnion; - const key = await groupBy(recordArgs); - if (typeof key !== "string") { - throw new Error( - `AtProto loader groupBy must return a string key for ${args.repo.handle ?? args.repo.did}/${args.collection}/${args.rkey}`, - ); - } - - const group = byKey.get(key) ?? []; - group.push(recordArgs); - byKey.set(key, group); - } - - return byKey; -}; - export interface RunPipelineArgs< Sources extends readonly AtProtoLoaderSource[], Entry extends { id: string }, @@ -53,10 +17,11 @@ export interface RunPipelineArgs< sources: Sources; callbacks: AtProtoRecordCallbacks; onSourceError?: OnSourceError; + caches: AtProtoCache; } /** - * Run a full read cycle across every source: + * Run the static loader's full read cycle across every source: * * - For each source: fetch, validate, parse, filter records * - Merge survivors in source declaration order @@ -64,14 +29,17 @@ export interface RunPipelineArgs< * - Run `transform` per record or per group => nullish returns drop the entry * - Dedupe entries by `id` * - * Error handling depends on `onSourceError`: + * All source reads are started concurrently and allowed to complete before their + * results are evaluated. Error handling then depends on `onSourceError`: * - * - `'throw'` => the first source error rethrows immediately and the rest of - * is abandoned + * - `'throw'` => rethrow the first failed source in declaration order after + * every source read has settled * - `'skip'` (or a function returning `'skip'`) => failing sources drop their - * contribution. If every source fails, it will still throws an `AggregateError` so - * the live loader's SWR cache can keep serving its last good snapshot and - * the static loader can fail the build + * contribution. If every source fails, throw an `AggregateError` containing + * every source failure so the static build fails with the full cause set + * + * The live loader does not use this path. It acquires records through its + * per-source stale-while-revalidate cache, then joins the resulting records. */ export const runPipeline = async < Sources extends readonly AtProtoLoaderSource[], @@ -80,23 +48,26 @@ export const runPipeline = async < sources, callbacks, onSourceError = "skip", + caches, }: RunPipelineArgs): Promise => { - const fetchRecord = createFetchRecord(); + const fetchRecord = createFetchRecord(caches); // Ask every source for records. const results = await Promise.allSettled( - sources.map((source) => fetchFromSource(source, callbacks, fetchRecord)), + sources.map((source) => + fetchFromSource(source, callbacks, fetchRecord, caches), + ), ); // Keep successful sources and report failed ones. - const buckets: AtProtoRecordCallbackArgs[][] = []; + const sourceRecords: AtProtoRecordCallbackArgs[][] = []; const errors: unknown[] = []; for (let i = 0; i < results.length; i++) { const result = results[i]!; const source = sources[i]!; if (result.status === "fulfilled") { - buckets.push(result.value); + sourceRecords.push(result.value); continue; } @@ -116,53 +87,5 @@ export const runPipeline = async < throw new AggregateError(errors, "All AtProto sources failed"); } - // Put all records in source order. - const merged = buckets.flat(); - const entries: Entry[] = []; - - if (!callbacks.groupBy) { - // Turn each record into an entry. - let dropped = 0; - for (const args of merged) { - const entry = await callbacks.transform(args as ArgsUnion); - if (entry === null || entry === undefined) { - dropped++; - continue; - } - entries.push(entry); - } - if (dropped > 0) { - console.debug( - `[atproto-loader] transform dropped ${dropped}/${merged.length} entries`, - ); - } - - // Keep the last entry for each id. - return dedupeEntries(entries); - } - - // Gather related records before transforming. - const byKey = await groupRecords(merged, callbacks.groupBy); - let dropped = 0; - for (const [key, records] of byKey) { - const groupArgs: AtProtoRecordGroupTransformArgs = { - key, - records, - fetchRecord, - }; - const entry = await callbacks.transform(groupArgs); - if (entry === null || entry === undefined) { - dropped++; - continue; - } - entries.push(entry); - } - if (dropped > 0) { - console.debug( - `[atproto-loader] transform dropped ${dropped}/${byKey.size} groups`, - ); - } - - // Keep the last entry for each id. - return dedupeEntries(entries); + return joinSourceRecords({ sourceRecords, callbacks, fetchRecord }); }; diff --git a/astro-atproto-loader/src/pipeline/single.ts b/astro-atproto-loader/src/pipeline/single.ts index 116c7c7..4302b33 100644 --- a/astro-atproto-loader/src/pipeline/single.ts +++ b/astro-atproto-loader/src/pipeline/single.ts @@ -3,6 +3,7 @@ import { isRecordValue, toRecordContext, } from "../client/records.ts"; +import type { AtProtoCache } from "../cache/index.ts"; import type { ArgsUnion, AtProtoLoaderSource, @@ -29,9 +30,10 @@ export const runSingleFetch = async < source: AtProtoLoaderSource, callbacks: AtProtoRecordCallbacks, rkey: string, + caches: AtProtoCache, ): Promise => { - const fetchRecord = createFetchRecord(); - const data = await getSingleRecord(source, rkey); + const fetchRecord = createFetchRecord(caches); + const data = await getSingleRecord(source, rkey, caches); if (!isRecordValue(data.value)) { throw new Error( @@ -39,7 +41,7 @@ export const runSingleFetch = async < ); } - const context = await toRecordContext(source, data); + const context = await toRecordContext(source, data, caches); let value: unknown = data.value; if (source.parseRecord) { @@ -92,3 +94,76 @@ export const runSingleFetch = async < } return entry; }; + +/** A single-entry request, as resolved from a loader's entry filter. */ +export interface EntryLookup { + requestedId: string | undefined; + rkey: string | undefined; + repo: string | undefined; + collection: string | undefined; +} + +/** + * Try to resolve a single requested entry with direct `getRecord` calls, + * instead of waiting on the full collection refresh. + * + * This is the request-time fast path for `loadEntry`: entry ids are the cannot + * be inferred from the record (they're the output of the `transform` callback) + * which means fetching a SINGLE entry by id could require listing and + * transforming every record in every source. + * + * This function tries to avoid that by using the `rkey` and `repo`/`collection` + * to narrow down the sources that could hold the requested entry, and then + * calling `getRecord` on each candidate source. The first source that returns + * a record whose transformed id matches the requested id is returned. + * + * When the requested entry is not found, this function returns `undefined` and + * the caller should fall back to looking inside the cached collection. + * + */ +export const findEntryViaFetch = async < + Sources extends readonly AtProtoLoaderSource[], + Entry extends { id: string }, +>( + sources: readonly AtProtoLoaderSource[], + callbacks: AtProtoRecordCallbacks, + { requestedId, rkey, repo, collection }: EntryLookup, + caches: AtProtoCache, +): Promise => { + if (!rkey) { + return undefined; + } + + const candidates = sources.filter( + (source) => + (!repo || source.repo === repo) && + (!collection || source.collection === collection), + ); + + const matchesRequestedId = (entry: Entry) => + !requestedId || entry.id === requestedId; + + if (!candidates[0]) return undefined; + + if (candidates.length > 1) { + // `Awaited` is just `Entry` here (entries are never thenables), + // but TS can't reduce it for an unresolved type parameter. + const results = (await Promise.allSettled( + candidates.map((source) => + runSingleFetch(source, callbacks, rkey, caches), + ), + )) as PromiseSettledResult[]; + + const match = results.find( + (result): result is PromiseFulfilledResult => + result.status === "fulfilled" && + result.value !== undefined && + matchesRequestedId(result.value), + ); + + return match?.value; + } + + const entry = await runSingleFetch(candidates[0], callbacks, rkey, caches); + return entry && matchesRequestedId(entry) ? entry : undefined; +}; diff --git a/astro-atproto-loader/src/pipeline/source.ts b/astro-atproto-loader/src/pipeline/source.ts index 4edad74..5971349 100644 --- a/astro-atproto-loader/src/pipeline/source.ts +++ b/astro-atproto-loader/src/pipeline/source.ts @@ -3,6 +3,7 @@ import { listRecordsPage, toRecordContext, } from "../client/records.ts"; +import type { AtProtoCache } from "../cache/index.ts"; import type { ArgsUnion, AtProtoLoaderSource, @@ -72,6 +73,7 @@ export const fetchFromSource = async < source: AtProtoLoaderSource, callbacks: AtProtoRecordFilterOptions, fetchRecord: FetchRecord, + caches: AtProtoCache, ): Promise[]> => { const window = resolveRecordsFetchWindow(source); const collected: AtProtoRecordCallbackArgs[] = []; @@ -95,16 +97,17 @@ export const fetchFromSource = async < ? Math.min(remaining, window.pageSize) : window.pageSize; - const data = await listRecordsPage(source, { - limit: thisPageSize, - cursor, - }); + const data = await listRecordsPage( + source, + { limit: thisPageSize, cursor }, + caches, + ); pageCount++; for (const record of data.records) { if (!isRecordValue(record.value)) continue; - const context = await toRecordContext(source, record); + const context = await toRecordContext(source, record, caches); let value: unknown = record.value; if (source.parseRecord) { diff --git a/astro-atproto-loader/src/types.ts b/astro-atproto-loader/src/types.ts index 594ab9f..a33a459 100644 --- a/astro-atproto-loader/src/types.ts +++ b/astro-atproto-loader/src/types.ts @@ -49,15 +49,15 @@ export interface AtProtoLoaderSource { } /** - * Identifier for the repo this record lives in. + * The resolved identity of the repo this record lives in: * - * `did` is always set (resolved from the record's AT-URI). - * `pds` is the resolved Personal Data Server URL for that DID. Useful for - * building blob URLs (`com.atproto.sync.getBlob`) or hitting any other PDS - * endpoint without re-resolving identity. - * `handle` is set only when the source config provided a handle for this - * repo. The loader never resolves a DID back to its handle, so callers that - * passed `repo: "did:..."` will see `handle: undefined`. + * - `did` is the repo's DID + * - `pds` is the resolved Personal Data Server URL for that DID. Useful for + * building blob URLs (`com.atproto.sync.getBlob`) or hitting any other PDS + * endpoint without re-resolving identity. + * - `handle` is set only when the source config provided a handle for this + * repo. The loader never resolves a DID back to its handle, so callers that + * passed `repo: "did:..."` will see `handle: undefined`. */ export interface AtProtoRecordRepo { did: DidString; @@ -76,9 +76,13 @@ export interface AtProtoRecordContext { /** * Fetch a single record by AT-URI from any public PDS. * - * Concurrent callers for the same URI within one pipeline cycle share a - * single network hop. Calling `fetchRecord({ atUri })` from many `transform` - * or `filter` callbacks for the same target only hits the network once. + * Concurrent callers asking for the same URI share a single network request. + * Cached time depends on the outcome: + * - Successes stay cached for five minutes. + * - Transient failures (network errors, 5xx) retry after five seconds. + * - Definitive failures (e.g. record doesn't exist) are held for five minutes + * like successes. + * - The cache holds at most 20,000 records. * * Resolves to `{ value, repo }` on success. `repo` carries the fetched * record's owning DID and PDS, so callers can hand it straight to @@ -97,10 +101,10 @@ export interface AtProtoRecordContext { * Each failure mode logs a distinct warning, so callers can tell which thing * went wrong from the console. */ -export type FetchRecord = (args: { +export type FetchRecord = (args: { atUri: string; - parse?: (value: unknown) => Parsed; -}) => Promise<{ value: Parsed; repo: AtProtoRecordRepo } | null>; + parse?: (value: unknown) => ParsedValue; +}) => Promise<{ value: ParsedValue; repo: AtProtoRecordRepo } | null>; /** * The bundle of args passed to each `filter` and `transform` callback for a @@ -179,7 +183,12 @@ export interface AtProtoRecordGroupTransformArgs< key: string; /** All filtered records that returned this key, in source declaration order. */ records: ArgsUnion[]; - /** Shared per-cycle record hydrator, same as the per-record callback helper. */ + /** + * Cached record hydrator, shared with every per-record callback. It reads + * through the process-wide caches context, using the fixed five-minute + * success TTL, five-second failure retry floor, and 20,000-record bound + * documented by `FetchRecord`. + */ fetchRecord: FetchRecord; } @@ -205,21 +214,40 @@ export type AtProtoRecordCallbacks< } ); +/** + * The `groupBy`/`transform` pairing loader options accept: either an ungrouped + * (and optional) per-record `transform`, or a `groupBy` with its required + * grouped `transform`. `resolveRecordCallbacks` turns this into + * `AtProtoRecordCallbacks` by filling in the default ungrouped transform. + */ +export type AtProtoTransformOptions< + Sources extends readonly AtProtoLoaderSource[], + Entry, +> = + | { + groupBy?: never; + transform?: AtProtoRecordTransform; + } + | { + groupBy: AtProtoRecordGroupBy; + transform: AtProtoRecordGroupTransform; + }; + /** * What the pipeline should do when one source in a multi-source load fails. * * - `'skip'` warns and drops that source's contribution, letting the rest of * the load continue. - * - `'throw'` rethrows immediately so the static loader can fail the build, - * or so the live loader's stale-while-revalidate cache holds onto the last - * good snapshot until the next refresh. + * - `'throw'` rethrows, so the static loader fails the build and the live + * loader hands the error up to its collection-level handling. * - A function gets the error and source and returns one of the two, for * case-by-case decisions. * - * Once the pipeline starts skipping errors, if _every_ remaining source ends - * up failing it throws an `AggregateError` so the failure isn't swallowed - * silently. (When the policy is `'throw'`, the first error fails the load - * right away, so the aggregate path doesn't apply.) + * If every source ends up skipped, the pipeline throws an `AggregateError` + * rather than silently returning nothing. In the live loader, `'throw'` only + * applies while a source is still cold: once it has succeeded once, a failed + * refresh serves that source's cached records instead. See the README's + * "Multi-source reads and `onSourceError`" section for the full behavior. */ export type OnSourceError = | "throw" diff --git a/astro-atproto-loader/src/utils.ts b/astro-atproto-loader/src/utils.ts index fc58bbb..4ce41f1 100644 --- a/astro-atproto-loader/src/utils.ts +++ b/astro-atproto-loader/src/utils.ts @@ -1,8 +1,12 @@ -import { BlobRef } from "@atproto/api"; +import { BlobRef, ComAtprotoRepoGetRecord } from "@atproto/api"; import type { AtProtoLoaderSource, AtProtoRecordCallbackArgs, + AtProtoRecordCallbacks, + AtProtoRecordFilterOptions, + AtProtoRecordTransform, + AtProtoTransformOptions, } from "./types.ts"; export type AtProtoSourceOptions< @@ -82,6 +86,18 @@ export const toError = (error: unknown, message: string) => export const getErrorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); +/** + * A record fetch failure that retrying won't fix: the fetch itself worked, + * but what came back can never become a usable record (e.g. its value isn't + * an object). Caches hold these as long as successes instead of applying + * their short retry floor. + */ +export class DefinitiveRecordError extends Error {} + +export const isDefinitiveRecordFailure = (error: unknown): boolean => + error instanceof DefinitiveRecordError || + error instanceof ComAtprotoRepoGetRecord.RecordNotFoundError; + export const normalizeSources = < Sources extends readonly AtProtoLoaderSource[], >( @@ -133,3 +149,36 @@ export const toNamespacedEntry = >({ id: `${repo.did}/${collection}/${rkey}`, data: value as Data, }); + +/** + * Resolve a loader's `filter`/`groupBy`/`transform` options into the + * `AtProtoRecordCallbacks` used by the pipeline. + */ +export const resolveRecordCallbacks = < + Sources extends readonly AtProtoLoaderSource[], + Data extends Record, + Entry extends { id: string; data: Data }, +>( + sources: readonly AtProtoLoaderSource[], + options: AtProtoRecordFilterOptions & + AtProtoTransformOptions, +): AtProtoRecordCallbacks => { + if (options.groupBy) { + return { + filter: options.filter, + groupBy: options.groupBy, + transform: options.transform, + }; + } + + // The fallback only runs when no custom `transform` was provided, which + // means `Entry` is just `{ id: string; data: Data }`. TS can't narrow the + // generic per branch, hence the cast. + const fallbackTransform = (sources.length > 1 + ? toNamespacedEntry + : toRkeyEntry) as unknown as AtProtoRecordTransform; + return { + filter: options.filter, + transform: options.transform ?? fallbackTransform, + }; +}; diff --git a/package-lock.json b/package-lock.json index 609c9ce..bb6922c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -466,7 +466,7 @@ "@atproto/syntax": "^0.4.2" }, "devDependencies": { - "@atproto/crypto": "^0.4.5", + "@fujocoded/msw-atproto": "^0.0.1", "astro": "^5.16.14", "msw": "^2.13.4", "tsdown": "^0.14.2",