Activity log GraphQL API

CONTRIBUTING.md

@@ -362,6 +362,20 @@ When adding a new object or field, follow the existing `builder.drizzleNode()`
 and `t.drizzleField()` patterns.  Keep resolver database access through
 `ctx.db`.
 
+Activity delivery observations live in `activity_logs`, independently of the
+ActivityPub `activities` resources.  The federation HTTP surface must pass
+through `createInboundRecorder` with a federation made by `createFederation`,
+which tracks the public keys, spans and measurements Fedify reports for each
+request, and the deployment's root origin.  Keep the public-key cache
+serialization compatible with the installed Fedify version, and read only the
+spans, events and metrics Fedify documents in its OpenTelemetry manual; never
+verify a request again.  Inbox listeners
+must call `markHandled()`, which is how a log tells a received activity from an
+acknowledged one.  Use `deliverActivity` for outgoing delivery, and create the
+federation through `createFederation`, which observes the outbox queue so that
+each attempt Fedify's worker makes settles its log.  Log contents are private
+to local instance members and administrators, including Relay node lookups.
+
 
 CLI and server changes
 ----------------------

mise.toml

@@ -98,7 +98,7 @@ node scripts/add-license/main.mts
 
 [tasks.test]
 description = "Run tests"
-depends = ["build"]
+depends = ["build:server"]
 usage = '''
 flag "-d --debug" help="Run tests in debug mode"
 '''

packages/drfed/src/index.ts

@@ -18,7 +18,9 @@ import { writeFile } from "node:fs/promises";
 import process from "node:process";
 
 import { createYogaServer } from "@drfed/graphql";
-import createFederation from "@drfed/graphql/federation";
+import createFederation, {
+  createInboundRecorder,
+} from "@drfed/graphql/federation";
 import { schema } from "@drfed/graphql/schema";
 import { migrate } from "@drfed/models";
 import { PgliteKvStore } from "@fedify/pglite";
@@ -48,7 +50,10 @@ async function runServer(options: ServerOptions) {
     "driver" in credentials
       ? new PgliteKvStore(credentials.client)
       : new PostgresKvStore(credentials.client);
-  const federation = await createFederation(options.drizzle.db, { kv });
+  const federation = await createFederation(options.drizzle.db, {
+    kv,
+    allowPrivateAddress: true,
+  });
   const { emailFrom, mailer, rootOrigin, loginOrigins } = options;
 
   const yogaServer = createYogaServer(options.drizzle.db, federation, {
@@ -60,7 +65,11 @@ async function runServer(options: ServerOptions) {
   await warnAboutStrandedInstances(options.drizzle.db, rootOrigin);
   const server = serve({
     fetch: createFetchHandler({
-      federation,
+      federation: createInboundRecorder({
+        db: options.drizzle.db,
+        federation,
+        rootOrigin,
+      }),
       rootOrigin,
       serveControlSurface: yogaServer.fetch,
     }),

packages/drfed/src/serving.test.ts

@@ -21,9 +21,13 @@ import {
   findStrandedInstances,
   warnAboutStrandedInstances,
 } from "@drfed/drfed/serving";
+import createFederation, {
+  createInboundRecorder,
+} from "@drfed/graphql/federation";
 import { migrate, relations, schema } from "@drfed/models";
 import { uuidV7 as uuid } from "@drfed/models/uuid";
 import { PGlite } from "@electric-sql/pglite";
+import { MemoryKvStore } from "@fedify/fedify";
 import { describe, it } from "@logtape/testing-node/autoload";
 import { drizzle } from "drizzle-orm/pglite";
 
@@ -241,3 +245,54 @@ describe("findStrandedInstances()", () => {
     }
   });
 });
+
+it("records inbox requests only on the instance surface", async () => {
+  const client = new PGlite();
+  try {
+    await migrate({ credentials: { driver: "pglite", client } });
+    const db = drizzle({ client, relations, schema });
+    const localId = uuid();
+    await db.insert(schema.localInstances).values({
+      id: localId,
+      slug: "logs",
+      expires: Temporal.Now.instant().add({ hours: 24 }),
+    });
+    await db
+      .insert(schema.instances)
+      .values({ id: localId, localId, host: "logs.drfed.net" });
+    const kv = new MemoryKvStore();
+    const federation = await createFederation(db, { kv });
+    const fetch = createFetchHandler({
+      rootOrigin,
+      federation: createInboundRecorder({ db, federation, rootOrigin }),
+      serveControlSurface: () => new Response("control"),
+    });
+    const body = JSON.stringify({
+      "@context": "https://www.w3.org/ns/activitystreams",
+      type: "Create",
+      actor: "https://remote.example/alice",
+    });
+    assert.equal(
+      (
+        await fetch(
+          new Request("https://logs.drfed.net/inbox", { method: "POST", body }),
+        )
+      ).status,
+      401,
+    );
+    assert.equal(
+      (
+        await fetch(
+          new Request("https://drfed.net/inbox", { method: "POST", body }),
+        )
+      ).status,
+      200,
+    );
+    const rows = await db.query.activityLogs.findMany();
+    assert.equal(rows.length, 1);
+    assert.equal(rows[0]?.instanceId, localId);
+    assert.equal(rows[0]?.status, "unverified");
+  } finally {
+    await client.close();
+  }
+});

packages/drfed/src/serving.ts

@@ -19,8 +19,8 @@ import type { Database } from "@drfed/models";
 import { getLogger } from "@logtape/logtape";
 
 /**
- * The part of a Fedify `Federation` that the router needs, narrowed so that
- * the routing can be exercised without building one.
+ * The federation surface (including the inbound recorder) that the router
+ * needs, narrowed so that the routing can be exercised without building one.
  */
 export interface FederationHandler {
   fetch(
@@ -43,7 +43,7 @@ export interface FetchHandlerOptions {
   readonly rootOrigin: URL;
 
   /**
-   * The ActivityPub surface, served on instance subdomains.
+   * The ActivityPub surface with inbox recording, served on instance subdomains.
    */
   readonly federation: FederationHandler;
 

packages/graphql/README.md

@@ -24,15 +24,16 @@ Usage
 
 ~~~~ ts
 import { createYogaServer } from "@drfed/graphql";
-import createFederation from "@drfed/graphql/federation";
+import createFederation, { createInboundRecorder } from "@drfed/graphql/federation";
 
 const federation = await createFederation(db, { kv });
+const recordedInbox = createInboundRecorder({ db, federation, rootOrigin });
 const yoga = createYogaServer(db, federation, {
   loginOrigins: new Set(["https://drfed.example.com"]),
 });
 serve({
   fetch: (request) =>
-    federation.fetch(request, { onNotFound: yoga.fetch, contextData: undefined }),
+    recordedInbox.fetch(request, { onNotFound: yoga.fetch, contextData: undefined }),
 });
 ~~~~
 
@@ -41,3 +42,106 @@ registered.  `createYogaServer` accepts a Drizzle database instance, that
 federation, and server options, and returns a GraphQL Yoga server
 ready to handle HTTP requests.  The federation is stored in the resolver
 context as is; `createYogaServer` never registers anything on it.
+
+
+Activity logs
+-------------
+
+`Instance.activityLogs` and `Actor.activityLogs` expose delivery observations,
+newest first, with direction, status, and type filters.  Only accepted local
+instance members and site administrators can read them, including through
+Relay node IDs.  `Actor.activityLogs` lists the deliveries that arrived at the
+actor's inbox, that the actor sent, and that are addressed to the actor through
+any inbox, the shared one included.  Addressing through a collection counts as
+far as DrFed has stored the collection's members.
+
+Wrap the federation HTTP surface with `createInboundRecorder`, passing a
+federation made by `createFederation` and the root origin.  Every inbox `POST`
+is logged, whether or not its body is JSON; logging errors never replace
+federation responses.  A request Fedify throws on is logged too, with the
+exception in `error` and no `statusCode`, and the exception is thrown again.
+An inbound log keeps:
+
+ -  The request as received: `rawBody` (or `rawBodyBase64` when the body is not
+    valid UTF-8), `requestHeaders`, and `requestUrl`.  `payload` is the body
+    parsed as JSON for querying, and is `null` when it does not parse or
+    holds U+0000 or an unpaired surrogate, which PostgreSQL cannot store.
+    `cookie` headers are dropped, and `authorization` is kept only for the
+    `Signature` scheme.
+ -  `inboxUrl`, the canonical IRI of the inbox, however the request spelled its
+    host or scheme.
+ -  `verificationMechanism` and `verificationResult`, what Fedify reported of
+    verifying the request, as its OpenTelemetry manual documents: the
+    `activitypub.signature.verification.duration` result of each mechanism it
+    tried, the `activitypub.signature.key_fetch.duration` and
+    `activitypub.key.lookup` results of the keys it fetched for each, the
+    `*.verify` spans naming their keys, and the
+    `activitypub.activity.received` event.  DrFed verifies nothing again.  The
+    result is `unattempted` when Fedify answered before verifying, such as for
+    a body that is not JSON or an inbox whose owner does not exist, even if
+    the body carries a signature or proof.  Proofs are found as JSON-LD,
+    however `proof` is spelled.
+ -  A refusal is `key_fetch_error` rather than `invalid_signature` when the
+    last key fetch of the mechanism brought no usable key, whichever
+    mechanism it is: the signature was then not checked, or only against a
+    cached key that did not verify.  `error` tells why as `keyFetchError:`
+    followed by the status the server of the key answered with, by `cached`
+    for the record of an earlier failure, or by the lookup result Fedify
+    counted, such as `network_error` or `invalid` for a document that holds no
+    key.  When Fedify itself names the fetch of an HTTP signature's key as
+    failed, what follows is the status or the type of the error it reports.
+ -  The result tells whether a signature or proof verified, not whether it
+    authenticated the activity.  Object Integrity Proofs that verify without
+    their keys' controllers covering the activity's actor are `verified`, and
+    the refusal is `status` `rejected`, with `error` telling that Fedify did
+    not accept them, even when the HTTP signature Fedify went on to failed.
+    Fedify reports a Linked Data Signature that verifies without its key's
+    owner being the actor the same as one that does not verify, so that one
+    is `invalid_signature`.
+ -  `verificationKey`, the public key version that mechanism used, even when
+    verification failed: the last key its cache entry held during a key fetch
+    of that one verification which brought a key, whether read or fetched,
+    even when fetching it again then failed.  A key another mechanism of the
+    same request found under the same IRI is not it, nor is one read from the
+    cache that the verification could not use.  `signedKeyIri` is the `keyId`
+    the HTTP signature declares.  The two may name different keys.
+ -  `status`, which follows the response Fedify gave: `received` when the inbox
+    listener ran, `acknowledged` when the request was answered 2xx without it
+    (a duplicate, for instance), `rejected` when a verified activity was
+    refused or its handling threw, and `unverified` otherwise.  With an inbox
+    queue, a log is
+    `acknowledged` when the request is answered and becomes `received` once the
+    queue worker runs the listener; the queued message carries its log ID.
+
+`KeyVersion.firstSeen` and `lastSeen` are DrFed observation times, not remote
+rotation times or evidence of continuous use.  `Key` and `KeyVersion` hold
+public material only and are readable by any authenticated viewer.
+
+`deliverActivity` is the outbound entry point for explicit recipients once
+local actor keys are available (#87).  It removes `bto` and `bcc` before
+delivery, records one row per destination inbox with every recipient sharing
+it in `recipientIris`, and settles each delivery independently.  A recipient
+without an ID or an inbox is left out, because Fedify does not deliver to it.
+The logged `payload` is the document before signing.  `attempts` keeps every
+attempt that ended, with the status the remote inbox answered, read from the
+responses `fetch()` publishes on `diagnostics_channel`, and the causes of
+network errors.  When the inbox redirects a delivery, the status is the one
+the redirects ended with, whether Fedify followed them, as it does when it
+signs the request, or `fetch()` did.  With a message queue, `createFederation`
+observes Fedify's outbox worker, and each queued message carries the log it
+belongs to.  A delivery becomes `sent` when Fedify reports
+`activitypub.activity.sent` for its inbox, stays `failed` while it is retried,
+and ends `permanently_failed`, or `abandoned` when Fedify measures it abandoned
+after its retries ran out.  A delivery Fedify returns from without sending to
+the inbox, or without enqueuing a message to it, is `permanently_failed` with
+no attempt, since Fedify makes none.  The queue is handed to Fedify as one
+without native retries, so that every retry follows Fedify's policy and is
+logged.  Activity resource persistence (#88),
+retention policies, and the activity-log UI (#13) are separate.
+
+URL fields hold only values that parse as URLs, and no text field holds
+U+0000; anything else a remote server sent stays in `rawBody` and
+`requestHeaders`, and in `payload` when PostgreSQL can store it.  `error` and
+`responseBody` keep what a remote server answered with U+FFFD for each U+0000.
+Logs are ordered by `created`, which for an inbound log is when the request
+arrived; `completed` is when DrFed answered it.

packages/graphql/package.json

@@ -81,6 +81,10 @@
     "./origin": {
       "types": "./dist/origin.d.mts",
       "default": "./dist/origin.mjs"
+    },
+    "./activity-log": {
+      "types": "./dist/activity-log/entry.d.mts",
+      "default": "./dist/activity-log/entry.mjs"
     }
   },
   "files": [
@@ -98,7 +102,8 @@
       "src/instance.ts",
       "src/schema.ts",
       "src/object.ts",
-      "src/origin.ts"
+      "src/origin.ts",
+      "src/activity-log/entry.ts"
     ],
     "dts": {
       "sourcemap": true,
@@ -125,6 +130,7 @@
     "@fedify/vocab": "catalog:",
     "@logtape/graphql-yoga": "catalog:",
     "@logtape/logtape": "catalog:",
+    "@opentelemetry/api": "^1.9.1",
     "@pothos/core": "^4.13.0",
     "@pothos/plugin-drizzle": "^0.17.4",
     "@pothos/plugin-errors": "^4.9.1",