fix(foreign): keep missing elements and raw ids of a multi reference

.agents/skills/verify-interface/features/README.md

@@ -11,7 +11,7 @@ One file per feature. `covers:` lists every source file that computes something
 | [access](access.md) | `@Access` decides what each route reads and writes. | access-modes, listable-alone, read-overrides |
 | [validation](validation.md) | `@Mandatory`, `@Validator` and `@Optional` check the body. | mandatory-and-validators, parsed-values, error-codes, empty-forms, mandatory-before-validators, holey-body |
 | [listing](listing.md) | A client pages, sorts and narrows a list. | pluck-modes, pagination, sorting, issue-92-getter-mode |
-| [foreign](foreign.md) | A key field answers the record it points to. | foreign-resolve, foreign-raw, issue-93-multi-foreign, no-foreign-strings, foreign-declaration-throws, foreign-unplucked |
+| [foreign](foreign.md) | A key field answers the record it points to. | foreign-resolve, foreign-raw, foreign-multi, no-foreign-strings, foreign-declaration-throws, foreign-unplucked |
 | [filters](filters.md) | A list request filters rows. | default-filter, custom-filters, filter-details, route-filters-list-only |
 | [modifiers](modifiers.md) | A per-request key selects a modifier's value. | localized-key |
 | [joined](joined.md) | A remote field is flattened onto the row. | joined-read, joined-access |
@@ -22,7 +22,7 @@ One file per feature. `covers:` lists every source file that computes something
 
 ## Known issues, driven on purpose
 
-Two journeys fail on head and cite their issue: `issue-90-edit-index` ([#90](https://github.com/AntelopeJS/interface-data-api/issues/90)) and `issue-93-multi-foreign` ([#93](https://github.com/AntelopeJS/interface-data-api/issues/93)). A verdict that runs their features is FAIL until the issue is fixed; the fix turns the journey green.
+One journey fails on head and cites its issue: `issue-90-edit-index` ([#90](https://github.com/AntelopeJS/interface-data-api/issues/90)). A verdict that runs its feature is FAIL until the issue is fixed; the fix turns the journey green.
 
 ## Not driven
 

.agents/skills/verify-interface/features/foreign.md

@@ -22,7 +22,7 @@ docs:
 journeys:
   - foreign-resolve
   - foreign-raw
-  - issue-93-multi-foreign
+  - foreign-multi
   - no-foreign-strings
   - foreign-declaration-throws
   - foreign-unplucked
@@ -36,7 +36,7 @@ A field that holds a key of another table answers the record it points to.
 
 - `foreign-resolve`: `@Foreign` answers the referenced record in place of the key: with `pluck`, through a secondary `index`, by table name, across schemas, and as an array (`multi`); a key that matches nothing answers `null`.
 - `foreign-raw`: the `noForeign` route option answers the raw keys of single references on a get and on a list.
-- `issue-93-multi-foreign`: fails on purpose until [#93](https://github.com/AntelopeJS/interface-data-api/issues/93) is fixed: a multi reference answers `null` for a missing element and raw ids with `noForeign`.
+- `foreign-multi`: a multi reference answers `null` for a missing element and its raw ids with `noForeign`.
 - `no-foreign-strings`: the Get `noForeign` option skips resolving for any non-empty string, `"false"` included, and not for `""` or when absent; the List option is a boolean.
 - `foreign-declaration-throws`: `@Foreign` throws at declaration for a schema mismatch between the class and the `schema` parameter, and for a string table name with no class registered in that schema.
 - `foreign-unplucked`: a foreign key without `pluck` answers the whole target row, including fields the target controller's `@Access` hides.
@@ -52,7 +52,3 @@ pnpm exec antelopejs-verify journey "$RUN" foreign
 ```
 
 Each journey compiles the consumer in `fixture/shop-1.ts.txt` and `fixture/shop-2.ts.txt`, joined in that order, starts `ajs project dev` with `@antelopejs/api` and `@antelopejs/mongodb` against a Docker MongoDB, drives the routes with curl and reads the writes back with `mongosh`.
-
-## Gotchas
-
-- A missing element of a multi reference answers `{}` and `noForeign` spreads its ids into character-index objects: [#93](https://github.com/AntelopeJS/interface-data-api/issues/93), driven by `issue-93-multi-foreign`.

.agents/skills/verify-interface/journeys-probe-1.ts

@@ -144,14 +144,14 @@ export const issue92 = shopJourney("#92: a getter that a list mode does not decl
   context.expect(brief.folder, [status(brief, 200), equal("#92: the brief mode lists only _id", keysOf(results(brief)[0] ?? null), ["_id"])]);
 });
 
-export const issue93 = shopJourney("fails on purpose until #93 is fixed: a multi reference answers null for a missing element and raw ids with noForeign", async (shop) => {
+export const foreignMulti = shopJourney("a multi reference answers null for a missing element and its raw ids with noForeign", async (shop) => {
   const { context } = shop;
   await seed(shop, `${ORDERS}\nd.verify__orders.insertOne({ _id: "o3", items: [], itemsAll: ["i1", "ghost"], labels: [], total: 1 });`);
   const resolved = await shop.request("an order with a missing element", "GET", "/orders/get?id=o3");
   const items = pick(resolved.json, "itemsAll");
-  context.expect(resolved.folder, [status(resolved, 200), equal("#93: the missing element is null", Array.isArray(items) ? items[1] : "not an array", null)]);
+  context.expect(resolved.folder, [status(resolved, 200), equal("the missing element is null", Array.isArray(items) ? items[1] : "not an array", null)]);
   const raw = await shop.request("the same order with noForeign", "GET", "/orders/raw?id=o3");
-  context.expect(raw.folder, [equal("#93: noForeign answers the raw ids of a multi reference", pick(raw.json, "itemsAll"), ["i1", "ghost"])]);
+  context.expect(raw.folder, [equal("noForeign answers the raw ids of a multi reference", pick(raw.json, "itemsAll"), ["i1", "ghost"])]);
 });
 
 export const noForeignStrings = shopJourney("the Get noForeign option skips resolving for any non-empty string, false included, and the List one is a boolean", async (shop) => {

.agents/skills/verify-interface/journeys-probe.ts

@@ -1,4 +1,4 @@
-import { guardSemantics, newRowContents, editSetters, filterDetails, listUnknown, listableAlone, editAnswersForeign, readOverrides, issue90, issue92, issue93, noForeignStrings } from "./journeys-probe-1.ts";
+import { guardSemantics, newRowContents, editSetters, filterDetails, listUnknown, listableAlone, editAnswersForeign, readOverrides, issue90, issue92, foreignMulti, noForeignStrings } from "./journeys-probe-1.ts";
 import { noMandatoryStrings, noPluckStrings, derivedOwnModel, foreignTargetMeta, noModelAnswers500, optionBeatsQuery, foreignDeclarationThrows, routeFiltersListOnly, emptyForms, mandatoryBeforeValidators, holeyBody, joinedAccess, computedAccess, foreignUnplucked } from "./journeys-probe-2.ts";
 
 export const PROBE_JOURNEYS = {
@@ -27,5 +27,5 @@ export const PROBE_JOURNEYS = {
   "foreign-unplucked": foreignUnplucked,
   "issue-90-edit-index": issue90,
   "issue-92-getter-mode": issue92,
-  "issue-93-multi-foreign": issue93,
+  "foreign-multi": foreignMulti,
 };

src/internal/validation.ts

@@ -67,7 +67,9 @@ export function Unlock(
     if (field.foreign && isObject(dbData[name]) && field.foreign[1]) {
       const foreignSchema = field.foreign[1];
       dbData[name] = Array.isArray(dbData[name])
-        ? dbData[name].map((entry) => fromDatabase(entry, foreignSchema))
+        ? dbData[name].map<unknown>((entry) =>
+            isObject(entry) ? fromDatabase(entry, foreignSchema) : entry,
+          )
         : fromDatabase(dbData[name], field.foreign[1]);
     }
   }
@@ -144,7 +146,9 @@ export function ClearInternal(
           return plainPlucked;
         };
         entry[name] = Array.isArray(entry[name])
-          ? entry[name].map(processForeign)
+          ? entry[name].map<unknown>((item) =>
+              isObject(item) ? processForeign(item) : item,
+            )
           : processForeign(entry[name]);
       }
     }

src/tests/components/foreign_multi.test.ts

@@ -0,0 +1,169 @@
+import path from "node:path";
+import { expect } from "chai";
+import { z } from "zod";
+import { Controller } from "@antelopejs/interface-api";
+import { Schema } from "@antelopejs/interface-database";
+import {
+  DataController,
+  DefaultRoutes,
+  RegisterDataController,
+} from "../../index";
+import {
+  Access,
+  AccessMode,
+  Foreign,
+  Listable,
+  ModelReference,
+} from "../../metadata";
+import {
+  BasicDataModel,
+  Field,
+  Model,
+  RegisterSchema,
+  RegisterTable,
+  Table,
+} from "@antelopejs/interface-database-decorators";
+
+import { listResponseSchema, readJson } from "../internal/http";
+import { getFunctionName, getSchemaInstance, request } from "../utils";
+
+const currentTestName = path
+  .basename(__filename)
+  .replace(/\.test\.(ts|js)$/, "");
+const itemTableName = `items-${currentTestName}`;
+const orderTableName = `orders-${currentTestName}`;
+const schemaName = "default";
+
+@RegisterTable(itemTableName, schemaName)
+class Item extends Table {
+  declare _id: string;
+
+  @Field("string")
+  declare name: string;
+}
+class ItemModel extends BasicDataModel(Item, itemTableName) {}
+
+@RegisterTable(orderTableName, schemaName)
+class Order extends Table {
+  declare _id: string;
+
+  @Field("array")
+  declare items: (string | null)[];
+
+  @Field("array")
+  declare itemsAll: (string | null)[];
+}
+class OrderModel extends BasicDataModel(Order, orderTableName) {}
+
+const orderSchema = z.looseObject({
+  _id: z.string(),
+  items: z.array(z.unknown()),
+  itemsAll: z.array(z.unknown()),
+});
+
+describe("Foreign multi references", () => {
+  it("gets null for a missing element", async () =>
+    await expectGetAnswersNullForMissing());
+  it("gets the raw ids with noForeign", async () =>
+    await expectGetAnswersRawIds());
+  it("lists the raw ids with noForeign", async () =>
+    await expectListAnswersRawIds());
+});
+
+async function _createDataController(testName: string) {
+  @RegisterDataController()
+  class _ForeignMultiAPI extends DataController(
+    Order,
+    {
+      get: DefaultRoutes.Get,
+      raw: DefaultRoutes.WithOptions(DefaultRoutes.Get, { noForeign: "true" }),
+      rawList: DefaultRoutes.WithOptions(DefaultRoutes.List, {
+        noForeign: true,
+      }),
+    },
+    Controller(`/${testName}`),
+  ) {
+    @ModelReference()
+    @Model(OrderModel)
+    declare orderModel: OrderModel;
+
+    @Listable()
+    @Access(AccessMode.ReadOnly)
+    declare _id: string;
+
+    @Listable()
+    @Foreign(Item, undefined, true, ["_id", "name"])
+    @Access(AccessMode.ReadOnly)
+    declare items: Item[];
+
+    @Listable()
+    @Foreign(Item, undefined, true)
+    @Access(AccessMode.ReadOnly)
+    declare itemsAll: Item[];
+  }
+
+  await RegisterSchema(schemaName);
+  const schema = Schema.get(schemaName);
+  if (schema) {
+    await schema.instance().table(orderTableName).delete();
+    await schema.instance().table(itemTableName).delete();
+  }
+
+  const schemaInstance = getSchemaInstance(schemaName);
+  const [itemId] = Object.values(
+    await new ItemModel(schemaInstance).insert([{ name: "Widget" }]),
+  );
+  const ids = [itemId, "ghost"];
+  const nullFirst = [null, itemId];
+  const [orderId, nullFirstId] = Object.values(
+    await new OrderModel(schemaInstance).insert([
+      { items: ids, itemsAll: ids },
+      { items: nullFirst, itemsAll: nullFirst },
+    ]),
+  );
+  return { itemId, orderId, nullFirstId };
+}
+
+async function _getOrder(testName: string, route: string, id: string) {
+  const response = await request(testName, route, "GET", undefined, { id });
+  expect(response.status, await response.clone().text()).to.equal(200);
+  return await readJson(orderSchema, response);
+}
+
+async function expectGetAnswersNullForMissing() {
+  const { itemId, orderId } = await _createDataController(getFunctionName());
+
+  const order = await _getOrder(getFunctionName(), "get", orderId);
+  expect(order.items).to.deep.equal([{ _id: itemId, name: "Widget" }, null]);
+  expect(order.itemsAll).to.have.length(2);
+  expect(order.itemsAll[0]).to.include({ _id: itemId, name: "Widget" });
+  expect(order.itemsAll[1]).to.equal(null);
+}
+
+async function expectGetAnswersRawIds() {
+  const { itemId, orderId, nullFirstId } =
+    await _createDataController(getFunctionName());
+
+  const order = await _getOrder(getFunctionName(), "raw", orderId);
+  expect(order.items).to.deep.equal([itemId, "ghost"]);
+  expect(order.itemsAll).to.deep.equal([itemId, "ghost"]);
+  const nullFirst = await _getOrder(getFunctionName(), "raw", nullFirstId);
+  expect(nullFirst.items).to.deep.equal([null, itemId]);
+  expect(nullFirst.itemsAll).to.deep.equal([null, itemId]);
+}
+
+async function expectListAnswersRawIds() {
+  const { itemId } = await _createDataController(getFunctionName());
+
+  const response = await request(getFunctionName(), "rawList", "GET");
+  expect(response.status, await response.clone().text()).to.equal(200);
+  const data = await readJson(listResponseSchema(orderSchema), response);
+  expect(data.results.map((o) => o.itemsAll)).to.have.deep.members([
+    [itemId, "ghost"],
+    [null, itemId],
+  ]);
+  expect(data.results.map((o) => o.items)).to.have.deep.members([
+    [itemId, "ghost"],
+    [null, itemId],
+  ]);
+}