feat(anki): send LWT reviews to Anki, tested against real Anki (#264)

.gitignore

@@ -74,6 +74,7 @@ coverage.xml
 /cypress/screenshots/
 /cypress/videos/
 /cypress/downloads/
+/cypress/anki-exchange/
 /cypress/coverage/
 
 # Lighthouse reports

CHANGELOG.md

@@ -28,13 +28,17 @@ ones are marked like "v1.0.0-fork".
   complete and machine-translated, with the same invitation: please open
   an issue for any string that reads wrong.
 
-* **Pull reviews from Anki without a file** (#264). A new page,
-  `/vocabulary/anki/sync`, linked from the `.apkg` import page, reads the reviews you did in Anki straight from Anki on your computer,
-  through the AnkiConnect add-on, and merges them into LWT's schedule. Anki
-  asks once whether to trust the page; nothing in Anki is changed. Pulling
-  again is safe, since reviews LWT already has are recognised and skipped.
-  Only cards LWT exported are matched. On a hosted LWT the browser may ask
-  for local network access to reach Anki.
+* **Sync reviews with Anki without a file** (#264). A new page,
+  `/vocabulary/anki/sync`, linked from the `.apkg` import page, talks to Anki
+  on your computer through the AnkiConnect add-on. It brings the reviews you
+  did in Anki into LWT's schedule, then sends the reviews you did in LWT to
+  Anki and gives the cards LWT's due dates. Anki asks once whether to trust
+  the page. Syncing again is safe: reviews either side already has are
+  skipped. Only cards LWT exported are matched, and only reviews and schedules
+  are synced, not translations or tags. With FSRS on in Anki, Anki's own
+  memory estimates catch up when you next optimise, since AnkiConnect cannot
+  write them. On a hosted LWT the browser may ask for local network access to
+  reach Anki.
 
 * **Language-aware plural forms** (#319). Counted messages now take the
   form the interface language needs, not just English's singular and

cypress/e2e/24-anki-live.cy.ts

@@ -0,0 +1,215 @@
+/// <reference types="cypress" />
+
+/**
+ * Pulling reviews from a real Anki (#264), not a stub.
+ *
+ * Needs Anki with AnkiConnect listening on 127.0.0.1:8765 and sharing
+ * cypress/anki-exchange with this machine as /exchange. The repo builds one:
+ *
+ *   npm run anki:build && npm run anki:up     (podman; see scripts/anki/container)
+ *
+ * Without it every test here is skipped, so CI and a plain checkout are
+ * unaffected. With it, this is the loop a learner runs: export from LWT, study
+ * in Anki, pull the reviews back on /vocabulary/anki/sync.
+ */
+
+const apiBase = '/api/v1';
+const anki = 'http://127.0.0.1:8765';
+
+function unwrap<T>(body: unknown): T {
+  const wrapped = body as { data?: T };
+  return (wrapped && typeof wrapped === 'object' && 'data' in wrapped ? wrapped.data : body) as T;
+}
+
+/** Call AnkiConnect directly, failing the test on an AnkiConnect error. */
+function ankiConnect<T>(action: string, params: Record<string, unknown> = {}): Cypress.Chainable<T> {
+  return cy
+    .request({ method: 'POST', url: anki, body: { action, version: 6, params } })
+    .then((response) => {
+      const body = response.body as { result: T; error: string | null };
+      expect(body.error, `AnkiConnect ${action}`).to.eq(null);
+      return body.result;
+    });
+}
+
+/** How many of these terms are due by the end of tomorrow, as the review page counts them. */
+function dueByTomorrow(termIds: number[]): Cypress.Chainable<number> {
+  return cy
+    .request(`${apiBase}/review/tomorrow-count?review_key=words&selection=${termIds.join(',')}`)
+    .then((response) => unwrap<{ count: number }>(response.body).count);
+}
+
+describe('Sync reviews with a real Anki', () => {
+  let ankiUp = false;
+  let languageId = 0;
+  const termIds: number[] = [];
+  let noteIds: number[] = [];
+  let deckName = '';
+  let cardIds: number[] = [];
+
+  before(() => {
+    cy.request({ method: 'POST', url: anki, body: { action: 'version', version: 6 }, failOnStatusCode: false, timeout: 3000 })
+      .then((response) => {
+        ankiUp = response.status === 200 && (response.body as { result?: number }).result === 6;
+      }, () => {
+        ankiUp = false;
+      });
+  });
+
+  beforeEach(function () {
+    if (!ankiUp) {
+      cy.log('AnkiConnect not reachable on 127.0.0.1:8765; run npm run anki:up');
+      this.skip();
+    }
+  });
+
+  after(() => {
+    if (!ankiUp) return;
+    if (noteIds.length > 0) {
+      ankiConnect('deleteNotes', { notes: noteIds });
+    }
+    if (deckName !== '') {
+      ankiConnect('deleteDecks', { decks: [deckName], cardsToo: true });
+    }
+    termIds.forEach((id) => {
+      cy.apiRequest({ method: 'DELETE', url: `${apiBase}/terms/${id}`, failOnStatusCode: false });
+    });
+    if (languageId > 0) {
+      cy.apiRequest({ method: 'DELETE', url: `${apiBase}/languages/${languageId}`, failOnStatusCode: false });
+    }
+  });
+
+  it('brings Anki answers into LWT and sends LWT answers to Anki, once each way', () => {
+    const name = `E2E Anki ${Date.now()}`;
+    const file = 'lwt-e2e.apkg';
+
+    // 1. Terms in LWT, exported as an Anki package into the shared folder.
+    cy.apiRequest({ method: 'POST', url: `${apiBase}/languages`, body: { name } }).then((response) => {
+      languageId = unwrap<{ id: number }>(response.body).id;
+      ['uno', 'dos', 'tres'].forEach((text) => {
+        cy.apiRequest({
+          method: 'POST',
+          url: `${apiBase}/terms/for-language`,
+          body: { language_id: languageId, text, translation: `${text} meaning`, status: 1 },
+        }).then((termResponse) => {
+          termIds.push(unwrap<{ id: number }>(termResponse.body).id);
+        });
+      });
+    });
+    cy.then(() =>
+      cy.request({ url: `/vocabulary/apkg/export?lang_id=${languageId}`, encoding: 'binary' })
+        .then((response) => cy.writeFile(`cypress/anki-exchange/${file}`, response.body, 'binary'))
+    );
+
+    // 2. Imported into Anki, which now holds one LWT note per term.
+    cy.then(() => ankiConnect('importPackage', { path: `/exchange/${file}` }));
+    cy.then(() =>
+      ankiConnect<number[]>('findNotes', { query: '"note:LWT Term"' }).then((all) =>
+        ankiConnect<{ noteId: number; cards: number[]; fields: Record<string, { value: string }> }[]>(
+          'notesInfo',
+          { notes: all }
+        ).then((notes) => {
+          const ours = notes.filter((n) => termIds.includes(Number(n.fields.LwtId?.value)));
+          expect(ours, 'one Anki note per exported term').to.have.length(3);
+          noteIds = ours.map((n) => n.noteId);
+          // In term order, so the grades below land on the terms named later.
+          ours.sort((a, b) => termIds.indexOf(Number(a.fields.LwtId.value)) - termIds.indexOf(Number(b.fields.LwtId.value)));
+          const cards = ours.map((n) => n.cards[0]);
+          cardIds = cards;
+          ankiConnect<{ deckName: string }[]>('cardsInfo', { cards }).then((info) => {
+            deckName = info[0].deckName;
+          });
+
+          // 3. Studied in Anki: Good, Again, Easy.
+          ankiConnect<boolean[]>('answerCards', {
+            answers: cards.map((cardId, i) => ({ cardId, ease: [3, 1, 4][i] })),
+          }).then((answered) => expect(answered).to.deep.equal([true, true, true]));
+        })
+      )
+    );
+
+    // Before the pull, all three new terms fall due by tomorrow.
+    cy.then(() => dueByTomorrow(termIds).then((count) => expect(count, 'due before the pull').to.eq(3)));
+
+    // 4. Synced through the page.
+    cy.intercept('POST', '**/api/v1/anki/reviews').as('pull');
+    cy.intercept('POST', '**/api/v1/anki/push-plan').as('plan');
+    cy.visit('/vocabulary/anki/sync');
+    cy.waitForAlpine();
+    cy.get('[x-data="ankiSync"] button.is-primary').click();
+    cy.wait('@pull').then(({ response }) => {
+      const summary = unwrap<{ applied: number; terms: number; duplicates: number }>(response?.body);
+      expect(summary.applied, 'reviews merged').to.eq(3);
+      expect(summary.terms, 'terms rescheduled').to.eq(3);
+    });
+    // LWT had nothing Anki lacks: everything it holds came from Anki.
+    cy.wait('@plan').then(({ response }) => {
+      const plan = unwrap<{ reviews: unknown[]; cards: unknown[] }>(response?.body);
+      expect(plan.reviews, 'nothing to send back yet').to.have.length(0);
+    });
+    cy.get('[x-data="ankiSync"] .notification.is-success').should('be.visible');
+
+    // The answers now decide the schedule: the term answered Again in Anki is
+    // still due by tomorrow, the one answered Easy is pushed out.
+    cy.then(() => {
+      const [, againTerm, easyTerm] = termIds;
+      dueByTomorrow([againTerm]).then((count) => expect(count, 'Again term due').to.eq(1));
+      dueByTomorrow([easyTerm]).then((count) => expect(count, 'Easy term due').to.eq(0));
+    });
+
+    // 5. Studied in LWT: the first term, answered Easy here.
+    cy.then(() =>
+      cy.apiRequest({ method: 'PUT', url: `${apiBase}/review/status`, body: { term_id: termIds[0], grade: 4 } })
+    );
+
+    // 6. The next sync sends that review and the schedule it led to.
+    let planned: { card_id: number; due_days: number; ivl: number } | undefined;
+    cy.get('[x-data="ankiSync"] button.is-primary').click();
+    cy.wait('@pull').then(({ response }) => {
+      const summary = unwrap<{ applied: number; duplicates: number }>(response?.body);
+      expect(summary.applied, 'nothing new from Anki').to.eq(0);
+      expect(summary.duplicates).to.eq(3);
+    });
+    cy.wait('@plan').then(({ response }) => {
+      const plan = unwrap<{ reviews: number[][]; cards: { card_id: number; due_days: number; ivl: number }[] }>(
+        response?.body
+      );
+      expect(plan.reviews, 'the LWT review').to.have.length(1);
+      expect(plan.reviews[0][1]).to.eq(cardIds[0]);
+      expect(plan.reviews[0][3], 'its grade').to.eq(4);
+      expect(plan.cards).to.have.length(1);
+      planned = plan.cards[0];
+    });
+    // The page writes to Anki after the plan arrives; wait for it to finish.
+    cy.get('[x-data="ankiSync"] button.is-primary').should('not.be.disabled');
+    cy.get('[x-data="ankiSync"] .notification.is-success').should('be.visible');
+
+    // Anki now has it in the card's history, and LWT's schedule on the card.
+    cy.then(() => {
+      ankiConnect<Record<string, { ease: number }[]>>('getReviewsOfCards', { cards: [String(cardIds[0])] })
+        .then((byCard) => {
+          // setDueDate adds Anki's own grade-less "rescheduled" row (ease 0),
+          // which the pull ignores; the answers are the Anki one and LWT's.
+          const answers = byCard[String(cardIds[0])].filter((row) => row.ease > 0);
+          // Order is not checked: LWT keeps seconds, so its answer's id can sort
+          // before an Anki answer given earlier within the same second.
+          expect(answers.map((row) => row.ease), 'Anki answer and LWT answer').to.have.members([3, 4]);
+        });
+      ankiConnect<{ interval: number; reps: number; type: number }[]>('cardsInfo', { cards: [cardIds[0]] })
+        .then(([card]) => {
+          expect(card.type, 'a review card').to.eq(2);
+          expect(card.interval, 'LWT interval').to.eq(planned?.ivl);
+          expect(card.reps, 'both answers counted').to.eq(2);
+        });
+    });
+
+    // 7. And a sync right after changes nothing in either direction.
+    cy.get('[x-data="ankiSync"] button.is-primary').click();
+    cy.wait('@pull').then(({ response }) => {
+      expect(unwrap<{ applied: number }>(response?.body).applied).to.eq(0);
+    });
+    cy.wait('@plan').then(({ response }) => {
+      expect(unwrap<{ reviews: unknown[] }>(response?.body).reviews).to.have.length(0);
+    });
+  });
+});

docs-src/developer/anki-sync-conflicts.md

@@ -5,11 +5,13 @@ description: How LWT and Anki should merge when both change the same terms betwe
 
 # Spike: Conflict Resolution for Live Anki Sync
 
-**Status:** Spike result (#264). **Step 1 of the build order is implemented**:
-reviews are pulled from Anki over AnkiConnect at `/vocabulary/anki/sync`, and
-both that and the `.apkg` importer merge through `MergeReviewHistory`, which
-fits an older review into the history instead of skipping it. Steps 2 and 3
-are not built. The roadmap asked for
+**Status:** Spike result (#264). **Steps 1 and 2 of the build order are
+implemented**, and tested against a real Anki in a container (see
+[Testing against real Anki](#testing-against-real-anki)): `/vocabulary/anki/sync`
+pulls Anki's reviews into LWT through `MergeReviewHistory`, which fits an
+older review into the history instead of skipping it, then sends LWT's reviews
+and schedule back to Anki. The `.apkg` importer merges the same way. Step 3
+(fields, tags, flags, existence) is not built. The roadmap asked for
 this before any: *"Spike conflict resolution FIRST (the Anki problem) — it
 looks easy and isn't. Decide last-writer-wins+tombstones vs CRDT before
 committing."*
@@ -58,7 +60,8 @@ Checked in the code and in AnkiConnect's API.
 These details come from the typed client `yanki-connect` 4.0.6, written
 against AnkiConnect 25.11.9.0; the official README at
 `git.sr.ht/~foosoft/anki-connect` was unreachable when this was written.
-**Verify each against it before coding.**
+**Since verified** against AnkiConnect's own source (the AnkiWeb package) and
+by running it, for everything steps 1 and 2 use; see below.
 
 - `notesInfo` returns fields, tags, cards and a `mod` time per **note** — one
   timestamp for the whole note, not per field.
@@ -123,12 +126,22 @@ from the earliest unseen review rather than append after the latest.
   comes back in `cardReviews`. Key each review by (term, timestamp to the
   second, grade) and drop repeats. Exports already round-trip LWT's own history
   this way, and the importer already relies on it.
-- **Into Anki.** Push LWT's unseen reviews with `insertReviews`, then
-  `setDueDate` the card to LWT's due date so the next Anki session agrees. Anki
-  keeps its own memory state in `cards.data`; whether Anki recomputes it from
-  the inserted revlog, and when, **is unverified** and decides whether Anki's
-  next interval matches LWT's. If it does not, the due date is still right and
-  the next answer in either app re-converges.
+- **Into Anki.** Push LWT's unseen reviews with `insertReviews`, then set the
+  card's due date (`setDueDate`) and its interval, reps and lapses
+  (`setSpecificValueOfCard`, with `warning_check`) so the next Anki session
+  agrees. Measured against Anki 26.9.3 and AnkiConnect, FSRS on and off:
+  - `insertReviews` only appends history. The card's interval, due date, reps,
+    lapses and FSRS stability are untouched, and Anki's next answer comes out
+    exactly as on a twin card without the inserted review: Anki does not
+    replay revlog when answering. So the schedule has to be set separately.
+  - AnkiConnect refuses to write FSRS memory state ("Coupled scheduler state
+    must use native scheduling transitions"). With FSRS on, Anki's estimate
+    for a card therefore lags until the learner reoptimises, which recomputes
+    it from the history this push sends. The due date is right regardless.
+  - `setDueDate` adds Anki's own grade-less "rescheduled" revlog row, which
+    the pull ignores like any ease-0 row.
+  - `insertReviews` rejects a whole batch if one id already exists, so the
+    plan only ever sends ids Anki does not have.
 
 ### Everything else: three-way merge against a sync base
 
@@ -249,10 +262,11 @@ The spike's acceptance list, to become the merge function's unit tests:
 
 ## Open questions to settle before building
 
-- Does Anki recompute `cards.data` from inserted revlog rows, and when? Test
-  against real Anki, as `composer test:anki-oracle` does for files.
-- Confirm every AnkiConnect detail above against the official README.
-- Which Anki deck(s) map to which LWT language: one deck per language, chosen
+- ~~Does Anki recompute `cards.data` from inserted revlog rows?~~ Answered by
+  measurement: no; only reoptimising does. See *Into Anki* above.
+- ~~Confirm every AnkiConnect detail above.~~ Done for steps 1 and 2, against
+  AnkiConnect's source and a running Anki.
+- For step 3: which Anki deck(s) map to which LWT language: one deck per language, chosen
   once, is the simplest rule.
 - Whether status 1–5 should follow Anki at all, or stay derived from LWT's own
   schedule (phase 2 decision 1 says derived).
@@ -268,10 +282,35 @@ live sync is built, sequence it so each step is useful alone:
    `getReviewsOfCards` rather than `cardReviews` from a high-water mark, and
    the server drops those it already has. Simpler and needs no stored cursor;
    revisit if large collections make the pull slow.
-2. **Push LWT reviews** (`insertReviews` + `setDueDate`), once the
-   `cards.data` question is answered.
+2. **Push LWT reviews** (`insertReviews` + `setDueDate` +
+   `setSpecificValueOfCard`). ✅ Implemented: after the pull, the browser asks
+   `POST /api/v1/anki/push-plan` what Anki lacks (`AnkiPushPlan`) and writes
+   it. FSRS memory state cannot be written; see above.
 3. **Fields, tags, flags and existence** through the sync base, with the
    conflict report.
 
 Step 1 alone delivers most of what "sync with Anki" means to a learner who
 reviews on the phone and reads in LWT.
+
+## Testing against real Anki
+
+`scripts/anki/container` builds Anki 26.9.3 with AnkiConnect, headless (Qt's
+offscreen platform), from first-party sources: Anki from PyPI, AnkiConnect
+from AnkiWeb checked against a pinned hash. It trusts LWT's dev-server origins
+and shares `cypress/anki-exchange` as `/exchange` for package imports.
+
+```bash
+npm run anki:build        # once
+npm run anki:up           # Anki on 127.0.0.1:8765
+npm run serve             # LWT on :8000, in another terminal
+npx cypress run --spec cypress/e2e/24-anki-live.cy.ts
+npm run anki:down
+```
+
+`24-anki-live.cy.ts` runs the learner's loop for real: export terms from LWT,
+import them into Anki (`importPackage`), answer them there (`answerCards`),
+sync, answer one in LWT, sync again, and check Anki's history and card, then
+that a third sync changes nothing. It skips itself when nothing answers on
+8765, so CI is unaffected. `ANKI_FSRS=1` on `podman run` starts Anki with FSRS
+on; the spec passes in both modes.
+