Isolate the PHP schedule catalog record from SQLite maintenance

.github/workflows/server-parity.yml

@@ -107,6 +107,7 @@ on:
       - tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs
       - tests/Unit/ServerParityUpdateValidationContractTest.mjs
       - tests/Unit/ServerParityFixtureSelectionTest.mjs
+      - tests/Unit/NativeImageCorpusTest.mjs
       - tests/Unit/ServerParityUpdateValidationRetryTest.mjs
       - tests/Unit/ServerParityUpdateValidationTimingTest.mjs
       - tests/Unit/ServerParityKilledAuthorPollTest.mjs
@@ -2230,6 +2231,9 @@ jobs:
           # Composer runs confined in the pinned image, never in the published image under test.
           bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \
             install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress
+          # This shell-orchestration test needs the runner's jq, which the
+          # frozen PHP runtime image does not contain.
+          node --test tests/Unit/NativeImageCorpusTest.mjs
           docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \
             "$image" --test \
               tests/Unit/ServerParityRunnerTest.mjs \
@@ -2389,6 +2393,7 @@ jobs:
             -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \
             --mode http --url http://parity-server:8080 --target php --output /evidence/php.json \
             --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
+            --exclude-fixture tests/Fixtures/ServerParity/schedule-catalog.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
@@ -2402,6 +2407,7 @@ jobs:
             -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_PARITY_RUNNER_REVISION "$image" scripts/conformance/server-parity.mjs record \
             --mode embedded --application-root /app --target embedded --output /evidence/embedded.json \
             --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
+            --exclude-fixture tests/Fixtures/ServerParity/schedule-catalog.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
@@ -2415,6 +2421,7 @@ jobs:
             -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \
             --mode http --url http://parity-rust:8080 --target rust --output /evidence/rust.json \
             --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
+            --exclude-fixture tests/Fixtures/ServerParity/schedule-catalog.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
@@ -2427,6 +2434,7 @@ jobs:
             -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \
             scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json \
             --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
+            --exclude-fixture tests/Fixtures/ServerParity/schedule-catalog.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
@@ -2440,11 +2448,16 @@ jobs:
           # control-plane retries on a service-call insert.
           service_calls_status=0
           bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 service-calls || service_calls_status=$?
+          # Server #1021: backfill can lose an occurrence to the frozen PHP
+          # reference's SQLite maintenance writer even after five retries.
+          schedule_catalog_status=0
+          bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 schedule-catalog || schedule_catalog_status=$?
           # A native fixture difference fails the step only after the
           # comparison has run, so one difference cannot hide the rest.
           test "$native_record_status" -eq 0
           test "$compare_status" -eq 0
           test "$service_calls_status" -eq 0
+          test "$schedule_catalog_status" -eq 0
       - name: Compare published patch, legacy-marker and range-repair profiles
         if: ${{ matrix.part == 'restarts' }}
         env:

scripts/conformance/server-parity/native-image-corpus.sh

@@ -29,10 +29,11 @@ export DW_PARITY_RUNNER_REVISION
 DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD)
 image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.8.json)
 
-# CI's corpus selection: the reviewed corpus without service-calls, which
-# records separately below with PHP's maintenance loop stopped (Server #796),
-# plus the pending candidates it carries.
-selection=(--exclude-fixture tests/Fixtures/ServerParity/service-calls.json)
+# The frozen PHP reference loses SQLite writes under maintenance contention.
+# service-calls (#796) and schedule-catalog (#1021) record separately below
+# with PHP's loop stopped; native maintenance remains enabled.
+selection=(--exclude-fixture tests/Fixtures/ServerParity/service-calls.json
+  --exclude-fixture tests/Fixtures/ServerParity/schedule-catalog.json)
 for pending in patch-insertion-activity-pending patch-insertion-activity-completed patch-created-activity-pending \
   patch-created-activity-completed workflow-waiting-history-type workflow-waiting-history-message workflow-task-transient-retry; do
   selection+=(--include-fixture "tests/Fixtures/ServerParityPending/$pending.json")
@@ -115,17 +116,22 @@ docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -v "$PWD
 service_calls_status=0
 bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 service-calls \
   || service_calls_status=$?
+schedule_catalog_status=0
+bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 schedule-catalog \
+  || schedule_catalog_status=$?
 
 native_id=$(docker image inspect --format '{{.Id}}' "$native")
 jq -n --arg native "$native" --arg id "$native_id" --arg php "$image" --arg revision "$DW_PARITY_RUNNER_REVISION" \
   --argjson reference "$status" --argjson nativeStatus "$native_status" --argjson compare "$compare_status" \
   --argjson serviceCalls "$service_calls_status" \
+  --argjson scheduleCatalog "$schedule_catalog_status" \
   '{schema:"durable-workflow.native-image-corpus.v1", native_image:$native, native_image_id:$id, php_reference:$php,
     runner_revision:$revision, backend:"sqlite",
-    status:{reference_records:$reference, native_record:$nativeStatus, compare:$compare, service_calls:$serviceCalls}}' \
+    status:{reference_records:$reference, native_record:$nativeStatus, compare:$compare, service_calls:$serviceCalls, schedule_catalog:$scheduleCatalog}}' \
   > parity-evidence/native-image-corpus.json
 cat parity-evidence/native-image-corpus.json
 test "$status" -eq 0
 test "$native_status" -eq 0
 test "$compare_status" -eq 0
 test "$service_calls_status" -eq 0
+test "$schedule_catalog_status" -eq 0

scripts/conformance/server-parity/update-validation-sqlite.sh

@@ -4,8 +4,8 @@
 # Server #723. The SQLite job records the update-validation fixture apart from
 # its main corpus, for all three targets, and compares the three records. The
 # optional third argument names another fixture recorded the same way:
-# service-calls, whose published PHP record under SQLite writer contention has
-# refused the first call to a limited operation with 429 (Server #796).
+# service-calls (#796) or schedule-catalog, whose backfill can lose an
+# occurrence to PHP's concurrent SQLite maintenance writer (#1021).
 #
 # The published PHP record runs with PHP's maintenance loop stopped. On SQLite
 # the API server and that loop share one database file. Under the loop's writer

tests/Unit/NativeImageCorpusTest.mjs

@@ -0,0 +1,102 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join, resolve} from 'node:path';
+import {spawnSync} from 'node:child_process';
+import {selectFixturePaths} from '../../scripts/conformance/server-parity/fixture-selection.mjs';
+
+const root = resolve('.');
+const bash = spawnSync('bash', ['-c', 'command -v bash'], {encoding: 'utf8'}).stdout.trim();
+const revision = 'a'.repeat(40);
+
+// Run the real orchestration script. Stand-ins replace only external work:
+// Docker, dependency setup, the three-target helper and the CLI download.
+function run(failure = '') {
+  const dir = mkdtempSync(join(tmpdir(), 'native-image-corpus-'));
+  try {
+    for (const name of ['scripts', 'tests']) symlinkSync(join(root, name), join(dir, name));
+    mkdirSync(join(dir, 'bin'));
+    for (const name of ['calls', 'helpers']) writeFileSync(join(dir, name), '');
+    const mocks = {
+      git: `echo ${revision}`,
+      curl: 'echo fixture > parity-consumers/dw.phar',
+      sha256sum: 'cat > /dev/null; echo checksum-ok',
+      bash: `
+case "$1" in
+  scripts/ci/run-composer.sh) exit 0 ;;
+  scripts/conformance/server-parity/update-validation-sqlite.sh)
+    echo "$4" >> "$MOCK_DIR/helpers"
+    test "$FAILURE" != "$4" || exit 17 ;;
+  *) exit 99 ;;
+esac`,
+      docker: `
+echo "$*" >> "$MOCK_DIR/calls"
+case "$1" in
+  network) exit 0 ;;
+  image) echo sha256:${'b'.repeat(64)} ;;
+  run)
+    stage=
+    while test "$#" -gt 0; do
+      case "$1" in
+        --target) stage=$2; shift ;;
+        compare) stage=compare ;;
+      esac
+      shift
+    done
+    test -z "$stage" || test "$FAILURE" != "$stage" || exit 17 ;;
+  *) exit 99 ;;
+esac`,
+    };
+    for (const [name, source] of Object.entries(mocks)) {
+      writeFileSync(join(dir, 'bin', name), `#!/bin/sh\n${source}\n`);
+      chmodSync(join(dir, 'bin', name), 0o755);
+    }
+    const result = spawnSync(bash, ['scripts/conformance/server-parity/native-image-corpus.sh', 'dw-native-test:corpus'], {
+      cwd: dir, encoding: 'utf8',
+      env: {...process.env, PATH: `${join(dir, 'bin')}:${process.env.PATH}`, MOCK_DIR: dir, FAILURE: failure,
+        GITHUB_RUN_ID: '7', GITHUB_RUN_ATTEMPT: '2', GITHUB_JOB: 'corpus', RESOURCE_SCOPE: 'dw-parity-7-2-corpus'},
+    });
+    assert.ok(existsSync(join(dir, 'parity-evidence/native-image-corpus.json')), result.stderr);
+    return {...result,
+      receipt: JSON.parse(readFileSync(join(dir, 'parity-evidence/native-image-corpus.json'), 'utf8')),
+      calls: readFileSync(join(dir, 'calls'), 'utf8').trim().split('\n').map(line => line.split(' ')),
+      helpers: readFileSync(join(dir, 'helpers'), 'utf8').trim().split('\n'),
+    };
+  } finally {
+    rmSync(dir, {recursive: true, force: true});
+  }
+}
+
+test('native image qualification records every selected fixture once across the two partitions', () => {
+  const result = run();
+  assert.equal(result.status, 0, result.stderr);
+  assert.deepEqual(result.helpers, ['service-calls', 'schedule-catalog']);
+  const commands = result.calls.filter(args => args.includes('scripts/conformance/server-parity.mjs'));
+  assert.equal(commands.length, 4, 'three main records and their comparison');
+  let first;
+  for (const args of commands) {
+    const paths = flag => args.flatMap((arg, i) => arg === flag ? [args[i + 1]] : []);
+    const exclude = paths('--exclude-fixture');
+    assert.deepEqual(exclude, result.helpers.map(name => `tests/Fixtures/ServerParity/${name}.json`));
+    const include = paths('--include-fixture');
+    const directory = join(root, 'tests/Fixtures/ServerParity');
+    const main = selectFixturePaths({root, directory, include, exclude});
+    const all = [...main, ...exclude.map(path => resolve(root, path))].sort();
+    assert.deepEqual(all, selectFixturePaths({root, directory, include}).sort(), 'no fixture lost or repeated');
+    if (first) assert.deepEqual(all, first, 'all record and comparison selections agree');
+    first = all;
+  }
+  assert.deepEqual(result.receipt.status, {reference_records: 0, native_record: 0, compare: 0, service_calls: 0, schedule_catalog: 0});
+  assert.equal(result.receipt.runner_revision, revision);
+});
+
+for (const [stage, field] of [
+  ['php', 'reference_records'], ['embedded', 'reference_records'], ['rust', 'native_record'],
+  ['compare', 'compare'], ['service-calls', 'service_calls'], ['schedule-catalog', 'schedule_catalog'],
+]) test(`failure in ${stage} is retained in the receipt and fails image qualification`, () => {
+  const result = run(stage);
+  assert.notEqual(result.status, 0, result.stderr);
+  assert.equal(result.receipt.status[field], 17);
+  assert.deepEqual(result.helpers, ['service-calls', 'schedule-catalog'], 'one failure does not hide the separate checks');
+});

tests/Unit/ServerParityFixtureSelectionTest.mjs

@@ -57,17 +57,22 @@ function mainCorpusCommands(job) {
   return job.split(/(?=docker run )/).filter(command => /server-parity\.mjs (record|compare)/.test(command));
 }
 
-test('the SQLite job keeps update-validation in its main corpus and records service-calls separately', () => {
+test('the SQLite job keeps update-validation in its main corpus and isolates service-calls and schedule-catalog', () => {
   const sqlite = jobs()['execution-and-codec'];
   const commands = mainCorpusCommands(sqlite);
   assert.equal(commands.length, 4, 'php, embedded and rust records plus their comparison');
   for (const command of commands) {
-    assert.deepEqual(command.match(/--exclude-fixture \S+/g), ['--exclude-fixture tests/Fixtures/ServerParity/service-calls.json']);
+    assert.deepEqual(command.match(/--exclude-fixture \S+/g), [
+      '--exclude-fixture tests/Fixtures/ServerParity/service-calls.json',
+      '--exclude-fixture tests/Fixtures/ServerParity/schedule-catalog.json',
+    ]);
   }
   assert.ok(corpus.includes('update-validation.json'));
   assert.ok(corpus.includes('service-calls.json'));
   assert.match(sqlite, /service_calls_status=0\n\s+bash scripts\/conformance\/server-parity\/update-validation-sqlite\.sh http:\/\/parity-server:8080 http:\/\/parity-rust:8080 service-calls \|\| service_calls_status=\$\?/);
   assert.match(sqlite, /test "\$service_calls_status" -eq 0/);
+  assert.match(sqlite, /schedule_catalog_status=0\n\s+bash scripts\/conformance\/server-parity\/update-validation-sqlite\.sh http:\/\/parity-server:8080 http:\/\/parity-rust:8080 schedule-catalog \|\| schedule_catalog_status=\$\?/);
+  assert.match(sqlite, /test "\$schedule_catalog_status" -eq 0/);
   assert.doesNotMatch(sqlite, /update-validation-sqlite\.sh http:\/\/parity-server:8080 http:\/\/parity-rust:8080 \|\|/);
 });
 

tests/Unit/ServerParityUpdateValidationRetryTest.mjs

@@ -158,13 +158,14 @@ test('a native or embedded failure is never retried, even after a PHP retry', ()
   }
 });
 
-test('a named fixture records and retries under its own evidence names', () => {
-  const result = run('php1=fail-locked', 'service-calls');
+for (const fixture of ['service-calls', 'schedule-catalog']) test(`${fixture} records and retries under its own evidence names`, () => {
+  const result = run('php1=fail-locked', fixture);
   assert.equal(result.status, 0, result.stderr);
   assert.deepEqual(result.records, ['php parity-v1', 'php parity-v1-retry', 'embedded parity-v1-retry', 'rust parity-v1-retry']);
-  assert.match(result.stdout, /^::warning title=Published PHP service-calls record retried::/m);
-  for (const name of ['service-calls-php.json', 'service-calls-embedded.json', 'service-calls-rust.json', 'service-calls-comparison.json',
-    'service-calls-php.attempt-1.json', 'service-calls-php.attempt-1.locked.txt', 'service-calls-maintenance-schedule-retry.txt']) {
+  assert.ok(result.stdout.includes(`::warning title=Published PHP ${fixture} record retried::`));
+  for (const suffix of ['php.json', 'embedded.json', 'rust.json', 'comparison.json',
+    'php.attempt-1.json', 'php.attempt-1.locked.txt', 'maintenance-schedule-retry.txt']) {
+    const name = `${fixture}-${suffix}`;
     assert.equal(result.evidence(name), true, name);
   }
   assert.equal(result.evidence('update-validation-php.json'), false);