Skip to content

Commit 01916d9

Browse files
committed
feat(server): build the attention acknowledgement index through store migration 0003
Main's numbered migrations pin each schema version's checksum, which covers indexes.sql, so the audit-log index the acknowledgement listing reads needs its own migration. 0003 only creates sqb_security_audit_attention, with IF NOT EXISTS so an index an operator pre-built CONCURRENTLY on PostgreSQL is skipped. Version 3's SQLite and PostgreSQL checksums are pinned, and the upgrade tests now expect the latest version. The server docs say how to pre-build the index before upgrading, since on a large audit log the migration holds up audit writes, including sign-ins. Coordination: the observed-runs server half adds its own migration (sqb_warehouse_runs) later; whichever lands first takes 0003, and the other renumbers to 0004 and pins its checksums again.
1 parent f13c235 commit 01916d9

7 files changed

Lines changed: 53 additions & 19 deletions

File tree

‎crates/sqlbuild-cli/tests/native_commands/test_control.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -379,13 +379,13 @@ fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migr
379379
StoreMigrationCommandTestCase {
380380
description: "migrate an empty store",
381381
arguments: &["server", "migrate"],
382-
expected_stdout: "The control store is empty: created schema version 2.\n",
382+
expected_stdout: "The control store is empty: created schema version 3.\n",
383383
expected_store: true,
384384
},
385385
StoreMigrationCommandTestCase {
386386
description: "dry run on a current store",
387387
arguments: &["server", "migrate", "--dry-run"],
388-
expected_stdout: "The control store is at schema version 2: nothing to migrate.\n",
388+
expected_stdout: "The control store is at schema version 3: nothing to migrate.\n",
389389
expected_store: true,
390390
},
391391
];

‎crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,18 @@ use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan};
1414
use crate::store::constants::{INDEXES, SCHEMA};
1515

1616
/// The migrations after the baseline, in order: entry `i` includes `store_migrations/sql/{NNNN}_{name}.{sqlite,postgres}.sql` and brings a store to version `NNNN = BASELINE_VERSION + i + 1`.
17-
pub(crate) const MIGRATIONS: &[StoreMigration] = &[StoreMigration {
18-
name: "large_project_server",
19-
sqlite: include_str!("../sql/0002_large_project_server.sqlite.sql"),
20-
postgres: include_str!("../sql/0002_large_project_server.postgres.sql"),
21-
}];
17+
pub(crate) const MIGRATIONS: &[StoreMigration] = &[
18+
StoreMigration {
19+
name: "large_project_server",
20+
sqlite: include_str!("../sql/0002_large_project_server.sqlite.sql"),
21+
postgres: include_str!("../sql/0002_large_project_server.postgres.sql"),
22+
},
23+
StoreMigration {
24+
name: "attention_acknowledgement_index",
25+
sqlite: include_str!("../sql/0003_attention_acknowledgement_index.sqlite.sql"),
26+
postgres: include_str!("../sql/0003_attention_acknowledgement_index.postgres.sql"),
27+
},
28+
];
2229

2330
/// The schema version a store created by this build has.
2431
pub(crate) fn current_version(migrations: &[StoreMigration]) -> i64 {
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- The index the Overview's attention acknowledgement listing reads: each item's latest
2+
-- acknowledgement or withdrawal. IF NOT EXISTS skips an index an operator pre-built.
3+
-- On a large audit log, pre-build it with CREATE INDEX CONCURRENTLY IF NOT EXISTS to keep sign-ins flowing.
4+
CREATE INDEX IF NOT EXISTS sqb_security_audit_attention ON sqb_security_audit((scope_json ->> 'location_id'), (scope_json ->> 'target'), (scope_json ->> 'key'), storage_id) WHERE action IN ('attention.acknowledged', 'attention.acknowledgement_withdrawn');
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
-- The index the Overview's attention acknowledgement listing reads: each item's latest
2+
-- acknowledgement or withdrawal. IF NOT EXISTS skips an index an operator pre-built.
3+
CREATE INDEX IF NOT EXISTS sqb_security_audit_attention ON sqb_security_audit((scope_json ->> 'location_id'), (scope_json ->> 'target'), (scope_json ->> 'key'), storage_id) WHERE action IN ('attention.acknowledged', 'attention.acknowledgement_withdrawn');

‎crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ const SQLITE_CHECKSUMS: &[(i64, &str)] = &[
3737
2,
3838
"4d4e3aa56b3a39d8550f691ae12fe463187b4f4cdeacc66ca94803701cab5e0a",
3939
),
40+
(
41+
3,
42+
"734163e22863ff86fbcf1329540736f279806d4ce5ea04f6d6ecb1703b3468c8",
43+
),
4044
];
4145
const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[
4246
(
@@ -47,6 +51,10 @@ const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[
4751
2,
4852
"99e13d88687d6e590e47149200a0d1cb03c0ff285a95339da40b76cba99e2a21",
4953
),
54+
(
55+
3,
56+
"b1250935a5adf1d4691836a2125bb57360c1ff0a2a296a77e4d11bf6a80a99d9",
57+
),
5058
];
5159
/// A version 1 store mid-run: a child run holding a concurrency slot, a jobs document and an audit result.
5260
const VERSION_1_LIVE_WORK: &str = "ALTER TABLE sqb_schema_version ADD COLUMN version BIGINT; \
@@ -170,13 +178,13 @@ fn given_a_version_1_store_with_live_work_when_migrating_then_rows_default_and_r
170178
description: "SQLite store",
171179
dialect: &SQLITE_STORE,
172180
foreign_key_check: "SELECT \"table\" || ' ' || parent AS item FROM pragma_foreign_key_check",
173-
expected_version: 2,
181+
expected_version: current_version(MIGRATIONS),
174182
},
175183
LiveWorkUpgradeTestCase {
176184
description: "PostgreSQL store",
177185
dialect: &POSTGRES_STORE,
178186
foreign_key_check: "SELECT 'enforced' AS item WHERE FALSE",
179-
expected_version: 2,
187+
expected_version: current_version(MIGRATIONS),
180188
},
181189
];
182190
for test_case in test_cases {

‎src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -348,11 +348,17 @@ server can't read what they wrote.
348348
the control server is older than this worker and refused its run report; upgrade the server before its workers
349349
```
350350

351-
Some releases add an index to the control database, which the server builds when it starts. On
352-
PostgreSQL that build holds a lock on its table, so on a large table writes wait for it: the index
353-
on acknowledgements in the audit log, for example, delays sign-ins until it is built. Upgrade when
354-
few people are signed in, or create the index yourself first with `CREATE INDEX CONCURRENTLY` and the
355-
definition in the release notes; the server then finds it already there.
351+
Some migrations add an index, which the server builds when it migrates. On PostgreSQL that build
352+
locks its table against writes, so on a large table those writes wait until it is built. Migration
353+
`0003` indexes acknowledgements in the audit log, so on a large audit log it holds up audit writes,
354+
including sign-ins. To avoid that, build the index yourself before you upgrade, without blocking
355+
writes:
356+
357+
```sql
358+
CREATE INDEX CONCURRENTLY IF NOT EXISTS sqb_security_audit_attention ON sqb_security_audit((scope_json ->> 'location_id'), (scope_json ->> 'target'), (scope_json ->> 'key'), storage_id) WHERE action IN ('attention.acknowledged', 'attention.acknowledgement_withdrawn');
359+
```
360+
361+
The migration then finds the index already there and skips it.
356362

357363
## Set retention
358364

‎website/src/content/docs/docs/cli/server.mdx‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -327,11 +327,17 @@ server can't read what they wrote.
327327
the control server is older than this worker and refused its run report; upgrade the server before its workers
328328
```
329329

330-
Some releases add an index to the control database, which the server builds when it starts. On
331-
PostgreSQL that build holds a lock on its table, so on a large table writes wait for it: the index
332-
on acknowledgements in the audit log, for example, delays sign-ins until it is built. Upgrade when
333-
few people are signed in, or create the index yourself first with `CREATE INDEX CONCURRENTLY` and the
334-
definition in the release notes; the server then finds it already there.
330+
Some migrations add an index, which the server builds when it migrates. On PostgreSQL that build
331+
locks its table against writes, so on a large table those writes wait until it is built. Migration
332+
`0003` indexes acknowledgements in the audit log, so on a large audit log it holds up audit writes,
333+
including sign-ins. To avoid that, build the index yourself before you upgrade, without blocking
334+
writes:
335+
336+
```sql
337+
CREATE INDEX CONCURRENTLY IF NOT EXISTS sqb_security_audit_attention ON sqb_security_audit((scope_json ->> 'location_id'), (scope_json ->> 'target'), (scope_json ->> 'key'), storage_id) WHERE action IN ('attention.acknowledged', 'attention.acknowledgement_withdrawn');
338+
```
339+
340+
The migration then finds the index already there and skips it.
335341

336342
## Set retention
337343

0 commit comments

Comments
 (0)