From 5eb9f2ca694b53bad8e414e50e3a50a30661d57f Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 20 Oct 2026 00:48:21 +1100 Subject: [PATCH 1/4] test(golang): run the Go SDK's rows and queries through EQL v4 in PostgreSQL Nothing in CI ran the path a Go program takes: rows the Go SDK stores in EQL columns, found by queries the Go SDK derives, through EQL's operators. encrypt/postgres_test.go stores terms in bytea columns with no EQL, and the Rust encryption suite runs EQL in PostgreSQL but derives its own queries, so a change that put Go and the SQL out of step failed nowhere (#1289). TestEQLTypesRoundTripAndQueryInPostgres walks eql.Types, the table generated from the EQL catalog, and for every type the engine produces declares a field of it at run time through gensupport, stores four rows in a column of its domain, and runs each query its indexes answer (=, <, <=, >, >= and @@) through the operand domain, asserting the exact rows. A type with no query for an operator must be refused with EQL's own error. The rows are read back from PostgreSQL, which stores jsonb re-serialised, and opened. A type whose plaintext or index the test has no case for fails rather than skipping, so a type the engine starts producing is covered or loudly not. With DATABASE_URL set and no eql_v4 schema it fails: a skip reads the same as a pass. CI runs it in a new eql-postgres job in tests-golang.yml, after wasi-check and on its guests, against postgres:17 with the bundle `mise run build` writes from the tree, through the new `mise run go:test:postgres` task (also the local entry point, against EQL_TEST_DATABASE_URL). The task fails if either PostgreSQL test skips. The existing bytea test moves there with it, since its database now carries EQL. The workflow's filters gain EQL's SQL sources and the codegen that writes the bundle. `eql_v4.between` is not covered: the Go SDK derives no EQL range term yet (Range on an EQL ordered field is refused by the engine), so there is no Go query to run through it. Closes #1289 Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .github/workflows/tests-golang.yml | 115 +++++- AGENTS.md | 4 +- languages/golang/encrypt/eql_postgres_test.go | 348 ++++++++++++++++++ languages/golang/go.mod | 2 +- mise.toml | 25 ++ 5 files changed, 473 insertions(+), 21 deletions(-) create mode 100644 languages/golang/encrypt/eql_postgres_test.go diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 6ef27e601..7494b4dcb 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -17,6 +17,11 @@ name: Tests (Go) # go-binding-cross # the same Go tests on macOS and Windows, against the guests # Linux built, after checking their sha256. +# eql-postgres +# the Go module's PostgreSQL tests, against PostgreSQL 17 with +# the EQL v4 bundle built from this tree installed: rows the Go +# SDK stores in every EQL type it produces, found by the queries +# it derives, through EQL's operators. # live the Go live tests and the stack-encrypt examples, against real # ZeroKMS with the repository's CS_* credentials. # @@ -36,6 +41,13 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # eql-postgres installs the EQL v4 bundle `mise run build` writes from + # these, so a SQL change starts the Go test that runs against it. + - packages/eql/src/** + - packages/eql/tasks/** + - packages/eql/crates/eql-codegen/** + - packages/eql/Cargo.toml + - packages/eql/mise.toml # The Go tests read the cross-language fixtures the EQL suite writes, # which are derived under EQL's lockfile. - packages/eql/tests/encryption/** @@ -65,6 +77,13 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # eql-postgres installs the EQL v4 bundle `mise run build` writes from + # these, so a SQL change starts the Go test that runs against it. + - packages/eql/src/** + - packages/eql/tasks/** + - packages/eql/crates/eql-codegen/** + - packages/eql/Cargo.toml + - packages/eql/mise.toml # The Go tests read the cross-language fixtures the EQL suite writes, # which are derived under EQL's lockfile. - packages/eql/tests/encryption/** @@ -111,24 +130,6 @@ jobs: # caches) the whole test toolset, which `mise run` would otherwise # install uncached. MISE_ENV: test - # The Go module's PostgreSQL test (encrypt/postgres_test.go) stores - # generated rows and runs the query methods against it through - # database/sql; it skips where DATABASE_URL is unset, as on the macOS - # and Windows jobs. - services: - postgres: - image: postgres:17 - env: - POSTGRES_PASSWORD: postgres - # Container port only: Docker picks the host port, and the Go step - # reads it from `job.services`, so the number is written once. - ports: - - 5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -242,7 +243,6 @@ jobs: - name: Go binding env: STACK_ENCRYPT_TESTS_REQUIRE_LOCK: "1" - DATABASE_URL: "postgres://postgres:postgres@localhost:${{ job.services.postgres.ports['5432'] }}/postgres?sslmode=disable" run: | ulimit -l "$(ulimit -H -l)" echo "RLIMIT_MEMLOCK: $(ulimit -l) KiB" @@ -370,6 +370,83 @@ jobs: - name: Go binding run: scripts/go-binding-test.sh languages/golang + # The Go SDK's rows and queries inside PostgreSQL with EQL installed: the + # path a program using the SDK runs. Its own job because it builds the EQL + # v4 bundle under packages/eql's mise config, which pins its own Rust; the + # guests are the ones wasi-check built and checked. + eql-postgres: + name: Go rows and queries in PostgreSQL with EQL v4 + needs: wasi-check + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 30 + services: + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: postgres + # Container port only: Docker picks the host port, and the test step + # reads it from `job.services`, so the number is written once. + ports: + - 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + # The root config for Go, and EQL's for the build of its bundle. + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + working_directory: . + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2204 + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + working_directory: packages/eql + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2204 + + # `mise run build` compiles eql-codegen to write the bundle. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: packages/eql + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: wasm-guests + path: languages/golang + + - name: The guests are the ones wasi-check built and checked + run: | + cd languages/golang + for guest in encrypt/testdata/stack_encrypt_guest_deterministic.wasm encrypt/testdata/stack_encrypt_guest_eql_deterministic.wasm; do + want=$(cat "$guest.sha256") + got=$(openssl dgst -sha256 "$guest" | awk '{print $NF}') + if [ "$want" != "$got" ]; then + echo "$guest checksum mismatch: artifact says $want, file is $got" >&2 + exit 1 + fi + echo "$guest sha256 $got" + done + + # Builds the bundle from the tree, installs it, and runs the tests; a + # skip fails the task, so a missing guest or database cannot pass. + - name: Go rows and queries through EQL v4 + env: + EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres sslmode=disable" + run: mise run go:test:postgres + # Round trips through real ZeroKMS. The Go live tests skip without their # STACK_ENCRYPT_TEST_* variables, so the secrets pre-flight runs first and # turns a missing credential into a failure rather than a silent skip. The diff --git a/AGENTS.md b/AGENTS.md index e9f5f82f5..7c1182636 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -731,7 +731,9 @@ exact pin would stop the suite sharing one `cts-common` with it. - **CI:** `tests-crates.yml` (fmt, clippy, nextest including the trybuild `tests/ui` binary, doctests, rustdoc, the node bindings and `stack-auth-wasm`), `tests-golang.yml` (WASI checks, the guests, the Go - module on Linux, macOS and Windows, and the live tests), `crap-crates.yml`, + module on Linux, macOS and Windows, its PostgreSQL tests against the EQL v4 + bundle built from the tree — `mise run go:test:postgres` locally — and the + live tests), `crap-crates.yml`, `mutants.yml`, `fuzz.yml`, `miri.yml` and `udeps.yml`. `scripts/__tests__/crates-ci.test.mjs` fails when a mise task in the root `mise.toml` or the five crate `tasks.toml` files loses its last CI caller. diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go new file mode 100644 index 000000000..bee34bb0f --- /dev/null +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -0,0 +1,348 @@ +package encrypt_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + _ "github.com/jackc/pgx/v5/stdlib" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Rows the Go SDK stores, found by queries the Go SDK derives, inside +// PostgreSQL with EQL v4 installed: the path a program using the SDK runs. +// Every EQL type the engine produces gets a temporary table with one column +// of its domain and four rows, low, middle, middle, high. Each query the +// type answers is derived through the SDK and run through EQL's operators, +// and must find exactly the rows its plaintexts say. The values are also +// read back from the database and opened, since PostgreSQL stores jsonb +// re-serialised. +// +// The list of types is eql.Types, generated from the EQL catalog, so a type +// the engine starts producing is covered here the day it lands. A type whose +// plaintext or index this file has no case for fails rather than being +// skipped. +// +// Skipped unless DATABASE_URL is set. With it set, EQL v4 must be installed +// there: a skip would read the same as a pass. CI installs the bundle built +// from the tree (tests-golang.yml); locally, `mise run go:test:postgres`. + +// The four rows' plaintexts for each wire kind, in ascending order with the +// middle two equal. Text is chosen so that only the middle rows share a +// trigram with the match query below. +var ( + eqlInt16s = [4]int16{-5, 7, 7, 40} + eqlInt32s = [4]int32{-5, 7, 7, 40} + eqlInt64s = [4]int64{-5_000_000_000, 7, 7, 40_000_000_000} + eqlFloat32s = [4]float32{-1.5, 2.25, 2.25, 10} + eqlFloat64s = [4]float64{-1.5, 2.25, 2.25, 10} + eqlBools = [4]bool{false, true, true, true} + eqlStrings = [4]string{"apple pie", "banana split", "banana split", "cherry tart"} + eqlMatch = "banana" + eqlDates = [4]encrypt.Date{ + encrypt.DateOf(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)), + encrypt.DateOf(time.Date(2021, 6, 15, 0, 0, 0, 0, time.UTC)), + encrypt.DateOf(time.Date(2021, 6, 15, 0, 0, 0, 0, time.UTC)), + encrypt.DateOf(time.Date(2023, 12, 31, 0, 0, 0, 0, time.UTC)), + } + eqlTimestamps = [4]time.Time{ + time.Date(2020, 1, 1, 8, 0, 0, 0, time.UTC), + time.Date(2021, 6, 15, 12, 30, 0, 0, time.UTC), + time.Date(2021, 6, 15, 12, 30, 0, 0, time.UTC), + time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC), + } + eqlDecimals = [4]string{"-1.50", "12.25", "12.25", "400"} +) + +// The rows each query must find, by the operator the SDK names it with. +var eqlExpected = map[string][]int{ + "eq": {2, 3}, + "lt": {1}, + "lte": {1, 2, 3}, + "gt": {4}, + "gte": {2, 3, 4}, +} + +// The SQL operator each SDK query operator is run through. The query +// operand is always on the right; the Rust suite covers it on the left. +var eqlOperators = map[string]string{ + "eq": "=", + "lt": "<", + "lte": "<=", + "gt": ">", + "gte": ">=", +} + +// The queries each index answers, by SDK operator. "match" is not here: it +// is a match query, derived differently, and handled by its own case. +var eqlIndexOperators = map[string][]string{ + "eq": {"eq"}, + "ope": {"lt", "lte", "gt", "gte"}, + "ore": {"lt", "lte", "gt", "gte"}, +} + +func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { + url := os.Getenv("DATABASE_URL") + if url == "" { + t.Skip("DATABASE_URL is not set") + } + ctx := context.Background() + c := deterministicEQLClient(t) + cipher := c.DefaultKeyset() + + db, err := sql.Open("pgx", url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + // One connection: the tables are temporary and live in this session. + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + + var installed bool + if err := conn.QueryRowContext(ctx, `SELECT to_regnamespace('eql_v4') IS NOT NULL`).Scan(&installed); err != nil { + t.Fatal(err) + } + if !installed { + t.Fatal("EQL v4 is not installed in DATABASE_URL: install the bundle built from the tree (`mise run build` in packages/eql writes release/cipherstash-encrypt-v4.sql), or run `mise run go:test:postgres`") + } + + covered := 0 + for _, typ := range eql.Types { + if !typ.Producible { + continue + } + covered++ + t.Run(typ.Name, func(t *testing.T) { + h := eqlHarness{t: t, ctx: ctx, cipher: cipher, conn: conn, typ: typ} + switch typ.Plaintext { + case "int16": + runEQLType(h, gensupport.Int16, eqlInt16s, nil) + case "int32": + runEQLType(h, gensupport.Int32, eqlInt32s, nil) + case "int64": + runEQLType(h, gensupport.Int64, eqlInt64s, nil) + case "float32": + runEQLType(h, gensupport.Float32, eqlFloat32s, nil) + case "float64": + runEQLType(h, gensupport.Float64, eqlFloat64s, nil) + case "bool": + runEQLType(h, gensupport.Bool, eqlBools, nil) + case "string": + runEQLType(h, gensupport.String, eqlStrings, func(f gensupport.Field[string]) ([]byte, error) { + q, err := gensupport.EQLMatches[[]byte](ctx, f, cipher, eqlMatch) + return q.Query(), err + }) + case "date": + runEQLType(h, gensupport.Date, eqlDates, nil) + case "timestamp": + runEQLType(h, gensupport.Timestamp, eqlTimestamps, nil) + case "decimal": + var ds [4]encrypt.Decimal + for i, s := range eqlDecimals { + d, err := encrypt.ParseDecimal(s) + if err != nil { + t.Fatal(err) + } + ds[i] = d + } + runEQLType(h, gensupport.Decimal, ds, nil) + default: + t.Fatalf("no plaintexts for wire kind %q: add a case so %s is tested", typ.Plaintext, typ.Name) + } + }) + } + if covered == 0 { + t.Fatal("eql.Types lists no producible type, so this test checked nothing") + } +} + +type eqlHarness struct { + t *testing.T + ctx context.Context + cipher *encrypt.Cipher + conn *sql.Conn + typ eql.Type +} + +// runEQLType stores the four values in a column of typ's domain, runs every +// query typ's indexes answer, and opens what the database hands back. +// match derives the match query for a type with the match index. +func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, match func(gensupport.Field[string]) ([]byte, error)) { + t, ctx, typ := h.t, h.ctx, h.typ + t.Helper() + decl := gensupport.Declare("go_eql_postgres").Passthrough("id").EncryptInto("v", kind, typ.Name) + if err := decl.Err(); err != nil { + t.Fatal(err) + } + field := gensupport.NewField[T](decl, "v") + table := "go_eql_" + strings.ToLower(typ.Name) + + exec := func(stmt string, args ...any) { + t.Helper() + if _, err := h.conn.ExecContext(ctx, stmt, args...); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + exec(fmt.Sprintf(`CREATE TEMPORARY TABLE %s (id int PRIMARY KEY, v %s NOT NULL)`, table, typ.SQLDomain)) + t.Cleanup(func() { _, _ = h.conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS `+table) }) + + for i, v := range values { + out, err := field.Encrypt(ctx, h.cipher, v) + if err != nil { + t.Fatal(err) + } + exec(fmt.Sprintf(`INSERT INTO %s VALUES ($1, $2::text::jsonb::%s)`, table, typ.SQLDomain), i+1, string(out.EQL)) + } + + ids := func(where string, operand []byte) ([]int, error) { + rows, err := h.conn.QueryContext(ctx, fmt.Sprintf(`SELECT id FROM %s WHERE %s ORDER BY id`, table, where), string(operand)) + if err != nil { + return nil, err + } + defer rows.Close() + var found []int + for rows.Next() { + var id int + if err := rows.Scan(&id); err != nil { + return nil, err + } + found = append(found, id) + } + return found, rows.Err() + } + operand := "$1::text::jsonb::" + typ.QuerySQLDomain + + answered := map[string]bool{} + for _, index := range typ.Indexes { + if index == "match" { + if match == nil { + t.Fatalf("%s has the match index but %s is not text", typ.Name, typ.Plaintext) + } + sf := gensupport.NewField[string](decl, "v") + q, err := match(sf) + if err != nil { + t.Fatal(err) + } + found, err := ids("v @@ "+operand, q) + if err != nil { + t.Fatalf("match: %v", err) + } + if !slices.Equal(found, []int{2, 3}) { + t.Errorf("v @@ match(%q) found rows %v, want [2 3]", eqlMatch, found) + } + continue + } + operators, ok := eqlIndexOperators[index] + if !ok { + t.Fatalf("no query for the %q index: add one so %s is tested", index, typ.Name) + } + for _, op := range operators { + if answered[op] { + continue + } + answered[op] = true + term, err := field.Term(ctx, h.cipher, op, values[1]) + if err != nil { + t.Fatalf("%s: %v", op, err) + } + found, err := ids(fmt.Sprintf("v %s %s", eqlOperators[op], operand), term) + if err != nil { + t.Fatalf("%s: %v", op, err) + } + if want := eqlExpected[op]; !slices.Equal(found, want) { + t.Errorf("v %s middle found rows %v, want %v", eqlOperators[op], found, want) + } + } + } + + // What the type does not answer, PostgreSQL refuses rather than + // comparing the stored jsonb. A storage-only type has no query domain, + // so it is compared with itself. + switch { + case len(typ.Indexes) == 0: + expectRefused(t, h.conn, ctx, fmt.Sprintf(`SELECT id FROM %s WHERE v = v`, table), "=", typ.SQLDomain) + case !answered["lt"]: + expectRefused(t, h.conn, ctx, fmt.Sprintf(`SELECT id FROM %s WHERE v < v`, table), "<", typ.SQLDomain) + } + + // Read back what PostgreSQL stored, and open it. + rows, err := h.conn.QueryContext(ctx, fmt.Sprintf(`SELECT v::text FROM %s ORDER BY id`, table)) + if err != nil { + t.Fatal(err) + } + var stored []gensupport.Record + for rows.Next() { + var text string + if err := rows.Scan(&text); err != nil { + t.Fatal(err) + } + stored = append(stored, gensupport.Record{"id": {Value: int32(len(stored) + 1)}, "v": {EQL: []byte(text)}}) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + opened, err := eqlCodec(decl).Decrypt(ctx, h.cipher, stored) + if err != nil { + t.Fatal(err) + } + for i, vals := range opened { + got, err := gensupport.Get[T](vals, "v") + if err != nil { + t.Fatal(err) + } + if !eqlSame(got, values[i]) { + t.Errorf("row %d opened as %v, want %v", i+1, got, values[i]) + } + } +} + +// eqlCodec opens records of one declaration without a generated type: the +// record is the encrypted form and the values map the plaintext one. +func eqlCodec(decl gensupport.Declaration) *gensupport.Codec[gensupport.Values, gensupport.Record] { + return gensupport.New(gensupport.Generated[gensupport.Values, gensupport.Record]{ + TypeName: "GoEQLPostgres", + Declaration: decl, + Source: func(v gensupport.Values) gensupport.Values { return v }, + Seal: func(r gensupport.Record) (gensupport.Record, error) { return r, nil }, + Open: func(r gensupport.Record) gensupport.Record { return r }, + Value: func(_ gensupport.Record, v gensupport.Values) (gensupport.Values, error) { return v, nil }, + }) +} + +// eqlSame compares two plaintexts, decimals by value rather than by their +// internal representation. +func eqlSame[T comparable](a, b T) bool { + if da, ok := any(a).(encrypt.Decimal); ok { + return da.String() == any(b).(encrypt.Decimal).String() + } + return a == b +} + +// expectRefused runs query and wants EQL's own refusal of operator on +// domain, not some other error that happens to stop the query. +func expectRefused(t *testing.T, conn *sql.Conn, ctx context.Context, query, operator, domain string) { + t.Helper() + rows, err := conn.QueryContext(ctx, query) + if err == nil { + err = rows.Close() + } + want := fmt.Sprintf("operator %s is not supported for %s", operator, domain) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || !strings.Contains(pgErr.Message, want) { + t.Errorf("%s: want PostgreSQL to refuse it with %q, got %v", query, want, err) + } +} diff --git a/languages/golang/go.mod b/languages/golang/go.mod index eb97e508c..2df28f1c2 100644 --- a/languages/golang/go.mod +++ b/languages/golang/go.mod @@ -10,7 +10,7 @@ require ( ) require ( - github.com/jackc/pgx/v5 v5.11.0 // test only: encrypt/postgres_test.go + github.com/jackc/pgx/v5 v5.11.0 // test only: encrypt/postgres_test.go, encrypt/eql_postgres_test.go golang.org/x/sys v0.49.0 golang.org/x/tools v0.51.0 google.golang.org/protobuf v1.36.12 diff --git a/mise.toml b/mise.toml index 2cc80b880..d64caf78f 100644 --- a/mise.toml +++ b/mise.toml @@ -367,6 +367,31 @@ description = "Format check, vet and test the Go module (languages/golang: encry # embeds is named relative to the module root. run = "scripts/go-binding-test.sh languages/golang" +[tasks."go:test:postgres"] +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives; needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" +# The bundle is built from packages/eql, as test:encryption:postgres builds +# it: the copy under crates/eql-bindings/sql is the RELEASED bundle and moves +# only at the lockstep bump, so it cannot show a SQL change in the same pull +# request. One connection string serves psql and the Go tests; pgx reads a +# libpq keyword/value string as well as a URL. +run = """ +#!/usr/bin/env bash +set -euo pipefail +: "${EQL_TEST_DATABASE_URL:?set EQL_TEST_DATABASE_URL to a libpq connection string for a disposable PostgreSQL}" +mise -C packages/eql run build +psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f packages/eql/release/cipherstash-encrypt-v4.sql +cd languages/golang +# The tests skip without DATABASE_URL and without their guest builds; -v and +# the grep turn either skip into a failure here, where both are provided. +DATABASE_URL="$EQL_TEST_DATABASE_URL" CGO_ENABLED=0 go test -count=1 -v \ + -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestQueriesRunAgainstPostgres)$' ./encrypt \ + | tee "${TMPDIR:-/tmp}/go-test-postgres.log" +if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then + echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 + exit 1 +fi +""" + [tasks."go:lint"] description = "Lint and format-check the Go module (languages/golang) with golangci-lint; config in languages/golang/.golangci.yaml" dir = "languages/golang" From 0986a925ddf35134652654da3e91c4b564b03e81 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 23:43:39 +1100 Subject: [PATCH 2/4] test(golang): query the block ORE EQL types in the PostgreSQL test next now produces the ten block ORE types (IntegerOrdOre, TextSearchOre and the rest), and the test fails on an index it has no query for, as it is meant to. ore_block answers lt, lte, gt and gte, as ore and ope do. All 49 producible types pass against PostgreSQL 17. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- languages/golang/encrypt/eql_postgres_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index bee34bb0f..d433b77b3 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -86,9 +86,10 @@ var eqlOperators = map[string]string{ // The queries each index answers, by SDK operator. "match" is not here: it // is a match query, derived differently, and handled by its own case. var eqlIndexOperators = map[string][]string{ - "eq": {"eq"}, - "ope": {"lt", "lte", "gt", "gte"}, - "ore": {"lt", "lte", "gt", "gte"}, + "eq": {"eq"}, + "ope": {"lt", "lte", "gt", "gte"}, + "ore": {"lt", "lte", "gt", "gte"}, + "ore_block": {"lt", "lte", "gt", "gte"}, } func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { From a8f09b4881f87ad4d6b1ef58ce4949e1706e274a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 23:53:26 +1100 Subject: [PATCH 3/4] test(golang): fail the PostgreSQL task when a test did not run, or read back short From review. go test -run with a pattern that matches nothing prints "no tests to run" and PASS, so a renamed test left the job green: the task now requires each test's own --- PASS line. The test also checks rows.Err() after reading the stored rows back, and that it opened as many rows as it stored, so a short read or decrypt cannot check fewer rows and pass. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- languages/golang/encrypt/eql_postgres_test.go | 7 +++++++ mise.toml | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index d433b77b3..ae704591d 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -293,6 +293,9 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m } stored = append(stored, gensupport.Record{"id": {Value: int32(len(stored) + 1)}, "v": {EQL: []byte(text)}}) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } if err := rows.Close(); err != nil { t.Fatal(err) } @@ -300,6 +303,10 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m if err != nil { t.Fatal(err) } + // A short read or a short decrypt would otherwise check fewer rows. + if len(opened) != len(values) { + t.Fatalf("read back and opened %d rows, want %d", len(opened), len(values)) + } for i, vals := range opened { got, err := gensupport.Get[T](vals, "v") if err != nil { diff --git a/mise.toml b/mise.toml index d64caf78f..507787ae8 100644 --- a/mise.toml +++ b/mise.toml @@ -390,6 +390,14 @@ if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 exit 1 fi +# A -run pattern that matches no test (one renamed, say) prints "no tests +# to run" and PASS, so each test has to be seen to pass by name. +for test in TestEQLTypesRoundTripAndQueryInPostgres TestQueriesRunAgainstPostgres; do + if ! grep -q -- "^--- PASS: $test " "${TMPDIR:-/tmp}/go-test-postgres.log"; then + echo "error: $test did not run and pass; was it renamed?" >&2 + exit 1 + fi +done """ [tasks."go:lint"] From 8f2c8f85729339f0568b32725f711c05cabe229e Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 24 Oct 2026 00:09:51 +1100 Subject: [PATCH 4/4] test(golang): store a nil row in every EQL type in the PostgreSQL test Optional fields are what #1298 is for, and nothing ran a stored nil through EQL in PostgreSQL. Each type now gets a fifth row with no value, stored as the engine requires for it. A type of fixed width with no index or equality alone stores a sealed null (17 types): its c must be as long as a value's, EqNull must find it and only it, and IS NULL must find nothing. Text and every ordered or match type store SQL NULL (32 types): IS NULL finds it, and no value query does. The nil row opens as nil. Breaking the length check and the EqNull check on purpose fails all 17 and all 8 types they cover. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- languages/golang/encrypt/eql_postgres_test.go | 125 +++++++++++++++--- 1 file changed, 108 insertions(+), 17 deletions(-) diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index ae704591d..aa4a07750 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -3,6 +3,7 @@ package encrypt_test import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "os" @@ -22,11 +23,16 @@ import ( // Rows the Go SDK stores, found by queries the Go SDK derives, inside // PostgreSQL with EQL v4 installed: the path a program using the SDK runs. // Every EQL type the engine produces gets a temporary table with one column -// of its domain and four rows, low, middle, middle, high. Each query the -// type answers is derived through the SDK and run through EQL's operators, -// and must find exactly the rows its plaintexts say. The values are also -// read back from the database and opened, since PostgreSQL stores jsonb -// re-serialised. +// of its domain and five rows: low, middle, middle, high, and a fifth with +// no value. Each query the type answers is derived through the SDK and run +// through EQL's operators, and must find exactly the rows its plaintexts +// say. The values are also read back from the database and opened, since +// PostgreSQL stores jsonb re-serialised. +// +// The fifth row is nil, stored as the engine requires for the type (#1298). +// A type of fixed width with no index or equality alone stores a sealed +// null, as long as a value, which EqNull finds; text and every type with an +// order or match index store SQL NULL, which no query finds. // // The list of types is eql.Types, generated from the EQL catalog, so a type // the engine starts producing is covered here the day it lands. A type whose @@ -185,12 +191,18 @@ type eqlHarness struct { func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, match func(gensupport.Field[string]) ([]byte, error)) { t, ctx, typ := h.t, h.ctx, h.typ t.Helper() - decl := gensupport.Declare("go_eql_postgres").Passthrough("id").EncryptInto("v", kind, typ.Name) + sealedNull := eqlSealsNull(typ) + mode, notNull := gensupport.NullSQL, "" + if sealedNull { + mode, notNull = gensupport.NullSealed, " NOT NULL" + } + decl := gensupport.Declare("go_eql_postgres").Passthrough("id").EncryptInto("v", kind, typ.Name).Null("v", mode) if err := decl.Err(); err != nil { t.Fatal(err) } field := gensupport.NewField[T](decl, "v") table := "go_eql_" + strings.ToLower(typ.Name) + nullRow := len(values) + 1 exec := func(stmt string, args ...any) { t.Helper() @@ -198,19 +210,36 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m t.Fatalf("%s: %v", stmt, err) } } - exec(fmt.Sprintf(`CREATE TEMPORARY TABLE %s (id int PRIMARY KEY, v %s NOT NULL)`, table, typ.SQLDomain)) + exec(fmt.Sprintf(`CREATE TEMPORARY TABLE %s (id int PRIMARY KEY, v %s%s)`, table, typ.SQLDomain, notNull)) t.Cleanup(func() { _, _ = h.conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS `+table) }) + insert := fmt.Sprintf(`INSERT INTO %s VALUES ($1, $2::text::jsonb::%s)`, table, typ.SQLDomain) + var first []byte for i, v := range values { out, err := field.Encrypt(ctx, h.cipher, v) if err != nil { t.Fatal(err) } - exec(fmt.Sprintf(`INSERT INTO %s VALUES ($1, $2::text::jsonb::%s)`, table, typ.SQLDomain), i+1, string(out.EQL)) + if i == 0 { + first = out.EQL + } + exec(insert, i+1, string(out.EQL)) + } + if sealedNull { + out, err := field.EncryptNull(ctx, h.cipher) + if err != nil { + t.Fatal(err) + } + if got, want := eqlCiphertextLen(t, out.EQL), eqlCiphertextLen(t, first); got != want { + t.Errorf("a sealed null's c is %d characters, a value's %d: the column shows which rows are absent", got, want) + } + exec(insert, nullRow, string(out.EQL)) + } else { + exec(insert, nullRow, nil) } - ids := func(where string, operand []byte) ([]int, error) { - rows, err := h.conn.QueryContext(ctx, fmt.Sprintf(`SELECT id FROM %s WHERE %s ORDER BY id`, table, where), string(operand)) + ids := func(where string, args ...any) ([]int, error) { + rows, err := h.conn.QueryContext(ctx, fmt.Sprintf(`SELECT id FROM %s WHERE %s ORDER BY id`, table, where), args...) if err != nil { return nil, err } @@ -238,7 +267,7 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m if err != nil { t.Fatal(err) } - found, err := ids("v @@ "+operand, q) + found, err := ids("v @@ "+operand, string(q)) if err != nil { t.Fatalf("match: %v", err) } @@ -260,7 +289,7 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m if err != nil { t.Fatalf("%s: %v", op, err) } - found, err := ids(fmt.Sprintf("v %s %s", eqlOperators[op], operand), term) + found, err := ids(fmt.Sprintf("v %s %s", eqlOperators[op], operand), string(term)) if err != nil { t.Fatalf("%s: %v", op, err) } @@ -270,6 +299,33 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m } } + // The nil row: a sealed null is the one row EqNull finds, and SQL NULL + // is the one row IS NULL finds. No query above found it. + if sealedNull && answered["eq"] { + term, err := field.NullTerm(ctx, h.cipher, "eq") + if err != nil { + t.Fatalf("eq of nil: %v", err) + } + found, err := ids("v = "+operand, string(term)) + if err != nil { + t.Fatalf("eq of nil: %v", err) + } + if !slices.Equal(found, []int{nullRow}) { + t.Errorf("v = EqNull found rows %v, want [%d]", found, nullRow) + } + } + found, err := ids("v IS NULL") + if err != nil { + t.Fatal(err) + } + if want := []int{nullRow}; sealedNull { + if len(found) != 0 { + t.Errorf("a sealed null is a value, but IS NULL found rows %v", found) + } + } else if !slices.Equal(found, want) { + t.Errorf("IS NULL found rows %v, want %v", found, want) + } + // What the type does not answer, PostgreSQL refuses rather than // comparing the stored jsonb. A storage-only type has no query domain, // so it is compared with itself. @@ -287,11 +343,16 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m } var stored []gensupport.Record for rows.Next() { - var text string + var text sql.NullString if err := rows.Scan(&text); err != nil { t.Fatal(err) } - stored = append(stored, gensupport.Record{"id": {Value: int32(len(stored) + 1)}, "v": {EQL: []byte(text)}}) + rec := gensupport.Record{"id": {Value: int32(len(stored) + 1)}} + if text.Valid { + // SQL NULL leaves the field out, as a generated type does. + rec["v"] = gensupport.Output{EQL: []byte(text.String)} + } + stored = append(stored, rec) } if err := rows.Err(); err != nil { t.Fatal(err) @@ -304,10 +365,13 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m t.Fatal(err) } // A short read or a short decrypt would otherwise check fewer rows. - if len(opened) != len(values) { - t.Fatalf("read back and opened %d rows, want %d", len(opened), len(values)) + if len(opened) != nullRow { + t.Fatalf("read back and opened %d rows, want %d", len(opened), nullRow) } - for i, vals := range opened { + if got, err := gensupport.GetPtr[T](opened[nullRow-1], "v"); err != nil || got != nil { + t.Errorf("the nil row opened as %v, %v; want nil", got, err) + } + for i, vals := range opened[:len(values)] { got, err := gensupport.Get[T](vals, "v") if err != nil { t.Fatal(err) @@ -318,6 +382,33 @@ func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, m } } +// eqlSealsNull says whether typ stores nil as a sealed null, as stashgen +// would choose for it: a plaintext of fixed width, not text, and no index +// but equality, since a null has an equality term and no other. +func eqlSealsNull(typ eql.Type) bool { + if typ.Plaintext == "string" { + return false + } + for _, index := range typ.Indexes { + if index != "eq" { + return false + } + } + return true +} + +// eqlCiphertextLen is the length of the `c` an EQL value stores. +func eqlCiphertextLen(t *testing.T, value []byte) int { + t.Helper() + var envelope struct { + C string `json:"c"` + } + if err := json.Unmarshal(value, &envelope); err != nil || envelope.C == "" { + t.Fatalf("the EQL value has no c: %v: %s", err, value) + } + return len(envelope.C) +} + // eqlCodec opens records of one declaration without a generated type: the // record is the encrypted form and the values map the plaintext one. func eqlCodec(decl gensupport.Declaration) *gensupport.Codec[gensupport.Values, gensupport.Record] {