From 33be518a38d23e8db07fa085e4bfe8786baa644e Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Fri, 31 Jul 2026 15:24:26 +0200 Subject: [PATCH 1/2] test(table): benchmark Table.Iter row scanning Extract the Iter scan loop into iterRows so it can be driven directly, and add a DB-independent benchmark using an in-memory fake pgx.Rows. This measures the per-row scan cost of the current implementation, which creates a fresh scany RowScanner (recomputing the column-to-field mapping) on every row via pgxscan.API.ScanRow. Serves as the baseline for the scanner-reuse optimization in the next commit. Co-Authored-By: Claude Opus 4.8 (1M context) --- iter_bench_test.go | 149 +++++++++++++++++++++++++++++++++++++++++++++ table.go | 7 ++- 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 iter_bench_test.go diff --git a/iter_bench_test.go b/iter_bench_test.go new file mode 100644 index 0000000..6bcbacb --- /dev/null +++ b/iter_bench_test.go @@ -0,0 +1,149 @@ +package pgkit + +import ( + "fmt" + "reflect" + "testing" + + "github.com/georgysavva/scany/v2/dbscan" + "github.com/georgysavva/scany/v2/pgxscan" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +// benchRow is a representative record: a mix of column types so struct field +// mapping has real work to do per row. +type benchRow struct { + ID int64 `db:"id"` + Name string `db:"name"` + Email string `db:"email"` + Age int32 `db:"age"` + Score float64 `db:"score"` + Active bool `db:"active"` +} + +func (r *benchRow) GetID() int64 { return r.ID } +func (r *benchRow) Validate() error { return nil } + +// benchScanAPI mirrors the scan API pgkit builds in ConnectWithPGX so the +// benchmark exercises the same reflection path as production. +func benchScanAPI(tb testing.TB) *pgxscan.API { + tb.Helper() + dbScanAPI, err := pgxscan.NewDBScanAPI(dbscan.WithAllowUnknownColumns(true)) + if err != nil { + tb.Fatal(err) + } + api, err := pgxscan.NewAPI(dbScanAPI) + if err != nil { + tb.Fatal(err) + } + return api +} + +func benchTable(tb testing.TB) *Table[benchRow, *benchRow, int64] { + return &Table[benchRow, *benchRow, int64]{ + DB: &DB{Query: &Querier{Scan: benchScanAPI(tb)}}, + } +} + +// makeBenchData builds n identical rows matching benchRow's columns. +func makeBenchData(n int) ([]string, [][]any) { + cols := []string{"id", "name", "email", "age", "score", "active"} + data := make([][]any, n) + for i := range data { + data[i] = []any{ + int64(i), + "account name", + "user@example.com", + int32(30), + float64(99.5), + true, + } + } + return cols, data +} + +func BenchmarkTableIter(b *testing.B) { + for _, n := range []int{100, 1000} { + b.Run(fmt.Sprintf("rows=%d", n), func(b *testing.B) { + cols, data := makeBenchData(n) + tbl := benchTable(b) + b.ReportAllocs() + for b.Loop() { + rows := newFakeRows(cols, data) + var count int + for _, err := range tbl.iterRows(rows) { + if err != nil { + b.Fatal(err) + } + count++ + } + if count != n { + b.Fatalf("scanned %d rows, want %d", count, n) + } + } + }) + } +} + +// fakeRows is an in-memory pgx.Rows used to benchmark the scan loop without a +// live database. Scan copies the current row's values into the positional +// destination pointers via reflection. +type fakeRows struct { + fields []pgconn.FieldDescription + data [][]any + pos int // 1-based index of the current row; 0 means before first Next + err error +} + +func newFakeRows(cols []string, data [][]any) *fakeRows { + fields := make([]pgconn.FieldDescription, len(cols)) + for i, c := range cols { + fields[i] = pgconn.FieldDescription{Name: c} + } + return &fakeRows{fields: fields, data: data} +} + +func (r *fakeRows) Close() {} +func (r *fakeRows) Err() error { return r.err } +func (r *fakeRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} } +func (r *fakeRows) FieldDescriptions() []pgconn.FieldDescription { return r.fields } + +func (r *fakeRows) Next() bool { + if r.pos >= len(r.data) { + return false + } + r.pos++ + return true +} + +func (r *fakeRows) Scan(dest ...any) error { + if r.pos == 0 || r.pos > len(r.data) { + return fmt.Errorf("Scan called out of range") + } + row := r.data[r.pos-1] + if len(dest) != len(row) { + return fmt.Errorf("scan target count %d != column count %d", len(dest), len(row)) + } + for i, d := range dest { + if d == nil { + continue + } + dv := reflect.ValueOf(d) + if dv.Kind() != reflect.Pointer || dv.IsNil() { + return fmt.Errorf("scan dest %d is not a non-nil pointer", i) + } + dv.Elem().Set(reflect.ValueOf(row[i])) + } + return nil +} + +func (r *fakeRows) Values() ([]any, error) { + if r.pos == 0 || r.pos > len(r.data) { + return nil, fmt.Errorf("Values called out of range") + } + return r.data[r.pos-1], nil +} + +func (r *fakeRows) RawValues() [][]byte { return nil } +func (r *fakeRows) Conn() *pgx.Conn { return nil } diff --git a/table.go b/table.go index 08b95a2..5073420 100644 --- a/table.go +++ b/table.go @@ -561,6 +561,11 @@ func (t *Table[T, P, I]) Iter(ctx context.Context, where sq.Sqlizer, orderBy []s return nil, fmt.Errorf("query rows: %w", err) } + return t.iterRows(rows), nil +} + +// iterRows yields records scanned from rows, closing rows when iteration ends. +func (t *Table[T, P, I]) iterRows(rows pgx.Rows) iter.Seq2[P, error] { return func(yield func(P, error) bool) { defer rows.Close() for rows.Next() { @@ -576,7 +581,7 @@ func (t *Table[T, P, I]) Iter(ctx context.Context, where sq.Sqlizer, orderBy []s if err := rows.Err(); err != nil { yield(nil, err) } - }, nil + } } // GetByID returns a record by its ID. From 3fda4b02cf58bf26f4f33754f8cdc534dc2bab19 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Fri, 31 Jul 2026 15:25:32 +0200 Subject: [PATCH 2/2] perf(table): reuse RowScanner across Iter rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iter created a fresh scany RowScanner per row via pgxscan.API.ScanRow, recomputing the column-to-field mapping on every iteration. Create one RowScanner before the loop and reuse it so the reflection work happens once per query instead of once per row. Benchmarked via BenchmarkTableIter (in-memory fake pgx.Rows), count=8: rows=1000 time 394.8µs -> 177.5µs (-55%) B/op 406.6Ki -> 63.2Ki (-84%) allocs 6002 -> 1008 (-83%, ~6/row -> ~1/row) All deltas p=0.000. GetAll/List/pagination already reuse a single RowScanner internally, so this only affects the Iter streaming path. Co-Authored-By: Claude Opus 4.8 (1M context) --- table.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/table.go b/table.go index 5073420..101d15e 100644 --- a/table.go +++ b/table.go @@ -565,12 +565,18 @@ func (t *Table[T, P, I]) Iter(ctx context.Context, where sq.Sqlizer, orderBy []s } // iterRows yields records scanned from rows, closing rows when iteration ends. +// +// A single RowScanner is created once and reused across every row: it caches +// the column-to-field mapping after the first Scan, so the reflection work is +// done once per query rather than once per row (which pgxscan.API.ScanRow +// would do by allocating a fresh RowScanner each call). func (t *Table[T, P, I]) iterRows(rows pgx.Rows) iter.Seq2[P, error] { return func(yield func(P, error) bool) { defer rows.Close() + rs := t.Query.Scan.NewRowScanner(rows) for rows.Next() { var record T - if err := t.Query.Scan.ScanRow(&record, rows); err != nil { + if err := rs.Scan(&record); err != nil { yield(nil, err) return }