diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b7eca6..69535fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,7 @@ jobs: sleep 2 done docker compose exec -T postgres psql -U postgres -tAc \ - "SELECT extversion FROM pg_extension WHERE extname='kafgres'" | grep -q 0.3.3 + "SELECT extversion FROM pg_extension WHERE extname='kafgres'" | grep -q 0.4.0 docker compose exec -T postgres psql -U postgres -tAc \ "SELECT count(*) FROM pg_stat_activity WHERE backend_type='kafgres_broker'" | grep -q '^1$' diff --git a/Cargo.lock b/Cargo.lock index 05a93bf..12b4667 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,7 +56,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "kafgres-codec" -version = "0.3.3" +version = "0.4.0" dependencies = [ "bytes", "crc32c", @@ -68,7 +68,7 @@ dependencies = [ [[package]] name = "kafgres-codec-gen" -version = "0.3.3" +version = "0.4.0" dependencies = [ "serde", "serde_json", diff --git a/README.md b/README.md index c4622b7..c9d38c2 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,10 @@ broker. Install the new build, restart Postgres, then run `ALTER EXTENSION kafgres UPDATE;` in the database the broker uses. The broker serves across the restart before the update -runs; the update changes who may call the administrative functions (see -[docs/producing.md](docs/producing.md#who-may-produce)) and pins `kafgres_produce()`'s -search path. +runs. Updating from 0.3.0 adds `kafgres_read()` ([docs/reading.md](docs/reading.md)), +executable by no role until it is granted. Updating from 0.2.0 also changes who may call +the administrative functions (see [docs/producing.md](docs/producing.md#who-may-produce)) +and pins `kafgres_produce()`'s search path. From 0.2.0, the first start on the segment engine removes the log's old-format `.timeindex` files. Nothing is lost: timestamp lookups scan the segments written before @@ -277,6 +278,7 @@ about 180 scenarios against both brokers in CI on every pull request. |-----|----------| | [docs/architecture.md](docs/architecture.md) | The design: one broker per instance, how durability maps onto Postgres, both storage engines, transactional produce, segment replication. | | [docs/producing.md](docs/producing.md) | The three produce paths, CDC mappings, and how to choose between them. | +| [docs/reading.md](docs/reading.md) | `kafgres_read()`: a topic's records as rows, by offset or time range, for SQL to filter and join. | | [docs/conformance.md](docs/conformance.md) | The client matrix, how the suite runs, and the catalogue of client-visible differences from Kafka. | | [docs/configuration.md](docs/configuration.md) | Every `kafgres.*` setting: scope, default, and what it controls. | @@ -297,7 +299,7 @@ codec/ kafgres-codec, the wire protocol. No pgrx, no Postgres, unit-tes src/generated/ emitted by codec-gen and checked in; do not edit codec-gen/ the generator. `cargo run -p kafgres-codec-gen` extension/ the pgrx extension. Its own workspace on purpose. -docs/ architecture.md, producing.md, conformance.md, configuration.md +docs/ architecture.md, producing.md, reading.md, conformance.md, configuration.md ``` ## Build diff --git a/codec-gen/Cargo.toml b/codec-gen/Cargo.toml index 9d51e64..3dc1b60 100644 --- a/codec-gen/Cargo.toml +++ b/codec-gen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kafgres-codec-gen" -version = "0.3.3" +version = "0.4.0" edition = "2021" license = "Elastic-2.0" description = "Emits kafgres-codec from the vendored Kafka message schemas." diff --git a/codec/Cargo.toml b/codec/Cargo.toml index 8d77fcc..d97814b 100644 --- a/codec/Cargo.toml +++ b/codec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kafgres-codec" -version = "0.3.3" +version = "0.4.0" edition = "2021" license = "Elastic-2.0" description = "Kafka wire protocol codec, generated from vendored schemas. No pgrx, no Postgres." diff --git a/codec/src/consume.rs b/codec/src/consume.rs new file mode 100644 index 0000000..c82f16b --- /dev/null +++ b/codec/src/consume.rs @@ -0,0 +1,288 @@ +//! Consumer-side batch handling: the `read_committed` filter, and records with their batch fields. + +use std::collections::HashSet; + +use bytes::Bytes; + +use crate::records::{BatchError, RecordBatch, LOG_APPEND_TIME_FLAG}; + +/// The Java consumer's aborted-transaction filter over one fetch. Batches go in log order. +#[derive(Debug, Default)] +pub struct AbortFilter { + /// `(producer_id, first_offset)` not yet reached, latest first. + pending: Vec<(i64, i64)>, + /// Producers whose open transaction is aborted, until their abort marker passes. + aborted: HashSet, +} + +impl AbortFilter { + pub fn new(aborted: impl IntoIterator) -> Self { + let mut pending: Vec<(i64, i64)> = aborted.into_iter().collect(); + pending.sort_unstable_by_key(|a| std::cmp::Reverse(a.1)); + AbortFilter { + pending, + aborted: HashSet::new(), + } + } + + /// Whether a `read_committed` consumer keeps this batch. Control batches never are; an + /// abort marker ends its producer's aborted span. + pub fn admit(&mut self, batch: &RecordBatch) -> bool { + let producer = batch.producer_id(); + if producer >= 0 { + while self + .pending + .last() + .is_some_and(|(_, first)| *first <= batch.last_offset()) + { + if let Some((p, _)) = self.pending.pop() { + self.aborted.insert(p); + } + } + } + if batch.is_control() { + if control_type(batch) == Some(ControlType::Abort) { + self.aborted.remove(&producer); + } + return false; + } + !(batch.is_transactional() && self.aborted.contains(&producer)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ControlType { + Abort, + Commit, +} + +/// The marker a control batch carries, from its first record's key: `version: i16, type: i16`. +pub fn control_type(batch: &RecordBatch) -> Option { + let record = batch.records_decompressed().ok()?.next()?.ok()?; + let key = record.key?; + match i16::from_be_bytes(key.get(2..4)?.try_into().ok()?) { + 0 => Some(ControlType::Abort), + 1 => Some(ControlType::Commit), + _ => None, + } +} + +/// One record, with the batch fields that apply to it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConsumedRecord { + pub offset: i64, + /// Milliseconds since the epoch; the batch's max timestamp under `LogAppendTime`. + pub timestamp: i64, + pub log_append_time: bool, + pub key: Option, + pub value: Option, + pub headers: Vec<(Bytes, Option)>, + pub leader_epoch: i32, + pub producer_id: i64, + pub producer_epoch: i16, + /// `-1` outside an idempotent producer's sequence space. + pub sequence: i32, + pub is_transactional: bool, +} + +/// A batch's records, decompressed, one at a time. A control batch yields its marker record. +/// A record declaring more than `max_headers` headers is an error, before any is decoded. +pub fn consumed_records( + batch: &RecordBatch, + max_headers: usize, +) -> Result> + 'static, BatchError> { + let base = batch.base_offset(); + let base_timestamp = batch.base_timestamp(); + let log_append_time = batch.attributes() & LOG_APPEND_TIME_FLAG != 0; + let max_timestamp = batch.max_timestamp(); + let leader_epoch = batch.partition_leader_epoch(); + let producer_id = batch.producer_id(); + let producer_epoch = batch.producer_epoch(); + let base_sequence = batch.base_sequence(); + let is_transactional = batch.is_transactional(); + Ok(batch + .records_decompressed()? + .max_headers(max_headers) + .map(move |r| { + let r = r?; + Ok(ConsumedRecord { + offset: base + r.offset_delta as i64, + timestamp: if log_append_time { + max_timestamp + } else { + base_timestamp.saturating_add(r.timestamp_delta) + }, + log_append_time, + key: r.key, + value: r.value, + headers: r.headers, + leader_epoch, + producer_id, + producer_epoch, + sequence: sequence_at(base_sequence, r.offset_delta), + is_transactional, + }) + })) +} + +/// A record's sequence number: sequences wrap from `i32::MAX` to 0, as Kafka's do. +fn sequence_at(base: i32, delta: i32) -> i32 { + if base < 0 { + return -1; + } + if base > i32::MAX - delta { + delta - (i32::MAX - base) - 1 + } else { + base + delta + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::records::{build_batch_full, build_control_batch, NewRecord}; + + fn at(bytes: Bytes, base: i64) -> RecordBatch { + RecordBatch::validated(bytes).unwrap().stamp(base, 7).view() + } + + fn data(base: i64, n: usize, producer: i64, transactional: bool) -> RecordBatch { + let records: Vec = (0..n) + .map(|i| NewRecord { + key: Some(format!("k{i}").into_bytes()), + value: Some(format!("v{i}").into_bytes()), + timestamp: 1_000 + i as i64, + }) + .collect(); + at(build_batch_full(&records, producer, transactional), base) + } + + fn marker(base: i64, producer: i64, commit: bool) -> RecordBatch { + at(build_control_batch(producer, 0, commit), base) + } + + /// One record with `n` empty, null-valued headers. + fn with_headers(n: i32) -> RecordBatch { + use crate::records::{put_varint_i32, put_varint_i64, LENGTH_OFFSET, MAGIC_V2}; + use bytes::{BufMut, BytesMut}; + let mut one = BytesMut::new(); + one.put_i8(0); + put_varint_i64(&mut one, 0); + put_varint_i32(&mut one, 0); + put_varint_i32(&mut one, -1); + put_varint_i32(&mut one, -1); + put_varint_i32(&mut one, n); + for _ in 0..n { + put_varint_i32(&mut one, 0); + put_varint_i32(&mut one, -1); + } + let mut b = BytesMut::new(); + b.put_i64(0); + b.put_i32(0); + b.put_i32(-1); + b.put_i8(MAGIC_V2); + b.put_u32(0); + b.put_i16(0); + b.put_i32(0); + b.put_i64(0); + b.put_i64(0); + b.put_i64(-1); + b.put_i16(-1); + b.put_i32(-1); + b.put_i32(1); + put_varint_i32(&mut b, one.len() as i32); + b.put_slice(&one); + let length = (b.len() - LENGTH_OFFSET - 4) as i32; + b[LENGTH_OFFSET..LENGTH_OFFSET + 4].copy_from_slice(&length.to_be_bytes()); + RecordBatch::new(b.freeze()).unwrap() + } + + #[test] + fn a_record_over_the_header_limit_is_refused_before_its_headers_are_decoded() { + let first = |n, max| { + consumed_records(&with_headers(n), max) + .unwrap() + .next() + .unwrap() + }; + assert_eq!(first(2, 2).unwrap().headers.len(), 2); + assert_eq!( + first(3, 2).unwrap_err(), + BatchError::TooManyHeaders { count: 3, max: 2 } + ); + } + + #[test] + fn records_carry_their_offsets_and_batch_metadata() { + let batch = data(40, 3, 9, true); + let got: Vec = consumed_records(&batch, usize::MAX) + .unwrap() + .map(Result::unwrap) + .collect(); + assert_eq!( + got.iter().map(|r| r.offset).collect::>(), + [40, 41, 42] + ); + assert_eq!( + got.iter().map(|r| r.timestamp).collect::>(), + [1_000, 1_001, 1_002] + ); + assert_eq!(got[1].key.as_deref(), Some(&b"k1"[..])); + assert_eq!(got[1].value.as_deref(), Some(&b"v1"[..])); + assert!(got + .iter() + .all(|r| r.leader_epoch == 7 && r.producer_id == 9 && r.is_transactional)); + assert!(got.iter().all(|r| r.sequence == -1 && !r.log_append_time)); + } + + #[test] + fn sequences_wrap_as_kafka_does() { + assert_eq!(sequence_at(5, 2), 7); + assert_eq!(sequence_at(i32::MAX, 0), i32::MAX); + assert_eq!(sequence_at(i32::MAX, 1), 0); + assert_eq!(sequence_at(i32::MAX - 1, 3), 1); + assert_eq!(sequence_at(-1, 3), -1); + } + + #[test] + fn an_aborted_transaction_is_skipped_until_its_marker() { + // Producer 5 aborts a transaction spanning two batches; producer 6 commits around it. + let mut f = AbortFilter::new([(5, 10)]); + assert!(f.admit(&data(0, 10, -1, false))); + assert!(!f.admit(&data(10, 2, 5, true))); + assert!(f.admit(&data(12, 2, 6, true))); + assert!(!f.admit(&data(14, 2, 5, true))); + assert!(!f.admit(&marker(16, 5, false))); + assert!(!f.admit(&marker(17, 6, true))); + assert!(f.admit(&data(18, 1, 5, true))); + } + + #[test] + fn an_abort_that_began_before_the_read_still_applies() { + let mut f = AbortFilter::new([(5, 3)]); + assert!(!f.admit(&data(20, 2, 5, true))); + assert!(f.admit(&data(22, 1, 6, true))); + } + + #[test] + fn a_producer_listed_as_aborted_is_kept_before_its_first_offset() { + let mut f = AbortFilter::new([(5, 30)]); + assert!(f.admit(&data(20, 2, 5, true))); + assert!(!f.admit(&data(30, 2, 5, true))); + } + + #[test] + fn non_transactional_batches_from_an_aborted_producer_are_kept() { + let mut f = AbortFilter::new([(5, 0)]); + assert!(f.admit(&data(0, 1, 5, false))); + } + + #[test] + fn control_batches_are_never_admitted_and_name_their_marker() { + let mut f = AbortFilter::new([]); + let commit = marker(3, 1, true); + assert_eq!(control_type(&commit), Some(ControlType::Commit)); + assert_eq!(control_type(&marker(4, 1, false)), Some(ControlType::Abort)); + assert!(!f.admit(&commit)); + } +} diff --git a/codec/src/lib.rs b/codec/src/lib.rs index 023597c..60b1548 100644 --- a/codec/src/lib.rs +++ b/codec/src/lib.rs @@ -1,6 +1,7 @@ //! Kafka wire protocol codec for kafgres. Free of pgrx and libpq, so protocol correctness pub mod compaction; +pub mod consume; pub mod errors; pub mod framing; pub mod header; diff --git a/codec/src/records.rs b/codec/src/records.rs index b07eb4d..ff6d892 100644 --- a/codec/src/records.rs +++ b/codec/src/records.rs @@ -25,6 +25,8 @@ const COMPRESSION_MASK: i16 = 0x07; /// Bit 4 of the batch attributes: `isTransactional`. A `read_committed` consumer applies pub const TRANSACTIONAL_FLAG: i16 = 0x10; const CONTROL_FLAG: i16 = 0x20; +/// Bit 3: `timestampType`. Set means `LogAppendTime`, and every record takes `maxTimestamp`. +pub const LOG_APPEND_TIME_FLAG: i16 = 0x08; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BatchError { @@ -66,6 +68,11 @@ pub enum BatchError { DecompressedTooLarge, /// The batch is compressed, and this decoder does not decompress — not malformed, Compressed(i16), + /// A record declares more headers than the caller's limit, checked before decoding them. + TooManyHeaders { + count: i64, + max: usize, + }, } impl BatchError { @@ -90,6 +97,7 @@ impl BatchError { // Not INVALID_RECORD: the batch is fine, just too large. MESSAGE_TOO_LARGE is BatchError::DecompressedTooLarge => ErrorCode::MessageTooLarge, BatchError::Compressed(_) => ErrorCode::UnknownServerError, + BatchError::TooManyHeaders { .. } => ErrorCode::InvalidRecord, } } } @@ -145,6 +153,9 @@ impl std::fmt::Display for BatchError { BatchError::DecompressedTooLarge => { write!(f, "batch expands past {MAX_DECOMPRESSED_BYTES} bytes") } + BatchError::TooManyHeaders { count, max } => { + write!(f, "a record declares {count} headers, more than the {max} allowed") + } BatchError::Compressed(codec) => { write!(f, "batch uses compression codec {codec}; this decoder reads uncompressed batches only") } @@ -987,6 +998,7 @@ pub struct RecordIter { done: bool, /// False reads keys only: values and headers are checked and skipped. full: bool, + max_headers: usize, } /// [`RecordIter`] reading keys only, with every record checked the same way. @@ -1054,11 +1066,18 @@ impl RecordBatch { declared: count, done: false, full, + max_headers: usize::MAX, }) } } impl RecordIter { + /// Refuse a record declaring more than `max` headers. Unlimited by default. + pub fn max_headers(mut self, max: usize) -> Self { + self.max_headers = max; + self + } + /// Reads stay inside the record's declared extent, and the fields must fill it exactly. fn next_record(&mut self) -> Result { let at = self.src.pos(); @@ -1110,6 +1129,12 @@ impl RecordIter { { return Err(BatchError::DecompressedTooLarge); } + if header_count as u64 > self.max_headers as u64 { + return Err(BatchError::TooManyHeaders { + count: i64::from(header_count), + max: self.max_headers, + }); + } let mut headers = Vec::new(); for _ in 0..header_count { if self.src.pos() >= end { diff --git a/demo/read/README.md b/demo/read/README.md new file mode 100644 index 0000000..94b54ce --- /dev/null +++ b/demo/read/README.md @@ -0,0 +1,158 @@ +# Demo: a bad deploy, investigated in SQL straight off the Kafka log + +Yesterday at 14:00, build 2.4.1 of a payments service went out with a retry bug that charged +some customers twice. It was rolled back at 14:20. The charges are events on a Kafka topic; +the orders are rows in a table in the same Postgres. Every step of the investigation below +is one SQL statement over the topic, using [`kafgres_read()`](../../docs/reading.md). + +| Normally | Here | +|---|---| +| A consumer app to copy the topic into a table before anyone can query it | `FROM kafgres_read('payments')` | +| A stream processor or ETL job to join events with the database | an ordinary `JOIN orders` | +| Consuming a whole day of events to look at twenty minutes of them | `from_time` / `to_time`, resolved like `offsetsForTimes` | + +## Running it + +```sh +docker compose up -d +pip install -r demo/read/requirements.txt +python3 demo/read/incident.py # 200,000 orders, about 30 s +SCALE=0.05 python3 demo/read/incident.py # 10,000 orders +``` + +The script produces a day of charges with a real idempotent Kafka producer (librdkafka, +lz4), with `trace-id` and `service-version` headers, and plants the double charges itself. +Each step prints its SQL and the rows it returned, then checks the answer against what was +planted, so the demo is also a test: it exits non-zero if any check fails. + +The outputs below are from one run with 200,000 orders on the segment engine (the table +engine gives the same answers). Dates are yesterday's, whenever it runs. + +## The helper + +The topic holds JSON values and two headers. One function, created once, turns it into typed +columns. It is a function rather than a view so that a time range is passed to +`kafgres_read()`, which then reads only that range: + +```sql +CREATE FUNCTION payments(from_time timestamptz DEFAULT NULL, to_time timestamptz DEFAULT NULL) +RETURNS TABLE (charged_at timestamptz, order_id bigint, amount_cents bigint, + version text, trace_id text, charge_no bigint) +LANGUAGE sql AS $$ + SELECT r.timestamp, p.order_id, p.amount_cents, + convert_from(r.header_values[array_position(r.header_keys, 'service-version')], 'UTF8'), + convert_from(r.header_values[array_position(r.header_keys, 'trace-id')], 'UTF8'), + row_number() OVER (PARTITION BY p.order_id ORDER BY r.partition, r.record_offset) + FROM kafgres_read('payments', from_time => from_time, to_time => to_time) r, + jsonb_to_record(convert_from(r.value, 'UTF8')::jsonb) + AS p(order_id bigint, amount_cents bigint) +$$; +``` + +`charge_no` numbers each order's charges in log order, so a repeat charge is `charge_no > 1`. + +## 1. Support: "customers say they were charged twice." How bad is it? + +```sql +SELECT count(*) AS customers_charged_twice, + sum(amount_cents) / 100.0 AS overcharged_usd + FROM payments() + WHERE charge_no > 1; +``` + +``` +customers_charged_twice overcharged_usd +----------------------- --------------- +405 100,546.59 +(1 rows, 595 ms) +``` + +## 2. When did it happen? + +```sql +SELECT to_char(date_trunc('hour', charged_at), 'HH24:MI') AS hour, + count(*) AS charges, + count(*) FILTER (WHERE charge_no > 1) AS repeat_charges, + repeat('#', (count(*) FILTER (WHERE charge_no > 1) / 10)::int) AS " " + FROM payments() + GROUP BY 1 ORDER BY 1; +``` + +``` +hour charges repeat_charges +----- ------- -------------- ---------------------------------------- +... +12:00 8,510 0 +13:00 8,510 0 +14:00 8,916 405 ######################################## +15:00 8,512 0 +16:00 8,511 0 +... +(24 rows, 627 ms) +``` + +Every other hour of the day has no repeat charges. + +## 3. What changed at 14:00? + +Zoom to 13:55 to 14:25, by minute. Only that half hour is read: `kafgres_read()` finds its +start and end offsets through the time index, the way Kafka's `offsetsForTimes` does. + +```sql +SELECT to_char(date_trunc('minute', charged_at), 'HH24:MI') AS minute, + version, + count(*) AS charges, + count(*) FILTER (WHERE charge_no > 1) AS repeat_charges + FROM payments('2026-10-13 13:55+00', '2026-10-13 14:25+00') + GROUP BY 1, 2 ORDER BY 1, 2; +``` + +``` +minute version charges repeat_charges +------ ------- ------- -------------- +13:58 2.4.0 141 0 +13:59 2.4.0 142 0 +14:00 2.4.1 162 19 +14:01 2.4.1 163 20 +... 2.4.1 19 to 21 every minute +14:19 2.4.1 162 21 +14:20 2.4.0 140 0 +14:21 2.4.0 144 0 +(30 rows, 18 ms) +``` + +2.4.1 shows up at 14:00, is the only build that charged twice, and is gone at 14:20. The +query read 4,657 of the day's 200,405 events, in 18 ms against 627 ms for the whole day. + +## 4. Who do we refund? + +```sql +CREATE TABLE refunds_due AS +SELECT o.id AS order_id, o.customer, p.amount_cents, p.trace_id AS duplicate_trace_id + FROM payments('2026-10-13 14:00+00', '2026-10-13 14:20+00') p + JOIN orders o ON o.id = p.order_id + WHERE p.charge_no > 1; +``` + +``` +order_id customer amount_cents duplicate_trace_id +-------- ---------- ------------ ------------------ +119149 cust-22690 7,405 50b21eb4b6b37fb8 +119156 cust-02105 37,047 54d3b77b1c1ac855 +119163 cust-32574 25,241 070db68829e09a22 +... +(405 rows, built in 15 ms) +``` + +## Checks + +``` +PASS how-bad 405 customers, $100,546.59: exactly the repeat charges that were planted +PASS when every repeat charge is in the 14:00 hour; the whole day read in 627 ms +PASS what-changed 2.4.1 ran 14:00 to 14:19 and is the only build that charged twice; read 4,657 of 200,405 events in 18 ms +PASS refunds 405 refunds, one per double-charged order, each for the repeated amount +PASS consistent kafgres_read() returned the same 200,405 records, byte for byte, as a librdkafka consumer +``` + +The last check is not part of the story: it compares every record `kafgres_read()` returned +with what a librdkafka consumer reads from the same topic. diff --git a/demo/read/incident.py b/demo/read/incident.py new file mode 100644 index 0000000..adf3dcc --- /dev/null +++ b/demo/read/incident.py @@ -0,0 +1,303 @@ +"""kafgres_read(): a bad deploy that double-charged customers, investigated in SQL off the log. + +Yesterday's payment events sit on a Kafka topic, written by a real idempotent producer, next +to the `orders` table in the same Postgres. Build 2.4.1 went out at 14:00 with a retry bug +that charged some customers twice, and was rolled back at 14:20. Each step of the +investigation is one SQL statement over the topic, checked against what the generator planted. + + pip install -r demo/read/requirements.txt + python3 demo/read/incident.py # 200,000 orders + SCALE=0.05 python3 demo/read/incident.py # 10,000 orders + +Needs the compose stack up (`docker compose up -d`). See demo/read/README.md. +""" + +import datetime +import decimal +import json +import os +import random +import sys +import time +import uuid + +import psycopg +from confluent_kafka import Consumer, Producer, TopicPartition +from confluent_kafka.admin import AdminClient, NewTopic + +BOOT = os.environ.get("BOOT", "127.0.0.1:9092") +# UTC, so the hours SQL buckets by are the ones the checks name. +DSN = ("host=127.0.0.1 port=5432 user=postgres password=postgres dbname=postgres " + "options='-c TimeZone=UTC'") +SCALE = float(os.environ.get("SCALE", "1")) +N_ORDERS = int(200_000 * SCALE) +RESULTS = [] +rng = random.Random(20261014) +UTC = datetime.timezone.utc + + +# --- plumbing ------------------------------------------------------------------------------ + +def say(*a): + print(*a, flush=True) + + +def step(title): + say("\n" + "=" * 92 + f"\n {title}\n" + "=" * 92) + + +def check(name, ok, detail, quiet=False): + RESULTS.append((name, ok, detail)) + if not quiet: + say(f" {'PASS' if ok else 'FAIL'} {detail}") + + +def sql(statement, show=True, max_rows=30): + """Run one statement, print it and its rows, return (rows, seconds).""" + with psycopg.connect(DSN, autocommit=True) as c: + t0 = time.perf_counter() + cur = c.execute(statement) + rows = cur.fetchall() if cur.description else [] + took = time.perf_counter() - t0 + cols = [d.name for d in cur.description] if cur.description else [] + if show: + say("\n" + "\n".join(" " + l for l in statement.strip("\n").splitlines())) + say("") + if cols: + table(cols, rows[:max_rows]) + if len(rows) > max_rows: + say(f" ... {len(rows) - max_rows} more rows") + say(f" ({len(rows)} rows, {took * 1000:.0f} ms)" if cols else f" ({took * 1000:.0f} ms)") + return rows, took + + +def table(cols, rows): + cells = [[str(v) if c.endswith("_id") else fmt(v) for c, v in zip(cols, r)] for r in rows] + width = [max([len(c)] + [len(r[i]) for r in cells]) for i, c in enumerate(cols)] + say(" " + " ".join(c.ljust(w) for c, w in zip(cols, width)).rstrip()) + say(" " + " ".join("-" * w for w in width)) + for r in cells: + say(" " + " ".join(v.ljust(w) for v, w in zip(r, width)).rstrip()) + + +def fmt(v): + if v is None: + return "" + if isinstance(v, datetime.datetime): + return v.astimezone(UTC).strftime("%H:%M:%S") + if isinstance(v, decimal.Decimal): + return f"{v:,.2f}" if v != v.to_integral_value() else f"{int(v):,}" + if isinstance(v, int): + return f"{v:,}" + return str(v) + + +def make_topic(name, partitions): + sql(f"SELECT kafgres_drop_topic('{name}')", show=False) + admin = AdminClient({"bootstrap.servers": BOOT}) + for f in admin.create_topics([NewTopic(name, partitions, 1)]).values(): + f.result(30) + time.sleep(0.5) + + +def consume_all(name): + """What a librdkafka consumer reads from the beginning: {(partition, offset): (key, value)}.""" + c = Consumer({"bootstrap.servers": BOOT, "group.id": f"audit-{uuid.uuid4()}", + "enable.auto.commit": False, "auto.offset.reset": "earliest"}) + parts = list(c.list_topics(name, timeout=10).topics[name].partitions) + c.assign([TopicPartition(name, p, 0) for p in parts]) + out, last = {}, time.time() + while time.time() - last < 4: + for m in c.consume(10_000, 0.5): + if not m.error(): + out[(m.partition(), m.offset())] = (m.key(), m.value()) + last = time.time() + c.close() + return out + + +# --- the day being investigated ------------------------------------------------------------ + +class Day: + """Yesterday's orders and charges, with 2.4.1 live from 14:00 to 14:20.""" + + def __init__(self): + today = datetime.datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) + self.date = today - datetime.timedelta(days=1) + self.start = int(self.date.timestamp() * 1000) + self.deploy = self.start + 14 * 3600 * 1000 + self.rollback = self.deploy + 20 * 60 * 1000 + step_ms = 23.5 * 3600 * 1000 / N_ORDERS + self.orders = [] # (id, customer, cents, placed ms) + for i in range(1, N_ORDERS + 1): + self.orders.append((i, f"cust-{rng.randint(1, max(1, N_ORDERS // 4)):05d}", + rng.randint(199, 49_999), self.start + int(i * step_ms))) + + # Each order is charged 0.2 to 4 s after it is placed. 2.4.1's retry bug charges every + # 7th order it handles a second time, 0.8 to 2.5 s later. + self.charges = [] # (ms, order, cents, version) + self.double = {} # order -> cents charged again + handled = 0 + for oid, _, cents, placed in self.orders: + at = placed + rng.randint(200, 4_000) + version = self.version(at) + self.charges.append((at, oid, cents, version)) + if version == "2.4.1": + handled += 1 + again = at + rng.randint(800, 2_500) + if handled % 7 == 0 and again < self.rollback: + self.charges.append((again, oid, cents, version)) + self.double[oid] = cents + self.charges.sort() + + def version(self, ms): + return "2.4.1" if self.deploy <= ms < self.rollback else "2.4.0" + + def at(self, hhmm): + """Yesterday at HH:MM, as a timestamptz literal.""" + h, m = map(int, hhmm.split(":")) + return (self.date + datetime.timedelta(hours=h, minutes=m)).strftime("%Y-%m-%d %H:%M+00") + + +SETUP_SQL = """ +CREATE FUNCTION payments(from_time timestamptz DEFAULT NULL, to_time timestamptz DEFAULT NULL) +RETURNS TABLE (charged_at timestamptz, order_id bigint, amount_cents bigint, + version text, trace_id text, charge_no bigint) +LANGUAGE sql AS $$ + SELECT r.timestamp, p.order_id, p.amount_cents, + convert_from(r.header_values[array_position(r.header_keys, 'service-version')], 'UTF8'), + convert_from(r.header_values[array_position(r.header_keys, 'trace-id')], 'UTF8'), + row_number() OVER (PARTITION BY p.order_id ORDER BY r.partition, r.record_offset) + FROM kafgres_read('payments', from_time => from_time, to_time => to_time) r, + jsonb_to_record(convert_from(r.value, 'UTF8')::jsonb) + AS p(order_id bigint, amount_cents bigint) +$$;""" + + +def load(day): + step("Setup: yesterday's orders in Postgres, and its charges on the 'payments' topic") + with psycopg.connect(DSN, autocommit=True) as c: + c.execute("DROP TABLE IF EXISTS orders, refunds_due") + c.execute("DROP FUNCTION IF EXISTS payments") + c.execute("""CREATE TABLE orders (id bigint PRIMARY KEY, customer text NOT NULL, + amount_cents bigint NOT NULL, placed_at timestamptz NOT NULL)""") + with c.cursor().copy("COPY orders FROM STDIN") as cp: + for oid, customer, cents, ms in day.orders: + cp.write_row((oid, customer, cents, datetime.datetime.fromtimestamp(ms / 1000, UTC))) + c.execute("ANALYZE orders") + make_topic("payments", 6) + + p = Producer({"bootstrap.servers": BOOT, "enable.idempotence": True, "linger.ms": 5, + "batch.size": 262_144, "compression.type": "lz4"}) + for at, oid, cents, version in day.charges: + value = json.dumps({"order_id": oid, "amount_cents": cents, "currency": "USD", + "psp_ref": uuid.UUID(int=rng.getrandbits(128)).hex}) + headers = [("trace-id", uuid.UUID(int=rng.getrandbits(128)).hex[:16].encode()), + ("service-version", version.encode())] + while True: + try: + p.produce("payments", key=str(oid), value=value, headers=headers, timestamp=at) + break + except BufferError: + p.poll(0.05) + p.poll(0) + p.flush(60) + say(f" {len(day.orders):,} orders, {len(day.charges):,} charge events over the day " + f"(6 partitions, idempotent producer, lz4)") + say(" Each event: JSON value {order_id, amount_cents, ...}, headers trace-id and service-version.") + say("\n One helper, created once, decodes the topic into typed columns. It is a function rather") + say(" than a view so a time range reaches kafgres_read() and bounds what it reads:") + sql(SETUP_SQL) + + +# --- the investigation --------------------------------------------------------------------- + +def how_bad(day): + step("1. Support: \"customers say they were charged twice.\" How bad is it?") + rows, _ = sql(""" +SELECT count(*) AS customers_charged_twice, + sum(amount_cents) / 100.0 AS overcharged_usd + FROM payments() + WHERE charge_no > 1;""") + want = (len(day.double), decimal.Decimal(sum(day.double.values())) / 100) + check("how-bad", (rows[0][0], rows[0][1]) == want, + f"{want[0]:,} customers, ${want[1]:,.2f}: exactly the repeat charges that were planted") + + +def when(day): + step("2. When did it happen?") + rows, took = sql(""" +SELECT to_char(date_trunc('hour', charged_at), 'HH24:MI') AS hour, + count(*) AS charges, + count(*) FILTER (WHERE charge_no > 1) AS repeat_charges, + repeat('#', (count(*) FILTER (WHERE charge_no > 1) / 10)::int) AS " " + FROM payments() + GROUP BY 1 ORDER BY 1;""") + spiked = [r for r in rows if r[2] > 0] + check("when", [r[0] for r in spiked] == ["14:00"] and spiked[0][2] == len(day.double), + f"every repeat charge is in the 14:00 hour; the whole day read in {took * 1000:.0f} ms") + day.full_ms = took + + +def what_changed(day): + step("3. What changed at 14:00? Zoom to 13:55-14:25, by minute") + say(" Only the window is read: the time range goes to kafgres_read(), which finds its start") + say(" and end offsets through the time index, as Kafka's offsetsForTimes does.") + rows, took = sql(f""" +SELECT to_char(date_trunc('minute', charged_at), 'HH24:MI') AS minute, + version, + count(*) AS charges, + count(*) FILTER (WHERE charge_no > 1) AS repeat_charges + FROM payments('{day.at("13:55")}', '{day.at("14:25")}') + GROUP BY 1, 2 ORDER BY 1, 2;""", max_rows=40) + good = all(((r[1] == "2.4.1") == ("14:00" <= r[0] < "14:20")) + and ((r[3] > 0) == (r[1] == "2.4.1")) for r in rows) + span = [r[0] for r in rows if r[1] == "2.4.1"] + in_window = sum(r[2] for r in rows) + check("what-changed", good and span[0] == "14:00" and span[-1] == "14:19", + f"2.4.1 ran 14:00 to 14:19 and is the only build that charged twice; read " + f"{in_window:,} of {len(day.charges):,} events in {took * 1000:.0f} ms") + + +def refunds(day): + step("4. Who do we refund?") + sql(f""" +CREATE TABLE refunds_due AS +SELECT o.id AS order_id, o.customer, p.amount_cents, p.trace_id AS duplicate_trace_id + FROM payments('{day.at("14:00")}', '{day.at("14:20")}') p + JOIN orders o ON o.id = p.order_id + WHERE p.charge_no > 1;""") + sql("SELECT * FROM refunds_due ORDER BY order_id LIMIT 5;") + got = dict(sql("SELECT order_id, amount_cents FROM refunds_due", show=False)[0]) + check("refunds", got == day.double, + f"{len(got):,} refunds, one per double-charged order, each for the repeated amount") + + +def consistent(day): + """Not part of the story: SQL must see the log a Kafka consumer sees.""" + consumed = consume_all("payments") + rows, _ = sql("SELECT partition, record_offset, key, value FROM kafgres_read('payments')", + show=False) + from_sql = {(p, o): (bytes(k), bytes(v)) for p, o, k, v in rows} + check("consistent", consumed == from_sql and len(from_sql) == len(day.charges), + f"kafgres_read() returned the same {len(from_sql):,} records, byte for byte, as a " + f"librdkafka consumer", quiet=True) + + +def main(): + version = sql("SELECT kafgres_version()", show=False)[0][0][0] + engine = sql("SHOW kafgres.storage_engine", show=False)[0][0][0] + say(f"kafgres {version}, {engine} engine") + day = Day() + load(day) + for f in (how_bad, when, what_changed, refunds): + f(day) + step("Checks") + consistent(day) + for name, ok, detail in RESULTS: + say(f" {'PASS' if ok else 'FAIL'} {name:<13} {detail}") + sys.exit(0 if all(ok for _, ok, _ in RESULTS) else 1) + + +if __name__ == "__main__": + main() diff --git a/demo/read/requirements.txt b/demo/read/requirements.txt new file mode 100644 index 0000000..7d3d845 --- /dev/null +++ b/demo/read/requirements.txt @@ -0,0 +1,2 @@ +confluent-kafka==2.15.1 +psycopg[binary] diff --git a/docs/reading.md b/docs/reading.md new file mode 100644 index 0000000..eddb899 --- /dev/null +++ b/docs/reading.md @@ -0,0 +1,134 @@ +# Reading records with SQL + +`kafgres_read()` returns a topic's records as rows, so any SQL can filter, join and +aggregate them. It is the read-side counterpart of `kafgres_produce()`. +[demo/read](../demo/read/README.md) investigates a bad deploy with it, one query per step. + +```sql +SELECT record_offset, convert_from(value, 'UTF8')::jsonb AS event + FROM kafgres_read('order-events', 0, from_offset => 1000, to_offset => 2000) + WHERE convert_from(value, 'UTF8')::jsonb ->> 'op' = 'D'; +``` + +## Arguments + +Only `topic` is required, and every bound is optional; the bounds that are given intersect. + +| argument | default | meaning | +|---|---|---| +| `topic` | | the topic to read | +| `partition` | NULL | one partition; NULL reads every partition, in partition order | +| `from_offset` | log start | first offset, inclusive | +| `to_offset` | log end | last offset, exclusive | +| `from_time` | | `timestamptz`: start at the first record stamped at or after it | +| `to_time` | | `timestamptz`: end before the first record stamped at or after it | +| `isolation` | `'read_committed'` | or `'read_uncommitted'` | +| `max_records` | NULL | stop after this many rows | + +Offsets are clamped to what the partition holds, so `from_offset => 0` on a partition +whose retention has moved its start is not an error. The log end is the high watermark, +or under `read_committed` the last stable offset. The range is fixed when the call starts, +and records produced while it runs are not returned. + +The time bounds are looked up in the time index the same way `offsetsForTimes` (ListOffsets) +looks them up, and that lookup is what turns them into an offset range. A record inside that +range is returned even if its own timestamp falls outside the bounds, which a producer +setting its own `CreateTime` out of order can cause. Add a `WHERE timestamp >= ...` clause to +drop such records. + +On the segment engine a time-bounded read waits up to 30 seconds for the broker to start, +which brings old time indexes up to date. A standby has no broker, so there a time-bounded +read is refused; offset bounds still work. + +Kafka offsets are per partition, so offset bounds given with `partition => NULL` apply to +every partition alike. Time bounds are the ones that mean the same thing across partitions. + +## Columns + +| column | type | | +|---|---|---| +| `partition` | `int` | | +| `record_offset` | `bigint` | | +| `timestamp` | `timestamptz` | NULL for a record written without one | +| `timestamp_type` | `text` | `CreateTime` or `LogAppendTime` | +| `key` | `bytea` | NULL for a null key | +| `value` | `bytea` | NULL for a tombstone | +| `header_keys` | `text[]` | in record order; bytes that are not UTF-8, and NUL bytes, which `text` cannot hold, read as U+FFFD | +| `header_values` | `bytea[]` | in the same order; an element is NULL for a null value | +| `leader_epoch` | `int` | the partition leader epoch the batch was written under | +| `producer_id` | `bigint` | NULL outside idempotent and transactional producers | +| `producer_epoch` | `smallint` | | +| `sequence` | `int` | the record's own sequence number, NULL without one | +| `is_transactional` | `boolean` | | + +A record declaring more than 65,536 headers ends the read with an error rather than being +decoded. + +`key` and `value` are bytes, as Kafka stores them. Use `convert_from(value, 'UTF8')` for +text, and add `::jsonb` for JSON. To pair the headers up, unnest both arrays together: + +```sql +SELECT r.record_offset, h.k, convert_from(h.v, 'UTF8') + FROM kafgres_read('order-events', 0) r, + unnest(r.header_keys, r.header_values) AS h(k, v); +``` + +## Composing + +`kafgres_read()` is an ordinary set-returning function. `LATERAL` reads a range from every +partition, here the last 100 records of each: + +```sql +SELECT r.* + FROM kafgres_partition_offsets('order-events') p, + LATERAL kafgres_read('order-events', p.partition, + from_offset => p.high_watermark - 100) r; +``` + +A function in `FROM` runs to completion before the query above it consumes its rows, so a +`LIMIT` over the call does not shorten the scan. Bound the range itself, or pass +`max_records`. + +A long scan is a long statement. While it runs, Postgres cannot prune the rows idempotent +producers update for every batch, and their produce slows. Bounded ranges keep scans short. + +A scan does not hold the log against retention, even inside a longer transaction: each +read releases its locks when it returns, so the broker can still roll and drop segments. +Records retention deletes while a scan runs can be missing from its result; a scan that +retention overtakes carries on from the new log start rather than failing. + +## What `read_committed` hides + +Under `read_committed`, the default, `kafgres_read()` returns what a `read_committed` Kafka +consumer would receive. That excludes records from aborted Kafka transactions and from +rolled-back `kafgres_produce()` calls, and nothing past the last stable offset is returned. +`read_uncommitted` returns them too, as it does for a consumer. Control records +(transaction markers) are never returned. + +`read_committed` needs a `READ COMMITTED` transaction, Postgres's default, and is refused +under `REPEATABLE READ` or `SERIALIZABLE`. Whether a Kafka transaction committed is read at +the time of each read, but those levels pin every read to the transaction's first snapshot, +so a Kafka transaction that aborted after it would read as committed. `read_uncommitted` +works at any level. + +## Who may read + +`kafgres_read()` is not executable by `PUBLIC`. Granting `EXECUTE` is how a role is allowed +to read, and with `kafgres.acls_enabled` off, the default, it allows reading every topic: + +```sql +GRANT EXECUTE ON FUNCTION kafgres_read TO reporting; +``` + +With `kafgres.acls_enabled` on, the role also needs `READ` on the topic in `kafgres_acls`, +as `User:`, the same principal `kafgres_produce()` checks for `WRITE`. A Postgres +superuser is not checked against `kafgres_acls`. + +```sql +SELECT kafgres_add_acl('User:reporting', 'READ', 'TOPIC', 'order-events'); +``` + +No grant on the tables behind the log is needed. They differ between storage engines, so +once both checks pass, `kafgres_read()` reads the log as the bootstrap superuser. Only the +read itself runs that way. Each row is handed back to the query as the calling role, so the +rest of the query runs as the caller. diff --git a/extension/Cargo.lock b/extension/Cargo.lock index 757d6f4..670de24 100644 --- a/extension/Cargo.lock +++ b/extension/Cargo.lock @@ -854,7 +854,7 @@ dependencies = [ [[package]] name = "kafgres" -version = "0.3.3" +version = "0.4.0" dependencies = [ "base64", "kafgres-codec", @@ -873,7 +873,7 @@ dependencies = [ [[package]] name = "kafgres-codec" -version = "0.3.3" +version = "0.4.0" dependencies = [ "bytes", "crc32c", diff --git a/extension/Cargo.toml b/extension/Cargo.toml index e66ef25..e7a690d 100644 --- a/extension/Cargo.toml +++ b/extension/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "kafgres" -version = "0.3.3" +version = "0.4.0" edition = "2021" license = "Elastic-2.0" diff --git a/extension/kafgres.control b/extension/kafgres.control index a22c161..545ad2c 100644 --- a/extension/kafgres.control +++ b/extension/kafgres.control @@ -1,5 +1,5 @@ comment = 'kafgres: a Kafka broker embedded in PostgreSQL' -default_version = '0.3.3' +default_version = '0.4.0' module_pathname = '$libdir/kafgres' relocatable = false superuser = true diff --git a/extension/sql/kafgres--0.3.3--0.4.0.sql b/extension/sql/kafgres--0.3.3--0.4.0.sql new file mode 100644 index 0000000..a3f0c2d --- /dev/null +++ b/extension/sql/kafgres--0.3.3--0.4.0.sql @@ -0,0 +1,32 @@ +-- kafgres 0.3.3 -> 0.4.0: adds kafgres_read() as a fresh install defines it, not PUBLIC. + +CREATE FUNCTION @extschema@.kafgres_read( + "topic" TEXT, + "partition" INT DEFAULT NULL, + "from_offset" bigint DEFAULT NULL, + "to_offset" bigint DEFAULT NULL, + "from_time" timestamp with time zone DEFAULT NULL, + "to_time" timestamp with time zone DEFAULT NULL, + "isolation" TEXT DEFAULT 'read_committed', + "max_records" bigint DEFAULT NULL +) RETURNS TABLE ( + "partition" INT, + "record_offset" bigint, + "timestamp" timestamp with time zone, + "timestamp_type" TEXT, + "key" bytea, + "value" bytea, + "header_keys" TEXT[], + "header_values" bytea[], + "leader_epoch" INT, + "producer_id" bigint, + "producer_epoch" smallint, + "sequence" INT, + "is_transactional" bool +) +SET search_path TO pg_catalog, @extschema@, pg_temp +LANGUAGE c +AS 'MODULE_PATHNAME', 'kafgres_read_wrapper'; + +REVOKE EXECUTE ON FUNCTION @extschema@.kafgres_read(text, int, bigint, bigint, timestamptz, + timestamptz, text, bigint) FROM PUBLIC; diff --git a/extension/src/acl.rs b/extension/src/acl.rs index fc32d8a..f74ce52 100644 --- a/extension/src/acl.rs +++ b/extension/src/acl.rs @@ -330,9 +330,9 @@ thread_local! { static SQL_ACLS: std::cell::RefCell = std::cell::RefCell::new(AclCache::default()); } -/// Whether a SQL caller may write `topic`, as `User:`. A Postgres superuser passes: +/// Whether a SQL caller may `op` on `topic`, as `User:`. A Postgres superuser passes: /// it can rewrite `kafgres_acls` anyway. -pub fn sql_caller_may_write(role: &str, host: &str, topic: &str) -> Result { +pub fn sql_caller_may(op: Operation, role: &str, host: &str, topic: &str) -> Result { let enabled = crate::acls_enabled(); if !enabled || unsafe { pgrx::pg_sys::superuser() } { return Ok(true); @@ -346,17 +346,34 @@ pub fn sql_caller_may_write(role: &str, host: &str, topic: &str) -> Result {} + Ok(false) => pgrx::ereport!( + pgrx::PgLogLevel::ERROR, + pgrx::PgSqlErrorCode::ERRCODE_INSUFFICIENT_PRIVILEGE, + format!("kafgres: User:{role} is not allowed to {} topic {topic:?}", op.as_str()), + format!( + "kafgres.acls_enabled is on and kafgres_acls grants this role no {} on the topic.", + op.as_str() + ) + ), + Err(e) => pgrx::error!("kafgres: could not read kafgres_acls: {e}"), + } } /// Switches the current user for one read; restored on drop, including on unwind. -struct AsBootstrapSuperuser { +pub(crate) struct AsBootstrapSuperuser { user: pgrx::pg_sys::Oid, context: std::os::raw::c_int, } impl AsBootstrapSuperuser { - fn enter() -> Self { + pub(crate) fn enter() -> Self { let mut user = pgrx::pg_sys::InvalidOid; let mut context = 0; // SAFETY: called inside a transaction; the pair is restored by Drop or by abort. diff --git a/extension/src/consume_sql.rs b/extension/src/consume_sql.rs new file mode 100644 index 0000000..844877f --- /dev/null +++ b/extension/src/consume_sql.rs @@ -0,0 +1,260 @@ +//! `kafgres_read()`: a topic's records as rows, for SQL to filter, join and aggregate. + +use std::collections::VecDeque; + +use pgrx::datum::TimestampWithTimeZone; +use pgrx::prelude::*; + +use kafgres_codec::consume::ConsumedRecord; + +use crate::acl::{AsBootstrapSuperuser, Operation}; +use crate::storage::cursor::{self, ReadBounds, RecordCursor}; +use crate::storage::IsolationLevel; + +/// Microseconds from the Unix epoch to the Postgres one, 2000-01-01. +const PG_EPOCH_OFFSET_US: i64 = 946_684_800_000_000; + +/// One partition's records, or every partition's when `partition` is NULL; the range is fixed +/// when the call starts. Gated by `GRANT EXECUTE` and, with ACLs on, `READ` on the topic, then +/// read as the bootstrap superuser, hence the pinned search path. +#[allow(clippy::too_many_arguments, clippy::type_complexity)] +#[pg_extern] +#[search_path(pg_catalog, @extschema@, pg_temp)] +fn kafgres_read( + topic: &str, + partition: default!(Option, "NULL"), + from_offset: default!(Option, "NULL"), + to_offset: default!(Option, "NULL"), + from_time: default!(Option, "NULL"), + to_time: default!(Option, "NULL"), + isolation: default!(&str, "'read_committed'"), + max_records: default!(Option, "NULL"), +) -> TableIterator< + 'static, + ( + name!(partition, Option), + name!(record_offset, Option), + name!(timestamp, Option), + name!(timestamp_type, Option<&'static str>), + name!(key, Option>), + name!(value, Option>), + name!(header_keys, Option>), + name!(header_values, Option>>>), + name!(leader_epoch, Option), + name!(producer_id, Option), + name!(producer_epoch, Option), + name!(sequence, Option), + name!(is_transactional, Option), + ), +> { + let isolation = match isolation { + "read_committed" => IsolationLevel::ReadCommitted, + "read_uncommitted" => IsolationLevel::ReadUncommitted, + other => error!( + "kafgres: isolation must be 'read_committed' or 'read_uncommitted', not {other:?}" + ), + }; + // REPEATABLE READ fixes the snapshot, so a Kafka transaction aborted since reads committed. + // SAFETY: a backend global, set for the transaction. + let fixed_snapshot = unsafe { pg_sys::XactIsoLevel } >= pg_sys::XACT_REPEATABLE_READ as i32; + if fixed_snapshot && matches!(isolation, IsolationLevel::ReadCommitted) { + pgrx::ereport!( + pgrx::PgLogLevel::ERROR, + pgrx::PgSqlErrorCode::ERRCODE_FEATURE_NOT_SUPPORTED, + "kafgres: kafgres_read() with isolation 'read_committed' needs a READ COMMITTED transaction", + "Run it under READ COMMITTED, or pass isolation => 'read_uncommitted'." + ); + } + let limit = match max_records { + Some(n) if n < 0 => error!("kafgres: max_records must not be negative"), + Some(n) => usize::try_from(n).unwrap_or(usize::MAX), + None => usize::MAX, + }; + let bounds = ReadBounds { + from_offset, + to_offset, + from_time: from_time.map(epoch_ms_ceil), + to_time: to_time.map(epoch_ms_ceil), + }; + + // Who is asking, before anything runs as someone else. + let (role, host) = caller(); + crate::acl::sql_require(Operation::Read, &role, &host, topic); + + let (topic_id, partitions) = { + let _owner = AsBootstrapSuperuser::enter(); + let topic_id = match crate::meta::topic_id_by_name(topic) { + Ok(Some(id)) => id, + Ok(None) => error!("kafgres: no such topic {topic:?}"), + Err(e) => error!("kafgres: {e}"), + }; + let count = + crate::meta::partition_count(topic_id).unwrap_or_else(|e| error!("kafgres: {e}")); + (topic_id, count) + }; + let selected: Vec = match partition { + Some(p) if !(0..partitions).contains(&p) => { + error!("kafgres: topic {topic:?} has no partition {p}") + } + Some(p) => vec![p], + None => (0..partitions).collect(), + }; + + // Until the broker starts, a segment may still have a pre-0.3.0 time index, which + // answers timestamp lookups wrongly. Offsets need no wait. + let by_time = bounds.from_time.is_some() || bounds.to_time.is_some(); + if by_time && crate::storage_engine_guc() == "segment" { + // No broker on a standby, so the wait would only time out. + // SAFETY: reads shared state; any backend may call it. + if unsafe { pg_sys::RecoveryInProgress() } { + pgrx::ereport!( + pgrx::PgLogLevel::ERROR, + pgrx::PgSqlErrorCode::ERRCODE_FEATURE_NOT_SUPPORTED, + "kafgres: kafgres_read() time bounds are not available on a standby with the segment engine", + "Use from_offset and to_offset here, or run the time-bounded read on the primary." + ); + } + crate::server::wait_for_epochs(); + } + + let cursors = { + let _owner = AsBootstrapSuperuser::enter(); + selected + .into_iter() + .map(|p| { + let store = crate::storage::open(); + let range = crate::dbtx::released(|| { + cursor::resolve(&*store, topic_id, p, &bounds, isolation) + }) + .unwrap_or_else(|e| error!("kafgres: partition {p}: {e}")); + RecordCursor::new(store, topic_id, p, range, isolation) + }) + .collect() + }; + + let rows = Rows { + cursors, + topic: topic.to_string(), + }; + TableIterator::new(rows.take(limit).map(|(partition, r)| { + let (header_keys, header_values) = r + .headers + .into_iter() + .map(|(k, v)| (header_key(&k), v.map(|v| v.to_vec()))) + .unzip(); + ( + Some(partition), + Some(r.offset), + timestamptz(r.timestamp), + Some(if r.log_append_time { + "LogAppendTime" + } else { + "CreateTime" + }), + r.key.map(|k| k.to_vec()), + r.value.map(|v| v.to_vec()), + Some(header_keys), + Some(header_values), + (r.leader_epoch >= 0).then_some(r.leader_epoch), + (r.producer_id >= 0).then_some(r.producer_id), + (r.producer_epoch >= 0).then_some(r.producer_epoch), + (r.sequence >= 0).then_some(r.sequence), + Some(r.is_transactional), + ) + })) +} + +/// Each step reads as the bootstrap superuser; rows reach the query as the caller. +struct Rows { + cursors: VecDeque, + topic: String, +} + +impl Iterator for Rows { + type Item = (i32, ConsumedRecord); + + fn next(&mut self) -> Option { + let _owner = AsBootstrapSuperuser::enter(); + loop { + let cursor = self.cursors.front_mut()?; + match cursor.next() { + Some(Ok(record)) => return Some((cursor.partition_index(), record)), + Some(Err(e)) => error!( + "kafgres: reading {:?} partition {}: {e}", + self.topic, + cursor.partition_index() + ), + None => { + self.cursors.pop_front(); + } + } + } + } +} + +/// A header key as `text`: invalid UTF-8 and NUL, which `text` cannot hold, read as U+FFFD. +fn header_key(key: &[u8]) -> String { + String::from_utf8_lossy(key).replace('\0', "\u{FFFD}") +} + +/// The calling role and its address, as `kafgres_produce()` names them to the ACLs. +fn caller() -> (String, String) { + // Not `Spi::get_two`, which assigns the caller's transaction an xid. + match Spi::connect(|client| { + client + .select( + "SELECT current_user::text, COALESCE(host(inet_client_addr()), 'localhost')", + Some(1), + &[], + )? + .first() + .get_two::() + }) { + Ok((role, host)) => (role.unwrap_or_default(), host.unwrap_or_default()), + Err(e) => error!("kafgres: {e}"), + } +} + +/// Rounded up, so a bound between two milliseconds excludes the earlier one either way. +fn epoch_ms_ceil(ts: TimestampWithTimeZone) -> i64 { + let pg_us: i64 = ts.into(); + match pg_us { + i64::MIN => i64::MIN, + i64::MAX => i64::MAX, + us => { + us.saturating_add(PG_EPOCH_OFFSET_US).div_euclid(1000) + + i64::from(us.rem_euclid(1000) != 0) + } + } +} + +/// NULL for a record with no timestamp, which Kafka writes as -1. +fn timestamptz(ms: i64) -> Option { + if ms < 0 { + return None; + } + ms.checked_mul(1000) + .and_then(|us| us.checked_sub(PG_EPOCH_OFFSET_US)) + .and_then(|us| TimestampWithTimeZone::try_from(us).ok()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_keys_never_hold_nul_or_invalid_utf8() { + assert_eq!(header_key(b"trace-id"), "trace-id"); + assert_eq!(header_key(b"a\0b"), "a\u{FFFD}b"); + assert_eq!(header_key(&[b'a', 0xff]), "a\u{FFFD}"); + } + + #[test] + fn timestamp_bounds_round_up_to_the_millisecond() { + let at = |us: i64| TimestampWithTimeZone::try_from(us - PG_EPOCH_OFFSET_US).unwrap(); + assert_eq!(epoch_ms_ceil(at(10_000_000)), 10_000); + assert_eq!(epoch_ms_ceil(at(10_000_001)), 10_001); + assert_eq!(epoch_ms_ceil(at(10_000_999)), 10_001); + assert_eq!(epoch_ms_ceil(at(-1)), 0); + } +} diff --git a/extension/src/dbtx.rs b/extension/src/dbtx.rs index 9aee2bd..9e934e7 100644 --- a/extension/src/dbtx.rs +++ b/extension/src/dbtx.rs @@ -130,6 +130,42 @@ pub fn atomically_coded( result } +/// Run a read in a subtransaction rolled back after it, releasing its relation locks: held to +/// the caller's transaction end, they block the table engine's segment roll. `f` must not write, +/// and must return nothing allocated in Postgres memory: the rollback frees its context. +pub fn released(f: impl FnOnce() -> T) -> T { + use pgrx::pg_sys; + use pgrx::pg_sys::pg_try::PgTryBuilder; + + struct Outer(pg_sys::MemoryContext, pg_sys::ResourceOwner); + impl Outer { + /// SAFETY: inside the subtransaction this was taken before. + unsafe fn restore(&self) { + pg_sys::RollbackAndReleaseCurrentSubTransaction(); + pg_sys::MemoryContextSwitchTo(self.0); + pg_sys::CurrentResourceOwner = self.1; + } + } + // SAFETY: called inside a transaction; the rollback below runs on both paths. + let outer = unsafe { + let outer = Outer(pg_sys::CurrentMemoryContext, pg_sys::CurrentResourceOwner); + pg_sys::BeginInternalSubTransaction(std::ptr::null_mut()); + AssertUnwindSafe(outer) + }; + let out = PgTryBuilder::new(AssertUnwindSafe(f)) + .catch_others(|caught| { + // `caught` holds a copy, so Postgres's pending error is flushed before the rollback. + unsafe { + pg_sys::FlushErrorState(); + outer.restore(); + } + caught.rethrow() + }) + .execute(); + unsafe { outer.restore() }; + out +} + fn with_subtransaction(f: impl FnOnce() -> Result) -> Result { atomically(f, |message| HandlerError::Internal(format!("query aborted: {message}"))) } diff --git a/extension/src/fresh.rs b/extension/src/fresh.rs new file mode 100644 index 0000000..eb0f51c --- /dev/null +++ b/extension/src/fresh.rs @@ -0,0 +1,175 @@ +//! Reads on a snapshot taken for each query, which assign the caller's transaction no xid. +//! +//! They see everything committed so far plus the transaction's own writes, as the broker's +//! reads do. pgrx has no such read: `Spi::get_one*` assigns an xid even to a SELECT, and +//! `SpiClient::select` reads the statement's snapshot while the current subtransaction has no +//! xid. Under REPEATABLE READ every query gets the transaction's snapshot. + +use std::marker::PhantomData; +use std::num::NonZeroUsize; + +use pgrx::datum::{DatumWithOid, TryFromDatumError}; +use pgrx::pg_sys; +use pgrx::spi::{Error, SpiErrorCodes}; +use pgrx::{FromDatum, IntoDatum}; + +/// A column type these reads decode to. Owned only: a borrowed `&str` or `&[u8]` would point +/// into SPI memory that is freed before the caller sees it. +pub trait Owned: FromDatum + IntoDatum {} +impl Owned for bool {} +impl Owned for i32 {} +impl Owned for i64 {} +impl Owned for String {} +impl Owned for Vec {} + +/// The first column of the first row; `None` for no rows or NULL. +pub fn one(sql: &str, args: &[DatumWithOid<'_>]) -> Result, Error> { + rows(sql, args, |mut rows| match rows.next() { + Some(row) => row.get(1), + None => Ok(None), + }) +} + +/// Runs `sql` and hands its rows to `f`. They are freed when `f` returns. +pub fn rows( + sql: &str, + args: &[DatumWithOid<'_>], + f: impl FnOnce(Rows<'_>) -> Result, +) -> Result { + let query = std::ffi::CString::new(sql).expect("SQL text contains no NUL"); + let mut types: Vec = args.iter().map(|a| a.oid()).collect(); + let mut values: Vec = args + .iter() + .map(|a| { + a.datum() + .map_or(pg_sys::Datum::from(0usize), |d| d.sans_lifetime()) + }) + .collect(); + let nulls: Vec = args + .iter() + .map(|a| if a.datum().is_some() { b' ' } else { b'n' } as std::ffi::c_char) + .collect(); + + pgrx::Spi::connect(|_| { + // SAFETY: SPI is connected for this closure, which outlives `f`'s use of the rows. + // `read_only = false` makes SPI take a snapshot for this query. + let (table, len) = unsafe { + let status = pg_sys::SPI_execute_with_args( + query.as_ptr(), + args.len() as i32, + types.as_mut_ptr(), + values.as_mut_ptr(), + nulls.as_ptr(), + false, + 0, + ); + if status < 0 { + return Err( + SpiErrorCodes::try_from(status).map_or(Error::NoTupleTable, Error::SpiError) + ); + } + (pg_sys::SPI_tuptable, pg_sys::SPI_processed as usize) + }; + if table.is_null() { + return Err(Error::NoTupleTable); + } + f(Rows { + table, + len, + next: 0, + _spi: PhantomData, + }) + }) +} + +/// The rows of one query, in order. +pub struct Rows<'spi> { + table: *mut pg_sys::SPITupleTable, + len: usize, + next: usize, + _spi: PhantomData<&'spi ()>, +} + +impl<'spi> Iterator for Rows<'spi> { + type Item = Row<'spi>; + + fn next(&mut self) -> Option> { + if self.next == self.len { + return None; + } + // SAFETY: `next` is in bounds, and the table lives as long as `'spi`. + let row = unsafe { + Row { + tuple: *(*self.table).vals.add(self.next), + desc: (*self.table).tupdesc, + _spi: PhantomData, + } + }; + self.next += 1; + Some(row) + } +} + +/// One row. Columns number from 1. +pub struct Row<'spi> { + tuple: pg_sys::HeapTuple, + desc: pg_sys::TupleDesc, + _spi: PhantomData<&'spi ()>, +} + +impl Row<'_> { + /// Column `column` as `T`, or an error if the column is not of `T`'s type. Values are copied + /// out of SPI's memory, so a long scan does not accumulate them in the transaction. + pub fn get(&self, column: usize) -> Result, Error> { + // SAFETY: the tuple and descriptor are SPI's and live as long as `'spi`; `column` is + // checked against the descriptor before either is read. + unsafe { + if column == 0 || column > (*self.desc).natts as usize { + let n = NonZeroUsize::new(column).unwrap_or(NonZeroUsize::MIN); + return Err(TryFromDatumError::NoSuchAttributeNumber(n).into()); + } + let column = column as i32; + let mut null = false; + let datum = pg_sys::SPI_getbinval(self.tuple, self.desc, column, &mut null); + Ok(T::try_from_datum( + datum, + null, + pg_sys::SPI_gettypeid(self.desc, column), + )?) + } + } +} + +#[cfg(any(test, feature = "pg_test"))] +#[pgrx::pg_schema] +mod tests { + use pgrx::prelude::*; + + #[pg_test] + fn reads_assign_no_xid() { + assert_eq!(super::one::("SELECT $1", &[7i64.into()]), Ok(Some(7))); + let no_xid = super::one::("SELECT pg_current_xact_id_if_assigned() IS NULL", &[]); + assert_eq!(no_xid, Ok(Some(true))); + } + + /// Even in a subtransaction with no xid, where `SpiClient::select` misses them. + #[pg_test] + fn reads_see_the_transactions_own_writes() { + Spi::run("CREATE TEMP TABLE fresh_own_writes (x int)").unwrap(); + Spi::run("INSERT INTO fresh_own_writes VALUES (1), (2)").unwrap(); + let count = crate::dbtx::released(|| { + super::one::("SELECT count(*) FROM fresh_own_writes", &[]) + }); + assert_eq!(count, Ok(Some(2))); + } + + #[pg_test] + fn a_column_of_another_type_is_an_error() { + assert!(super::one::("SELECT 'x'::text", &[]).is_err()); + assert!(super::rows("SELECT 1::bigint", &[], |mut r| r + .next() + .unwrap() + .get::(2)) + .is_err()); + } +} diff --git a/extension/src/lib.rs b/extension/src/lib.rs index 87c7b9c..2751b9c 100644 --- a/extension/src/lib.rs +++ b/extension/src/lib.rs @@ -12,8 +12,10 @@ pub mod archive; pub mod cdc; pub mod decoding; pub mod config; +pub mod consume_sql; pub mod clock; mod dbtx; +mod fresh; mod ddl; pub mod group; pub mod handlers; diff --git a/extension/src/meta.rs b/extension/src/meta.rs index b470e9c..ee4d13e 100644 --- a/extension/src/meta.rs +++ b/extension/src/meta.rs @@ -156,7 +156,7 @@ pub fn all_partitions() -> Result, spi::Error> { pub fn topic_id_by_name(name: &str) -> Result, spi::Error> { Ok( - Spi::get_one_with_args::( + crate::fresh::one::( // Scalar subquery: NULL when absent — a bare WHERE returns zero rows, which pgrx reports as an error. "SELECT (SELECT topic_id::int FROM kafgres_topics WHERE name = $1)", &[name.into()], @@ -317,7 +317,7 @@ pub fn create_topic( } pub fn partition_count(topic_id: u32) -> Result { - Ok(Spi::get_one_with_args::( + Ok(crate::fresh::one::( "SELECT (SELECT num_partitions FROM kafgres_topics WHERE topic_id = $1::oid)", &[(topic_id as i32).into()], )? diff --git a/extension/src/produce_sql.rs b/extension/src/produce_sql.rs index 983e90c..3b6c32e 100644 --- a/extension/src/produce_sql.rs +++ b/extension/src/produce_sql.rs @@ -69,16 +69,7 @@ fn kafgres_produce( "kafgres_produce() needs INSERT on kafgres_markers." ); } - match crate::acl::sql_caller_may_write(&role, &host, topic) { - Ok(true) => {} - Ok(false) => pgrx::ereport!( - pgrx::PgLogLevel::ERROR, - pgrx::PgSqlErrorCode::ERRCODE_INSUFFICIENT_PRIVILEGE, - format!("kafgres: User:{role} is not allowed to WRITE topic {topic:?}"), - "kafgres.acls_enabled is on and kafgres_acls grants this role no WRITE on the topic." - ), - Err(e) => error!("kafgres: could not read kafgres_acls: {e}"), - } + crate::acl::sql_require(crate::acl::Operation::Write, &role, &host, topic); // murmur2 on the key, matching what Kafka clients do, so a key produced through SQL // lands on the partition a client would have chosen for it. diff --git a/extension/src/storage/cursor.rs b/extension/src/storage/cursor.rs new file mode 100644 index 0000000..d53f454 --- /dev/null +++ b/extension/src/storage/cursor.rs @@ -0,0 +1,242 @@ +//! A partition's records as a consumer sees them, over [`LogStore::read`]. Engine-agnostic. + +use std::collections::VecDeque; +use std::ops::Range; + +use kafgres_codec::bytes::Bytes; +use kafgres_codec::consume::{consumed_records, AbortFilter, ConsumedRecord}; +use kafgres_codec::records::{BatchIter, RecordBatch}; + +use super::{IsolationLevel, LogStore, StoreError, StoreResult, TopicId}; + +/// Asked of the store per read; a single larger batch still comes back whole. +const CHUNK_BYTES: usize = 1024 * 1024; + +/// Headers a record may declare. Decoded headers cost ~32x their minimum wire size, so an +/// unlimited count lets one 64 MiB batch exhaust the backend (I8). +pub const MAX_HEADERS: usize = 65_536; + +type Records = Box>>; + +/// Optional bounds that intersect. Times are epoch milliseconds resolved to offsets as +/// `offsetsForTimes` does, so an out-of-order timestamp inside the range is still returned. +#[derive(Debug, Clone, Copy, Default)] +pub struct ReadBounds { + pub from_offset: Option, + /// Exclusive. + pub to_offset: Option, + pub from_time: Option, + /// Exclusive. + pub to_time: Option, +} + +/// The offsets `bounds` select, clamped to what is readable now: from the log start to the +/// high watermark, or to the last stable offset under `read_committed`. +pub fn resolve( + store: &dyn LogStore, + topic: TopicId, + partition: i32, + bounds: &ReadBounds, + isolation: IsolationLevel, +) -> StoreResult> { + let log_start = store.log_start_offset(topic, partition)?; + let log_end = match isolation { + IsolationLevel::ReadCommitted => store.last_stable_offset(topic, partition)?, + IsolationLevel::ReadUncommitted => store.high_watermark(topic, partition)?, + }; + // Negative timestamps are the store's sentinels. + let by_time = |ms: i64| -> StoreResult { + Ok(store + .offset_for_timestamp(topic, partition, ms.max(0))? + .map_or(log_end, |(offset, _)| offset)) + }; + + let mut from = bounds.from_offset.unwrap_or(log_start).max(log_start); + let mut to = bounds.to_offset.unwrap_or(log_end).min(log_end); + if let Some(ms) = bounds.from_time { + from = from.max(by_time(ms)?); + } + if let Some(ms) = bounds.to_time { + to = to.min(by_time(ms)?); + } + Ok(from..to.max(from)) +} + +/// A partition's records in `range`, oldest first. Never control batches; under +/// `read_committed`, no aborted transactions. +pub struct RecordCursor { + store: Box, + topic: TopicId, + partition: i32, + isolation: IsolationLevel, + range: Range, + next: i64, + /// The current chunk's batches not yet decoded, and the filter judging them. + batches: VecDeque, + filter: Option, + /// The batch being decoded, one record at a time (I8), and its base offset. + records: Option<(Records, i64)>, +} + +impl RecordCursor { + pub fn new( + store: Box, + topic: TopicId, + partition: i32, + range: Range, + isolation: IsolationLevel, + ) -> Self { + RecordCursor { + store, + topic, + partition, + isolation, + next: range.start, + range, + batches: VecDeque::new(), + filter: None, + records: None, + } + } + + pub fn partition_index(&self) -> i32 { + self.partition + } + + fn read_chunk(&mut self) -> StoreResult<()> { + let slice = loop { + let read = crate::dbtx::released(|| { + self.store.read( + self.topic, + self.partition, + self.next, + CHUNK_BYTES, + self.isolation, + ) + }); + match read { + // Retention moved the log start past the cursor since the range was fixed: + // what it deleted is missing from the result, as the docs say. Each pass + // moves `next` forward, so this ends. + Err(StoreError::OffsetOutOfRange) => { + let start = crate::dbtx::released(|| { + self.store.log_start_offset(self.topic, self.partition) + })?; + // Not retention: the offset is past the log end (a truncated follower). + if start <= self.next { + return Err(StoreError::OffsetOutOfRange); + } + self.next = start.min(self.range.end); + if self.next >= self.range.end { + return Ok(()); + } + } + other => break other?, + } + }; + // A batch reaching where the aborted list was cut short is read again from there. + let judged_below = slice.aborted_cut_at.unwrap_or(i64::MAX); + let served = !slice.bytes.is_empty(); + + let mut advanced = self.next; + for batch in BatchIter::new(Bytes::from(slice.bytes)) { + let batch = batch.map_err(|e| { + StoreError::Io(format!("undecodable batch after offset {advanced}: {e}")) + })?; + if batch.base_offset() >= self.range.end { + advanced = self.range.end; + break; + } + if batch.last_offset() >= judged_below { + break; + } + advanced = batch.last_offset() + 1; + self.batches.push_back(batch); + } + + if self.batches.is_empty() && served && advanced < self.range.end { + return Err(StoreError::Io(format!( + "more aborted transactions overlap offset {} than one read can list", + self.next + ))); + } + // Nothing served: resume where the store stopped looking, or the range is done. + self.next = if served { + advanced + } else if slice.next_offset > self.next { + slice.next_offset.min(self.range.end) + } else { + self.range.end + }; + // Rebuilt per chunk, like a Fetch response: each read lists every abort overlapping its + // start, since an entry is kept until the log start passes its marker. + self.filter = match self.isolation { + IsolationLevel::ReadCommitted => Some(AbortFilter::new( + slice + .aborted + .iter() + .map(|a| (a.producer_id, a.first_offset)), + )), + IsolationLevel::ReadUncommitted => None, + }; + Ok(()) + } + + fn start(&mut self, batch: RecordBatch) -> StoreResult<()> { + let keep = match &mut self.filter { + Some(filter) => filter.admit(&batch), + None => !batch.is_control(), + }; + if keep { + let records = consumed_records(&batch, MAX_HEADERS) + .map_err(|e| undecodable(batch.base_offset(), e))?; + self.records = Some((Box::new(records), batch.base_offset())); + } + Ok(()) + } + + fn step(&mut self) -> StoreResult> { + loop { + if let Some((records, base)) = &mut self.records { + match records.next() { + Some(Ok(record)) if record.offset >= self.range.end => self.records = None, + // The first batch may begin before the range. + Some(Ok(record)) if record.offset < self.range.start => {} + Some(Ok(record)) => return Ok(Some(record)), + Some(Err(e)) => return Err(undecodable(*base, e)), + None => self.records = None, + } + continue; + } + if let Some(batch) = self.batches.pop_front() { + self.start(batch)?; + continue; + } + if self.next >= self.range.end { + return Ok(None); + } + self.read_chunk()?; + } + } +} + +fn undecodable(base: i64, e: kafgres_codec::records::BatchError) -> StoreError { + StoreError::Io(format!("undecodable batch at offset {base}: {e}")) +} + +impl Iterator for RecordCursor { + type Item = StoreResult; + + /// An error ends the cursor. + fn next(&mut self) -> Option { + match self.step() { + Ok(record) => record.map(Ok), + Err(e) => { + self.next = self.range.end; + self.batches.clear(); + self.records = None; + Some(Err(e)) + } + } + } +} diff --git a/extension/src/storage/mod.rs b/extension/src/storage/mod.rs index b857393..2d8a8d2 100644 --- a/extension/src/storage/mod.rs +++ b/extension/src/storage/mod.rs @@ -1,5 +1,6 @@ //! Storage boundary: every read or write of log data goes through [`LogStore`]. +pub mod cursor; pub mod pmeta; pub mod segment; pub mod table; @@ -356,6 +357,9 @@ pub struct FetchSlice { /// Last Stable Offset — equal to `high_watermark` when no transaction is in flight. pub last_stable_offset: i64, pub aborted: Vec, + /// Set when `aborted` hit its cap: transactions first written at or after this offset + /// may be missing from it, so a batch reaching it cannot be judged from this slice. + pub aborted_cut_at: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/extension/src/storage/pmeta.rs b/extension/src/storage/pmeta.rs index 445b087..31a6164 100644 --- a/extension/src/storage/pmeta.rs +++ b/extension/src/storage/pmeta.rs @@ -251,38 +251,36 @@ pub fn committed_markers( from: i64, to: i64, ) -> StoreResult<(std::collections::HashSet, i64)> { - Spi::connect(|client| { - let rows = client.select( - "SELECT base_offset FROM kafgres_markers - WHERE topic_id = $1::oid AND partition = $2 - AND base_offset >= $3 AND base_offset < $4 - ORDER BY base_offset LIMIT $5", - None, - &[ - (topic as i32).into(), - partition.into(), - from.into(), - to.into(), - (MARKERS_PER_FETCH as i64).into(), - ], - )?; - let mut out = std::collections::HashSet::new(); - let mut highest = from; - for row in rows { - if let Some(b) = row.get::(1)? { - out.insert(b); - highest = highest.max(b); + crate::fresh::rows( + "SELECT base_offset FROM kafgres_markers + WHERE topic_id = $1::oid AND partition = $2 + AND base_offset >= $3 AND base_offset < $4 + ORDER BY base_offset LIMIT $5", + &[ + (topic as i32).into(), + partition.into(), + from.into(), + to.into(), + (MARKERS_PER_FETCH as i64).into(), + ], + |rows| { + let mut out = std::collections::HashSet::new(); + let mut highest = from; + for row in rows { + if let Some(b) = row.get::(1)? { + out.insert(b); + highest = highest.max(b); + } } - } - // Under the cap the range is covered exhaustively; at the cap, knowledge stops - // after the highest marker loaded. - let ceiling = if out.len() < MARKERS_PER_FETCH { - to - } else { - (highest + 1).min(to) - }; - Ok::<_, spi::Error>((out, ceiling)) - }) + // At the cap, the range is known only up to the highest marker loaded. + let ceiling = if out.len() < MARKERS_PER_FETCH { + to + } else { + (highest + 1).min(to) + }; + Ok((out, ceiling)) + }, + ) .map_err(spi_err) } @@ -372,7 +370,7 @@ pub fn aborted_at(topic: TopicId, partition: i32, producer_id: i64, offset: i64) /// The Last Stable Offset contributed by in-flight Kafka transactions, if any; `None` pub fn kafka_txn_lso(topic: TopicId, partition: i32) -> StoreResult> { - Spi::get_one_with_args::( + crate::fresh::one::( "SELECT (SELECT MIN(p.first_offset) FROM kafgres_txn_partitions p JOIN kafgres_txns t ON t.producer_id = p.producer_id @@ -459,40 +457,45 @@ pub fn record_aborted_txn( /// How many aborted transactions one Fetch reports, per partition. A row count, so it pub const MAX_ABORTED_PER_FETCH: i64 = 1_000; -/// Aborted transactions overlapping `[from, to)`, oldest first. `last_offset >= from` +/// Aborted transactions overlapping `[from, to)`, oldest first, at most +/// `MAX_ABORTED_PER_FETCH`. Second, where the cap cut the list short: every entry starting +/// below it is listed, and entries starting there or later are not. pub fn aborted_txns( topic: TopicId, partition: i32, from: i64, to: i64, -) -> StoreResult> { - Spi::connect(|client| { - let rows = client.select( - "SELECT producer_id, first_offset FROM kafgres_txn_aborted - WHERE topic_id = $1::oid AND partition = $2 - AND last_offset >= $3 AND first_offset < $4 - ORDER BY first_offset - LIMIT $5", - None, - &[ - (topic as i32).into(), - partition.into(), - from.into(), - to.into(), - MAX_ABORTED_PER_FETCH.into(), - ], - )?; - let mut out = Vec::new(); - for row in rows { - if let (Some(p), Some(f)) = (row.get::(1)?, row.get::(2)?) { - out.push(super::AbortedTxn { - producer_id: p, - first_offset: f, - }); +) -> StoreResult<(Vec, Option)> { + crate::fresh::rows( + "SELECT producer_id, first_offset FROM kafgres_txn_aborted + WHERE topic_id = $1::oid AND partition = $2 + AND last_offset >= $3 AND first_offset < $4 + ORDER BY first_offset + LIMIT $5 + 1", + &[ + (topic as i32).into(), + partition.into(), + from.into(), + to.into(), + MAX_ABORTED_PER_FETCH.into(), + ], + |rows| { + let mut out = Vec::new(); + for row in rows { + if let (Some(p), Some(f)) = (row.get::(1)?, row.get::(2)?) { + out.push(super::AbortedTxn { + producer_id: p, + first_offset: f, + }); + } } - } - Ok::<_, spi::Error>(out) - }) + // `first_offset` is unique per partition, so everything below the extra row is in. + let cut = (out.len() as i64 > MAX_ABORTED_PER_FETCH) + .then(|| out.pop().map(|a| a.first_offset)) + .flatten(); + Ok((out, cut)) + }, + ) .map_err(spi_err) } diff --git a/extension/src/storage/segment.rs b/extension/src/storage/segment.rs index 3e079ae..489c5b3 100644 --- a/extension/src/storage/segment.rs +++ b/extension/src/storage/segment.rs @@ -2087,6 +2087,7 @@ impl LogStore for SegmentStore { let mut out: Vec = Vec::new(); let mut next = offset; let mut aborted: Vec = Vec::new(); + let mut aborted_cut_at = None; let mut offset = offset; if offset < ceiling { @@ -2239,7 +2240,10 @@ impl LogStore for SegmentStore { // Kafka's aborts, scoped to what this response returned: bounding by the ceiling rather if matches!(isolation, IsolationLevel::ReadCommitted) { - aborted.extend(pmeta::aborted_txns(topic, partition, offset, next.max(offset + 1))?); + let (listed, cut) = + pmeta::aborted_txns(topic, partition, offset, next.max(offset + 1))?; + aborted.extend(listed); + aborted_cut_at = cut; } } @@ -2250,6 +2254,7 @@ impl LogStore for SegmentStore { log_start_offset: log_start, last_stable_offset: lso, aborted, + aborted_cut_at, }) } diff --git a/extension/src/storage/table.rs b/extension/src/storage/table.rs index 186da71..7947764 100644 --- a/extension/src/storage/table.rs +++ b/extension/src/storage/table.rs @@ -231,6 +231,41 @@ fn stable_bound(topic: TopicId, partition: i32, high_watermark: i64) -> StoreRes .map_or(high_watermark, |o| o.min(high_watermark))) } +/// A batch column copied out in SPI's own context. pgrx's `get::>` detoasts into the +/// transaction's context, so a long statement would hold every batch it read. +fn bytea_column(row: &pgrx::spi::SpiHeapTupleData<'_>, ordinal: usize) -> Result>, spi::Error> { + let Some(RawBytea(datum)) = row.get::(ordinal)? else { + return Ok(None); + }; + // SAFETY: a non-null bytea from the row; the detoasted copy is in SPI's current context, + // freed at SPI_finish, and is copied out before then. + unsafe { + let plain = pg_sys::pg_detoast_datum_packed(datum.cast_mut_ptr()); + let len = pgrx::varlena::varsize_any_exhdr(plain); + let data = pgrx::varlena::vardata_any(plain) as *const u8; + Ok(Some(std::slice::from_raw_parts(data, len).to_vec())) + } +} + +/// A `bytea` datum as SPI holds it, neither detoasted nor copied. +struct RawBytea(pg_sys::Datum); + +impl FromDatum for RawBytea { + unsafe fn from_polymorphic_datum(datum: pg_sys::Datum, is_null: bool, _: pg_sys::Oid) -> Option { + (!is_null).then_some(RawBytea(datum)) + } +} + +impl IntoDatum for RawBytea { + fn into_datum(self) -> Option { + Some(self.0) + } + + fn type_oid() -> pg_sys::Oid { + pg_sys::BYTEAOID + } +} + fn spi_err(e: impl std::fmt::Display) -> StoreError { StoreError::Io(e.to_string()) } @@ -415,12 +450,12 @@ impl LogStore for TableStore { let mut bytes: Vec = Vec::new(); let mut next_offset = offset; let mut aborted: Vec = Vec::new(); + let mut aborted_cut_at = None; if offset < ceiling { - Spi::connect(|client| { // Start at the batch *containing* `offset`, not the first at or past it: resume - let rows = client.select( + crate::fresh::rows( "SELECT base_offset, last_offset, batch FROM kafgres_log WHERE topic_id = $1::oid AND partition = $2 @@ -432,37 +467,36 @@ impl LogStore for TableStore { AND base_offset < $4 ORDER BY base_offset LIMIT 1000", - None, &[ (topic as i32).into(), partition.into(), offset.into(), ceiling.into(), ], - )?; - - for row in rows { - let last: i64 = row.get(2)?.unwrap_or(0); - let blob: Vec = match row.get::>(3)? { - Some(b) => b, - None => continue, - }; - if !bytes.is_empty() && bytes.len() + blob.len() > max_bytes { - break; - } - bytes.extend_from_slice(&blob); - next_offset = last + 1; - if bytes.len() >= max_bytes && ALWAYS_RETURN_FIRST_BATCH { - break; - } - } - Ok::<_, spi::Error>(()) - }) - .map_err(spi_err)?; + |rows| { + for row in rows { + let last: i64 = row.get(2)?.unwrap_or(0); + let Some(blob) = row.get::>(3)? else { + continue; + }; + if !bytes.is_empty() && bytes.len() + blob.len() > max_bytes { + break; + } + bytes.extend_from_slice(&blob); + next_offset = last + 1; + if bytes.len() >= max_bytes && ALWAYS_RETURN_FIRST_BATCH { + break; + } + } + Ok(()) + }, + ) + .map_err(spi_err)?; // Aborted txns from the index, not the batches in hand: a txn spanning fetches reads as committed. if matches!(isolation, IsolationLevel::ReadCommitted) { - aborted = pmeta::aborted_txns(topic, partition, offset, next_offset.max(offset + 1))?; + (aborted, aborted_cut_at) = + pmeta::aborted_txns(topic, partition, offset, next_offset.max(offset + 1))?; } } @@ -473,6 +507,7 @@ impl LogStore for TableStore { log_start_offset: log_start, last_stable_offset: lso, aborted, + aborted_cut_at, }) } @@ -482,7 +517,7 @@ impl LogStore for TableStore { partition: i32, timestamp: i64, ) -> StoreResult> { - let blob = Spi::get_one_with_args::>( + let blob = crate::fresh::one::>( "SELECT (SELECT batch FROM kafgres_log WHERE topic_id = $1::oid AND partition = $2 AND max_timestamp >= $3 ORDER BY base_offset @@ -512,7 +547,7 @@ impl LogStore for TableStore { )?; let mut found: Option> = None; for row in rows { - found = row.get::>(1)?; + found = bytea_column(&row, 1)?; } Ok::<_, spi::Error>(found) }) @@ -522,7 +557,7 @@ impl LogStore for TableStore { } fn high_watermark(&self, topic: TopicId, partition: i32) -> StoreResult { - Spi::get_one_with_args::( + crate::fresh::one::( "SELECT (SELECT next_offset FROM kafgres_partitions WHERE topic_id = $1::oid AND partition = $2)", &[(topic as i32).into(), partition.into()], @@ -553,7 +588,7 @@ impl LogStore for TableStore { } fn log_start_offset(&self, topic: TopicId, partition: i32) -> StoreResult { - Spi::get_one_with_args::( + crate::fresh::one::( "SELECT (SELECT log_start_offset FROM kafgres_partitions WHERE topic_id = $1::oid AND partition = $2)", &[(topic as i32).into(), partition.into()], @@ -1043,7 +1078,7 @@ impl Cleanable { let mut out = Vec::new(); for row in rows { if let (Some(base), Some(blob), Some(ts)) = - (row.get::(1)?, row.get::>(2)?, row.get::(3)?) + (row.get::(1)?, bytea_column(&row, 2)?, row.get::(3)?) { out.push((base, blob, ts)); } diff --git a/scripts/dev/upgrade-test.py b/scripts/dev/upgrade-test.py index d865de5..eb35283 100644 --- a/scripts/dev/upgrade-test.py +++ b/scripts/dev/upgrade-test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Upgrade a populated release install to the current build and check nothing moved. - python3 scripts/dev/upgrade-test.py [--engine segment|table] [--from 0.2.0] [--keep] + python3 scripts/dev/upgrade-test.py [--engine segment|table] [--from 0.3.0] [--keep] Seeds a volume under the tagged release, restarts it on the current build, and checks the same records, group offsets and timestamp lookups before and after @@ -206,7 +206,7 @@ def epoch(): def main(): ap = argparse.ArgumentParser() ap.add_argument("--engine", choices=["segment", "table"], default="segment") - ap.add_argument("--from", dest="old", default="0.2.0") + ap.add_argument("--from", dest="old", default="0.3.0") ap.add_argument("--keep", action="store_true", help="leave the container and volume") ap.add_argument("--no-build", action="store_true") args = ap.parse_args() @@ -235,7 +235,8 @@ def main(): stdin="k1:after-upgrade\n") if args.engine == "segment": sql("SELECT kafgres_produce('up-sql', 'k9', 'after-upgrade')") - assert "pre-0.3.0 .timeindex" in logs(), "the time index was not migrated" + if tuple(map(int, args.old.split("."))) < (0, 3, 0): + assert "pre-0.3.0 .timeindex" in logs(), "the time index was not migrated" step("ALTER EXTENSION kafgres UPDATE") sql("ALTER EXTENSION kafgres UPDATE") @@ -256,9 +257,16 @@ def main(): missing = set(before["records"][topic]) - set(after["records"][topic]) assert not missing, f"{topic} lost {len(missing)} records after the update" assert after["groups"] == before["groups"] + # kafgres_read() arrives with 0.4.0, not executable by PUBLIC, and reads the old log. + assert sql("SELECT has_function_privilege('public', 'kafgres_read(text,int,bigint,bigint," + "timestamptz,timestamptz,text,bigint)', 'EXECUTE')") == "f", \ + "the update left kafgres_read() public" + read = sql("SELECT count(*) FROM kafgres_read('up-plain')") + assert int(read) >= len(before["records"]["up-plain"]), \ + f"kafgres_read() saw {read} of {len(before['records']['up-plain'])} records" assert after["lookup"] == before["exact"] - step("compacting what 0.2.0 wrote") + step(f"compacting what {args.old} wrote") sql("SELECT kafgres_enforce_retention()") compacted = read_topic("up-compact") kept = latest(compacted) diff --git a/tests/integration/produce_fetch/test_sql_read.py b/tests/integration/produce_fetch/test_sql_read.py new file mode 100644 index 0000000..bcb890f --- /dev/null +++ b/tests/integration/produce_fetch/test_sql_read.py @@ -0,0 +1,356 @@ +"""`kafgres_read()`: a topic's records as rows. + +The reference for what a row should hold is a real consumer reading the same partition, +so most of these compare against kcat rather than against what this file produced. +""" + +import subprocess +import time + +import pytest + +from conftest import BROKER_HOST, BROKER_PORT, PSQL, sql +from support.docker import CLIENTS + +BROKER = f"{BROKER_HOST}:{BROKER_PORT}" +ROLE = "sqlreader" + + +def kcat(*args, stdin=None, timeout=120): + return subprocess.run( + ["docker", "run", "--rm", "-i", "--network", "host", CLIENTS, "kcat", "-b", BROKER, + *args], + input=stdin, capture_output=True, text=True, timeout=timeout, + ) + + +def produce(topic, lines, *extra): + out = kcat("-t", topic, "-P", *extra, stdin="".join(f"{l}\n" for l in lines)) + assert out.returncode == 0, out.stderr + + +def consumed(topic, isolation="read_committed", partition=0): + """`{offset: value}` as a Kafka consumer sees the partition.""" + out = kcat("-t", topic, "-p", str(partition), "-C", "-e", "-q", "-o", "beginning", + "-X", f"isolation.level={isolation}", "-f", "%o\t%s\n") + got = {} + for line in out.stdout.splitlines(): + if "\t" in line: + off, value = line.split("\t", 1) + got[int(off)] = value + return got + + +def rows(query): + """Tab-separated rows of `query`.""" + out = sql(f"COPY ({query}) TO STDOUT") + return [line.split("\t") for line in out.splitlines() if line] + + +def read(topic, *args, isolation=None): + """`{offset: value}` from `kafgres_read()`, the values decoded as UTF-8.""" + named = list(args) + ([f"isolation => '{isolation}'"] if isolation else []) + call = ", ".join([f"'{topic}'"] + named) + return {int(o): v for o, v in rows( + f"SELECT record_offset, convert_from(value, 'UTF8') FROM kafgres_read({call})")} + + +def script(statements): + """Each statement's output lines, command tags included. Before Postgres 15, `psql -c` + prints only the last statement's, so the script goes in on stdin.""" + out = subprocess.run( + PSQL[:-1] + ["-tA", "-v", "ON_ERROR_STOP=1"], + input=statements, capture_output=True, text=True, timeout=60, + ) + if out.returncode != 0: + raise RuntimeError(f"psql failed: {out.stderr.strip()}") + return out.stdout.splitlines() + + +def offset_at(topic, when_ms): + """`kcat -Q`, which is `offsetsForTimes`.""" + out = kcat("-Q", "-t", f"{topic}:0:{when_ms}") + for line in out.stdout.splitlines(): + if "offset" in line: + return int(line.rsplit(" ", 1)[1]) + raise AssertionError(f"no offset in {out.stdout!r} {out.stderr!r}") + + +def ts(ms): + return f"to_timestamp({ms} / 1000.0)" + + +@pytest.fixture +def topic(request): + name = f"sqlread-{request.node.name.replace('_', '-')[5:40]}" + sql(f"SELECT kafgres_drop_topic('{name}')") + sql(f"SELECT kafgres_create_topic('{name}', 1)") + yield name + sql(f"SELECT kafgres_drop_topic('{name}')") + + +def test_rows_match_what_a_consumer_reads(topic): + produce(topic, [f"value-{i}" for i in range(50)]) + produce(topic, [f"lz4-{i}" for i in range(50)], "-z", "lz4") + expected = consumed(topic) + assert len(expected) == 100 + assert read(topic) == expected + + +def test_keys_headers_and_metadata_ride_along(topic): + produce(topic, ["k1:v1", "k2:v2"], "-K", ":", "-H", "trace=abc", "-H", "empty=") + got = rows(f"""SELECT record_offset, convert_from(key, 'UTF8'), convert_from(value, 'UTF8'), + header_keys::text, timestamp_type, leader_epoch IS NOT NULL, + timestamp > now() - interval '5 minutes', is_transactional + FROM kafgres_read('{topic}')""") + assert got == [ + ["0", "k1", "v1", "{trace,empty}", "CreateTime", "t", "t", "f"], + ["1", "k2", "v2", "{trace,empty}", "CreateTime", "t", "t", "f"], + ] + values = sql(f"""SELECT string_agg(coalesce(convert_from(v, 'UTF8'), 'NULL'), ',') + FROM kafgres_read('{topic}', 0, to_offset => 1), + unnest(header_values) v""") + assert values == "abc," + + +def test_offset_bounds_are_half_open_and_clamped(topic): + produce(topic, [str(i) for i in range(20)]) + assert sorted(read(topic, "0", "from_offset => 5", "to_offset => 8")) == [5, 6, 7] + assert sorted(read(topic, "0", "from_offset => -10", "to_offset => 2")) == [0, 1] + assert sorted(read(topic, "0", "from_offset => 18", "to_offset => 1000")) == [18, 19] + assert read(topic, "0", "from_offset => 8", "to_offset => 8") == {} + assert read(topic, "0", "from_offset => 9", "to_offset => 3") == {} + + +def test_time_bounds_follow_offsets_for_times(topic): + """The time index decides where a time range starts and ends, as offsetsForTimes does.""" + marks = [] + for wave in range(4): + marks.append(int(time.time() * 1000)) + produce(topic, [f"w{wave}-{i}" for i in range(10)]) + time.sleep(1.1) + start, end = offset_at(topic, marks[1]), offset_at(topic, marks[3]) + assert (start, end) == (10, 30), (start, end) + got = read(topic, "0", f"from_time => {ts(marks[1])}", f"to_time => {ts(marks[3])}") + assert sorted(got) == list(range(start, end)) + # Time and offset bounds intersect. + got = read(topic, "0", f"from_time => {ts(marks[1])}", "to_offset => 15") + assert sorted(got) == list(range(10, 15)) + assert read(topic, "0", f"from_time => {ts(int(time.time() * 1000) + 60_000)}") == {} + + +def test_it_composes_with_ordinary_sql(topic): + produce(topic, [f'{{"n": {i}, "kind": "{"even" if i % 2 == 0 else "odd"}"}}' + for i in range(30)]) + got = sql(f"""SELECT string_agg((convert_from(value, 'UTF8')::jsonb ->> 'n'), ',' + ORDER BY record_offset) + FROM kafgres_read('{topic}', 0, from_offset => 10, to_offset => 20) + WHERE convert_from(value, 'UTF8')::jsonb ->> 'kind' = 'odd'""") + assert got == "11,13,15,17,19" + got = sql(f"""SELECT count(*) FROM kafgres_partition_offsets('{topic}') p, + LATERAL kafgres_read('{topic}', p.partition, + from_offset => p.high_watermark - 5) r""") + assert got == "5" + assert sql(f"SELECT count(*) FROM (SELECT * FROM kafgres_read('{topic}') LIMIT 3) s") == "3" + assert sql(f"SELECT count(*) FROM kafgres_read('{topic}', max_records => 4)") == "4" + + +def test_every_partition_when_none_is_named(): + name = "sqlread-partitions" + sql(f"SELECT kafgres_drop_topic('{name}')") + sql(f"SELECT kafgres_create_topic('{name}', 3)") + try: + for p in range(3): + produce(name, [f"p{p}-{i}" for i in range(p + 2)], "-p", str(p)) + got = rows(f"SELECT partition, count(*) FROM kafgres_read('{name}') GROUP BY 1 ORDER BY 1") + assert got == [["0", "2"], ["1", "3"], ["2", "4"]] + with pytest.raises(RuntimeError, match="has no partition 3"): + sql(f"SELECT * FROM kafgres_read('{name}', 3)") + finally: + sql(f"SELECT kafgres_drop_topic('{name}')") + + +def test_aborted_kafka_transactions_are_filtered_like_a_consumer_would(topic): + """The aborted list and the abort marker, applied as the Java consumer applies them.""" + for outcome in ("abort", "commit", "abort"): + out = subprocess.run( + ["docker", "run", "--rm", "--network", "host", CLIENTS, + "sarama-conformance", BROKER, f"txn-{outcome}", topic], + capture_output=True, text=True, timeout=180, + ) + assert f"OK {outcome}" in out.stdout, out.stderr[-800:] + for isolation in ("read_committed", "read_uncommitted"): + assert read(topic, isolation=isolation) == consumed(topic, isolation), isolation + assert len(read(topic)) < len(read(topic, isolation="read_uncommitted")) + + +def test_a_full_abort_list_is_not_mistaken_for_one_cut_short(topic): + """Exactly the cap of aborted transactions overlapping a record lists them all; one + more cannot be listed in one read. The entries are planted, under another producer.""" + produce(topic, ["only"]) + for count, readable in ((1000, True), (1001, False)): + sql(f"""DELETE FROM kafgres_txn_aborted + WHERE topic_id = (SELECT topic_id FROM kafgres_topics WHERE name = '{topic}'); + INSERT INTO kafgres_txn_aborted + (topic_id, partition, producer_id, first_offset, last_offset) + SELECT topic_id, 0, 999999, -n, 1000000 + FROM kafgres_topics, generate_series(0, {count - 1}) n + WHERE name = '{topic}'""") + if readable: + assert read(topic) == {0: "only"} + else: + with pytest.raises(RuntimeError, match="more aborted transactions overlap"): + read(topic) + sql(f"""DELETE FROM kafgres_txn_aborted + WHERE topic_id = (SELECT topic_id FROM kafgres_topics WHERE name = '{topic}')""") + + +@pytest.mark.skipif(sql("SHOW kafgres.storage_engine") != "segment", + reason="kafgres_produce() is the segment engine's") +def test_a_rolled_back_sql_produce_is_read_only_uncommitted(topic): + sql(f"BEGIN; SELECT kafgres_produce('{topic}', NULL, 'kept'); COMMIT") + sql(f"BEGIN; SELECT kafgres_produce('{topic}', NULL, 'rolled-back'); ROLLBACK") + sql(f"BEGIN; SELECT kafgres_produce('{topic}', NULL, 'kept-too'); COMMIT") + assert list(read(topic).values()) == ["kept", "kept-too"] + assert list(read(topic, isolation="read_uncommitted").values()) == [ + "kept", "rolled-back", "kept-too"] + got = rows(f"SELECT is_transactional, producer_id IS NOT NULL FROM kafgres_read('{topic}')") + assert got == [["t", "t"], ["t", "t"]] + + +def test_bad_arguments_are_refused(topic): + with pytest.raises(RuntimeError, match="isolation must be"): + sql(f"SELECT * FROM kafgres_read('{topic}', isolation => 'snapshot')") + with pytest.raises(RuntimeError, match="max_records must not be negative"): + sql(f"SELECT * FROM kafgres_read('{topic}', max_records => -1)") + with pytest.raises(RuntimeError, match="no such topic"): + sql("SELECT * FROM kafgres_read('sqlread-no-such-topic')") + + +def set_acls(value, topic): + """Set `kafgres.acls_enabled` and wait until new sessions and the broker both see it. The + broker reloads in its own loop, and anonymous Metadata for `topic` shows which it has.""" + sql(f"ALTER SYSTEM SET kafgres.acls_enabled = {value}") + sql("SELECT pg_reload_conf()") + deadline = time.monotonic() + 30 + while True: + meta = kcat("-L", "-t", topic, timeout=30) + denied = "authorization failed" in (meta.stdout + meta.stderr).lower() + if sql("SHOW kafgres.acls_enabled") == value and denied == (value == "on"): + return + assert time.monotonic() < deadline, f"kafgres.acls_enabled never became {value}" + time.sleep(0.2) + + +@pytest.fixture +def reader_role(topic): + if sql(f"SELECT count(*) FROM pg_roles WHERE rolname = '{ROLE}'") == "1": + sql(f"DROP OWNED BY {ROLE}; DROP ROLE {ROLE}") + sql(f"CREATE ROLE {ROLE}") + yield topic + set_acls("off", topic) + sql(f"DELETE FROM kafgres_acls WHERE principal = 'User:{ROLE}'") + sql(f"DROP OWNED BY {ROLE}; DROP ROLE {ROLE}") + + +def as_role(statement): + try: + return True, sql(f"SET ROLE {ROLE}; {statement}") + except RuntimeError as e: + return False, str(e) + + +def test_reading_takes_a_grant_and_then_the_acls(reader_role): + topic = reader_role + produce(topic, ["secret"]) + query = f"SELECT convert_from(value, 'UTF8') FROM kafgres_read('{topic}')" + + ok, out = as_role(query) + assert not ok and "permission denied for function kafgres_read" in out, out + + # EXECUTE alone is enough: the tables behind the log are not granted. + sql(f"GRANT EXECUTE ON FUNCTION kafgres_read TO {ROLE}") + ok, out = as_role(query) + assert ok and out.endswith("secret"), out + + set_acls("on", topic) + ok, out = as_role(query) + assert not ok and "not allowed to READ" in out, out + + sql(f"SELECT kafgres_add_acl('User:{ROLE}', 'READ', 'TOPIC', '{topic}')") + time.sleep(1.5) + ok, out = as_role(query) + assert ok and out.endswith("secret"), out + + # The ACL check reads without taking an xid either. + ok, out = as_role(f"{query}; SELECT pg_current_xact_id_if_assigned() IS NULL") + assert ok and out.endswith("t"), out + + # The read runs elevated, but nothing after it does. + ok, out = as_role(f"SELECT count(*) FROM kafgres_read('{topic}'), " + f"LATERAL (SELECT current_user::text AS who) u WHERE u.who = '{ROLE}'") + assert ok and out.endswith("1"), out + + +@pytest.mark.skipif(int(sql("SHOW server_version_num")) < 140000, + reason="pg_backend_memory_contexts is Postgres 14+") +def test_a_scan_does_not_hold_what_it_read_until_the_transaction_ends(topic): + """A scan frees each batch as it goes, not at transaction end.""" + produce(topic, ["x" * 4000] * 5000) # 20 MB, toasted on the table engine + got = script(f"""BEGIN; + SELECT count(*) FROM kafgres_read('{topic}'); + SELECT sum(total_bytes) FROM pg_backend_memory_contexts + WHERE name = 'TopTransactionContext'; + COMMIT;""") + # BEGIN, the count, the bytes held, COMMIT. + held = int(got[2]) + assert held < 4 * 1024 * 1024, f"{held} bytes still held after scanning 20 MB" + + +def test_a_scan_leaves_its_transaction_no_locks_on_the_log_and_no_xid(topic): + """Held locks block the table engine's segment roll; a held xid slows every producer.""" + produce(topic, [str(i) for i in range(20)]) + got = script(f"""BEGIN; + SELECT count(*) FROM kafgres_read('{topic}', from_time => now() - interval '1 hour'); + SELECT count(*) FROM pg_locks l JOIN pg_class c ON c.oid = l.relation + WHERE l.pid = pg_backend_pid() AND c.relname LIKE 'kafgres_log%'; + SELECT pg_current_xact_id_if_assigned() IS NULL; + COMMIT;""") + assert got[1:4] == ["20", "0", "t"], got + + +def test_a_transaction_aborted_during_the_statement_is_still_filtered(topic): + """Transaction state is read after the log it judges: on the segment engine the log is live.""" + import threading + produce(topic, [str(i) for i in range(5)]) + got = {} + + def statement(): + # The lateral reference runs kafgres_read() after the sleep, in the same statement. + got["rows"] = rows(f"""SELECT count(*), count(*) FILTER (WHERE r.is_transactional) + FROM (SELECT pg_sleep(8), 100000 AS n) s, + LATERAL kafgres_read('{topic}', 0, max_records => s.n) r""") + + reader = threading.Thread(target=statement) + reader.start() + time.sleep(1) + out = subprocess.run( + ["docker", "run", "--rm", "--network", "host", CLIENTS, + "sarama-conformance", BROKER, "txn-abort", topic], + capture_output=True, text=True, timeout=60, + ) + assert "OK abort" in out.stdout, out.stderr[-800:] + reader.join(30) + assert got["rows"] == [["5", "0"]], got + assert len(read(topic, isolation="read_uncommitted")) > 5 + + +def test_read_committed_is_refused_under_repeatable_read(topic): + produce(topic, ["a", "b"]) + for level in ("REPEATABLE READ", "SERIALIZABLE"): + with pytest.raises(RuntimeError, match="needs a READ COMMITTED transaction"): + sql(f"BEGIN ISOLATION LEVEL {level}; SELECT count(*) FROM kafgres_read('{topic}'); COMMIT") + got = script(f"""BEGIN ISOLATION LEVEL {level}; + SELECT count(*) FROM kafgres_read('{topic}', isolation => 'read_uncommitted'); + COMMIT;""") + assert got[1] == "2", got