ci: add duckdb s3 tests

.github/workflows/ci.yml

@@ -645,6 +645,48 @@ jobs:
           command: "cargo nextest run --cargo-profile ci -p vortex-sqllogictest --no-run"
           message-format-flag: "--cargo-message-format"
 
+  duckdb-s3-test:
+    name: "DuckDB S3 tests"
+    needs: duckdb-ready
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=duckdb-s3-test', github.run_id)
+          || 'ubuntu-latest' }}
+    timeout-minutes: 30
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
+      - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: "true"
+      - name: Build DuckDB tests
+        shell: bash
+        run: |
+          cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --no-run
+      - name: Start MinIO
+        shell: bash
+        run: |
+          curl -sSL https://dl.min.io/server/minio/release/linux-amd64/minio -o "${RUNNER_TEMP}/minio"
+          chmod +x "${RUNNER_TEMP}/minio"
+          mkdir -p "${RUNNER_TEMP}/minio-data/vortex-test"
+          MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin \
+            "${RUNNER_TEMP}/minio" server "${RUNNER_TEMP}/minio-data" --address 127.0.0.1:9000 &
+          until curl -sf http://127.0.0.1:9000/minio/health/live; do sleep 1; done
+      - name: Run tests
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: minioadmin
+          AWS_SECRET_ACCESS_KEY: minioadmin
+          AWS_ENDPOINT: "http://127.0.0.1:9000"
+          AWS_REGION: "us-east-1"
+          AWS_ALLOW_HTTP: "true"
+          VORTEX_DUCKDB_S3_URI: "s3://vortex-test/duckdb"
+        run: |
+          cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --run-ignored all s3_
+
   wasm-integration:
     name: "WASM integration smoke test"
     timeout-minutes: 30

vortex-duckdb/src/e2e_test/mod.rs

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+#[cfg(test)]
+mod s3_test;
 #[cfg(test)]
 mod spatial_pushdown_test;
 #[cfg(test)]

vortex-duckdb/src/e2e_test/s3_test.rs

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use crate::duckdb::Connection;
+use crate::duckdb::Database;
+
+fn s3_uri() -> String {
+    std::env::var("VORTEX_DUCKDB_S3_URI")
+        .expect("VORTEX_DUCKDB_S3_URI must be set")
+        .trim_end_matches('/')
+        .to_string()
+}
+
+fn database_connection() -> Connection {
+    let db = Database::open_in_memory().unwrap();
+    db.register_vortex_scan_replacement().unwrap();
+    crate::initialize(&db).unwrap();
+    db.connect().unwrap()
+}
+
+fn row_count(conn: &Connection, query: &str) -> u64 {
+    conn.query(query)
+        .unwrap_or_else(|err| panic!("query failed: {query}\n{err}"))
+        .into_iter()
+        .map(|chunk| chunk.len())
+        .sum()
+}
+
+#[test]
+#[ignore = "requires an S3 endpoint"]
+fn s3_roundtrip() {
+    let uri = format!("{}/write_then_read.vortex", s3_uri());
+
+    let writer = database_connection();
+    writer
+        .query(&format!(
+            "COPY (SELECT i AS id, i * 2 AS doubled FROM range(1000) t(i)) \
+             TO '{uri}' (FORMAT VORTEX)"
+        ))
+        .expect("COPY TO s3 should succeed");
+
+    let reader = database_connection();
+    let total = row_count(&reader, &format!("SELECT id, doubled FROM '{uri}'"));
+    assert_eq!(total, 1000);
+}
+
+#[test]
+#[ignore = "requires an S3 endpoint"]
+fn s3_read_with_filter() {
+    let uri = format!("{}/read_with_filter.vortex", s3_uri());
+
+    let conn = database_connection();
+    conn.query(&format!(
+        "COPY (SELECT i AS id, i * 2 AS doubled FROM range(1000) t(i)) \
+         TO '{uri}' (FORMAT VORTEX)"
+    ))
+    .expect("COPY TO s3 should succeed");
+
+    let matching = row_count(
+        &conn,
+        &format!("SELECT id FROM '{uri}' WHERE id >= 250 AND doubled = id * 2"),
+    );
+    assert_eq!(matching, 750);
+}