From c120870e4b2d3e25b5ea6d6ad6754c0612afc10e Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 26 Oct 2026 07:54:56 -0700 Subject: [PATCH 1/2] chore: rename rust crates to lance-csr for crates.io Upstream owns the lance-graph and lance-graph-catalog names on crates.io, so publishing needs our own. Renames the crates, directories and Rust paths to lance-csr*, points the manifests at this repository, and ships LICENSE and NOTICE with the published crates. --- .bumpversion.toml | 8 +- .github/workflows/python-test.yml | 2 +- .github/workflows/rust-test.yml | 2 +- AGENTS.md | 12 +- Cargo.lock | 114 +++++++++--------- Cargo.toml | 8 +- README.md | 2 +- .../Cargo.toml | 6 +- .../README.md | 6 +- .../benches/graph_execution.rs | 2 +- .../Cargo.toml | 6 +- crates/lance-csr-catalog/LICENSE | 1 + crates/lance-csr-catalog/NOTICE | 1 + .../README.md | 0 .../src/lib.rs | 0 .../src/namespace/directory.rs | 0 .../src/namespace/mod.rs | 0 .../src/source_catalog.rs | 0 .../Cargo.lock | 114 +++++++++--------- .../Cargo.toml | 6 +- .../src/executor.rs | 0 .../src/graph.rs | 16 +-- .../src/lib.rs | 0 .../src/namespace.rs | 2 +- .../CONTRIBUTING.md | 4 +- crates/{lance-graph => lance-csr}/Cargo.toml | 8 +- crates/lance-csr/LICENSE | 1 + crates/lance-csr/NOTICE | 1 + crates/{lance-graph => lance-csr}/README.md | 14 +-- crates/{lance-graph => lance-csr}/src/ast.rs | 0 .../src/case_insensitive.rs | 6 +- .../{lance-graph => lance-csr}/src/config.rs | 0 .../src/csr_expand.rs | 0 .../src/csr_index.rs | 0 .../src/datafusion_planner/analysis.rs | 0 .../builder/aggregate_ops.rs | 0 .../datafusion_planner/builder/basic_ops.rs | 0 .../builder/comprehension_ops.rs | 0 .../datafusion_planner/builder/expand_ops.rs | 0 .../src/datafusion_planner/builder/helpers.rs | 0 .../builder/join_builder.rs | 0 .../src/datafusion_planner/builder/mod.rs | 0 .../src/datafusion_planner/config_helpers.rs | 2 +- .../src/datafusion_planner/cypher_udf.rs | 0 .../src/datafusion_planner/expression.rs | 0 .../src/datafusion_planner/join_ops.rs | 2 +- .../src/datafusion_planner/mod.rs | 2 +- .../src/datafusion_planner/scan_ops.rs | 2 +- .../src/datafusion_planner/test_fixtures.rs | 4 +- .../src/datafusion_planner/udf.rs | 0 .../src/datafusion_planner/vector_ops.rs | 0 .../{lance-graph => lance-csr}/src/error.rs | 0 .../src/key_statistics.rs | 0 .../src/lance_vector_search.rs | 6 +- crates/{lance-graph => lance-csr}/src/lib.rs | 6 +- .../src/logical_plan.rs | 0 .../src/parameter_substitution.rs | 0 .../{lance-graph => lance-csr}/src/parser.rs | 0 .../src/pattern_alternatives.rs | 0 .../src/pattern_order.rs | 0 .../{lance-graph => lance-csr}/src/query.rs | 40 +++--- .../src/semantic.rs | 0 .../src/shortest_path.rs | 0 .../src/spark_dialect.rs | 0 .../src/sql_query.rs | 4 +- .../tests/test_case_insensitivity.rs | 2 +- .../tests/test_complex_return_clauses.rs | 4 +- .../tests/test_datafusion_pipeline.rs | 4 +- .../tests/test_datafusion_scenarios.rs | 4 +- .../test_datafusion_varlength_complex.rs | 4 +- .../tests/test_datafusion_vector_search.rs | 4 +- .../tests/test_datafusion_with_context.rs | 4 +- .../tests/test_explain_output.rs | 4 +- .../tests/test_lance_vector_search.rs | 6 +- .../tests/test_sql_query.rs | 2 +- .../tests/test_to_spark_sql.rs | 2 +- .../tests/test_to_sql.rs | 2 +- docs/AGENTS.md | 2 +- docs/execution.md | 2 +- docs/project_structure.md | 16 +-- python/DEVELOPMENT.md | 6 +- python/Makefile | 2 +- python/README.md | 2 +- python/pyproject.toml | 2 +- python/python/lance_csr/__init__.py | 2 +- 85 files changed, 244 insertions(+), 242 deletions(-) rename crates/{lance-graph-benches => lance-csr-benches}/Cargo.toml (71%) rename crates/{lance-graph-benches => lance-csr-benches}/README.md (78%) rename crates/{lance-graph-benches => lance-csr-benches}/benches/graph_execution.rs (99%) rename crates/{lance-graph-catalog => lance-csr-catalog}/Cargo.toml (80%) create mode 120000 crates/lance-csr-catalog/LICENSE create mode 120000 crates/lance-csr-catalog/NOTICE rename crates/{lance-graph-catalog => lance-csr-catalog}/README.md (100%) rename crates/{lance-graph-catalog => lance-csr-catalog}/src/lib.rs (100%) rename crates/{lance-graph-catalog => lance-csr-catalog}/src/namespace/directory.rs (100%) rename crates/{lance-graph-catalog => lance-csr-catalog}/src/namespace/mod.rs (100%) rename crates/{lance-graph-catalog => lance-csr-catalog}/src/source_catalog.rs (100%) rename crates/{lance-graph-python => lance-csr-python}/Cargo.lock (99%) rename crates/{lance-graph-python => lance-csr-python}/Cargo.toml (82%) rename crates/{lance-graph-python => lance-csr-python}/src/executor.rs (100%) rename crates/{lance-graph-python => lance-csr-python}/src/graph.rs (99%) rename crates/{lance-graph-python => lance-csr-python}/src/lib.rs (100%) rename crates/{lance-graph-python => lance-csr-python}/src/namespace.rs (93%) rename crates/{lance-graph => lance-csr}/CONTRIBUTING.md (62%) rename crates/{lance-graph => lance-csr}/Cargo.toml (85%) create mode 120000 crates/lance-csr/LICENSE create mode 120000 crates/lance-csr/NOTICE rename crates/{lance-graph => lance-csr}/README.md (92%) rename crates/{lance-graph => lance-csr}/src/ast.rs (100%) rename crates/{lance-graph => lance-csr}/src/case_insensitive.rs (98%) rename crates/{lance-graph => lance-csr}/src/config.rs (100%) rename crates/{lance-graph => lance-csr}/src/csr_expand.rs (100%) rename crates/{lance-graph => lance-csr}/src/csr_index.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/analysis.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/builder/aggregate_ops.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/builder/basic_ops.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/builder/comprehension_ops.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/builder/expand_ops.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/builder/helpers.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/builder/join_builder.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/builder/mod.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/config_helpers.rs (99%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/cypher_udf.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/expression.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/join_ops.rs (99%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/mod.rs (99%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/scan_ops.rs (99%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/test_fixtures.rs (92%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/udf.rs (100%) rename crates/{lance-graph => lance-csr}/src/datafusion_planner/vector_ops.rs (100%) rename crates/{lance-graph => lance-csr}/src/error.rs (100%) rename crates/{lance-graph => lance-csr}/src/key_statistics.rs (100%) rename crates/{lance-graph => lance-csr}/src/lance_vector_search.rs (99%) rename crates/{lance-graph => lance-csr}/src/lib.rs (92%) rename crates/{lance-graph => lance-csr}/src/logical_plan.rs (100%) rename crates/{lance-graph => lance-csr}/src/parameter_substitution.rs (100%) rename crates/{lance-graph => lance-csr}/src/parser.rs (100%) rename crates/{lance-graph => lance-csr}/src/pattern_alternatives.rs (100%) rename crates/{lance-graph => lance-csr}/src/pattern_order.rs (100%) rename crates/{lance-graph => lance-csr}/src/query.rs (98%) rename crates/{lance-graph => lance-csr}/src/semantic.rs (100%) rename crates/{lance-graph => lance-csr}/src/shortest_path.rs (100%) rename crates/{lance-graph => lance-csr}/src/spark_dialect.rs (100%) rename crates/{lance-graph => lance-csr}/src/sql_query.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_case_insensitivity.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_complex_return_clauses.rs (97%) rename crates/{lance-graph => lance-csr}/tests/test_datafusion_pipeline.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_datafusion_scenarios.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_datafusion_varlength_complex.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_datafusion_vector_search.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_datafusion_with_context.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_explain_output.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_lance_vector_search.rs (98%) rename crates/{lance-graph => lance-csr}/tests/test_sql_query.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_to_spark_sql.rs (99%) rename crates/{lance-graph => lance-csr}/tests/test_to_sql.rs (99%) diff --git a/.bumpversion.toml b/.bumpversion.toml index 32fbd64..dedef9a 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -29,22 +29,22 @@ first_value = "0" # Rust Cargo.toml files [[tool.bumpversion.files]] -filename = "crates/lance-graph/Cargo.toml" +filename = "crates/lance-csr/Cargo.toml" search = 'version = "{current_version}"' replace = 'version = "{new_version}"' [[tool.bumpversion.files]] -filename = "crates/lance-graph-catalog/Cargo.toml" +filename = "crates/lance-csr-catalog/Cargo.toml" search = 'version = "{current_version}"' replace = 'version = "{new_version}"' [[tool.bumpversion.files]] -filename = "crates/lance-graph-benches/Cargo.toml" +filename = "crates/lance-csr-benches/Cargo.toml" search = 'version = "{current_version}"' replace = 'version = "{new_version}"' [[tool.bumpversion.files]] -filename = "crates/lance-graph-python/Cargo.toml" +filename = "crates/lance-csr-python/Cargo.toml" search = 'version = "{current_version}"' replace = 'version = "{new_version}"' diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index e8c9ef9..05f5d1e 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -35,7 +35,7 @@ jobs: with: shared-key: "lance-csr-python" # maturin builds the bindings crate, which sits outside the workspace. - workspaces: crates/lance-graph-python + workspaces: crates/lance-csr-python - name: Install dependencies run: | sudo apt update diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 44edee0..35f4201 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -39,4 +39,4 @@ jobs: - name: Run tests run: cargo test --workspace - name: Check benchmarks - run: cargo check -p lance-graph-benches --benches + run: cargo check -p lance-csr-benches --benches diff --git a/AGENTS.md b/AGENTS.md index d445755..5d64328 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,18 +1,18 @@ # Repository Guidelines ## Project Structure & Module Organization -- `crates/lance-graph/` hosts the Rust Cypher engine; keep new modules under `src/` and co-locate helpers inside `query/` or feature-specific submodules. -- `crates/lance-graph-catalog/` provides the graph source catalog and `DirNamespace` (Unity Catalog and Delta Lake support were removed). -- `crates/lance-graph-benches/` contains performance benchmarks in a dedicated unpublished crate; add new benchmarks here rather than in the main crate. -- `crates/lance-graph-python/src/` contains the PyO3 bridge; `python/python/lance_csr/` holds the pure-Python facade and packaging metadata. +- `crates/lance-csr/` hosts the Rust Cypher engine; keep new modules under `src/` and co-locate helpers inside `query/` or feature-specific submodules. +- `crates/lance-csr-catalog/` provides the graph source catalog and `DirNamespace` (Unity Catalog and Delta Lake support were removed). +- `crates/lance-csr-benches/` contains performance benchmarks in a dedicated unpublished crate; add new benchmarks here rather than in the main crate. +- `crates/lance-csr-python/src/` contains the PyO3 bridge; `python/python/lance_csr/` holds the pure-Python facade and packaging metadata. - `python/python/tests/` stores functional tests; mirror new features with targeted cases here and in the corresponding Rust module. - `examples/` demonstrates Cypher usage; update or add examples when introducing new public APIs. - `docs/` is the Zensical documentation site (configured by `zensical.toml`); follow `docs/AGENTS.md`. -- Keep the docs in step with the code. Any substantial code change (a new or changed Python method, parameter, return type, exception or Cypher feature, or a change in behaviour users can observe) updates the affected guide pages in the same pull request, and `docs/api-reference.md` whenever the Python API changes. That page is written by hand from `crates/lance-graph-python/src/`, so check every signature and statement against the code. Then run `uv run --project docs --locked zensical build --strict` and `uvx typos@1.26.0 .` (the spell check CI runs). +- Keep the docs in step with the code. Any substantial code change (a new or changed Python method, parameter, return type, exception or Cypher feature, or a change in behaviour users can observe) updates the affected guide pages in the same pull request, and `docs/api-reference.md` whenever the Python API changes. That page is written by hand from `crates/lance-csr-python/src/`, so check every signature and statement against the code. Then run `uv run --project docs --locked zensical build --strict` and `uvx typos@1.26.0 .` (the spell check CI runs). ## Build, Test, and Development Commands - `cargo test --workspace` (from the repository root) runs the Rust tests; building needs `protoc`. -- `cargo bench --bench graph_execution -p lance-graph-benches` measures performance-critical changes; include shortened runs with `--warm-up-time 1`. +- `cargo bench --bench graph_execution -p lance-csr-benches` measures performance-critical changes; include shortened runs with `--warm-up-time 1`. - Never use `pip`. In `python/`, `make install` runs `uv sync --locked --no-install-project` (locked dependencies and the `dev` group) and then a debug build with `uv run --no-sync maturin develop`; re-run it after Rust edits. Change dependencies with `uv add`/`uv remove` and commit `uv.lock`. - `make test` runs the Python tests; `make lint` runs `ruff`, formatting checks, and `pyright`. diff --git a/Cargo.lock b/Cargo.lock index 0f372ba..1b45076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3945,6 +3945,63 @@ dependencies = [ "url", ] +[[package]] +name = "lance-csr" +version = "0.2.0" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-schema", + "async-trait", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-sql", + "futures", + "lance", + "lance-arrow", + "lance-csr-catalog", + "lance-linalg", + "lance-namespace", + "lance-table", + "nom 7.1.3", + "serde", + "serde_json", + "snafu 0.8.9", + "tempfile", + "tokio", +] + +[[package]] +name = "lance-csr-benches" +version = "0.2.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "criterion", + "futures", + "lance", + "lance-csr", + "tempfile", + "tokio", +] + +[[package]] +name = "lance-csr-catalog" +version = "0.2.0" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion", + "lance-namespace", + "serde", + "serde_json", + "snafu 0.8.9", + "tokio", +] + [[package]] name = "lance-datafusion" version = "13.0.0" @@ -4074,63 +4131,6 @@ dependencies = [ "serde", ] -[[package]] -name = "lance-graph" -version = "0.2.0" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-schema", - "async-trait", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-sql", - "futures", - "lance", - "lance-arrow", - "lance-graph-catalog", - "lance-linalg", - "lance-namespace", - "lance-table", - "nom 7.1.3", - "serde", - "serde_json", - "snafu 0.8.9", - "tempfile", - "tokio", -] - -[[package]] -name = "lance-graph-benches" -version = "0.2.0" -dependencies = [ - "arrow-array", - "arrow-schema", - "criterion", - "futures", - "lance", - "lance-graph", - "tempfile", - "tokio", -] - -[[package]] -name = "lance-graph-catalog" -version = "0.2.0" -dependencies = [ - "arrow-schema", - "async-trait", - "datafusion", - "lance-namespace", - "serde", - "serde_json", - "snafu 0.8.9", - "tokio", -] - [[package]] name = "lance-index" version = "13.0.0" diff --git a/Cargo.toml b/Cargo.toml index 666404e..72a5d29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [workspace] members = [ - "crates/lance-graph", - "crates/lance-graph-catalog", - "crates/lance-graph-benches", + "crates/lance-csr", + "crates/lance-csr-catalog", + "crates/lance-csr-benches", ] # Python package needs to be built by maturin. -exclude = ["crates/lance-graph-python"] +exclude = ["crates/lance-csr-python"] resolver = "2" diff --git a/README.md b/README.md index 70e5ddf..92f6ae7 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ DataFusion as extension operators, falling back to a join for any hop the CSR ca ## Repository layout -- `crates/lance-graph` – the Cypher-capable query engine implemented in Rust +- `crates/lance-csr` – the Cypher-capable query engine implemented in Rust - `python/` – PyO3 bindings and Python packages: - `lance_csr` – thin wrapper around the Rust query engine diff --git a/crates/lance-graph-benches/Cargo.toml b/crates/lance-csr-benches/Cargo.toml similarity index 71% rename from crates/lance-graph-benches/Cargo.toml rename to crates/lance-csr-benches/Cargo.toml index 6d0b613..3c03181 100644 --- a/crates/lance-graph-benches/Cargo.toml +++ b/crates/lance-csr-benches/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "lance-graph-benches" +name = "lance-csr-benches" version = "0.2.0" edition = "2021" publish = false -description = "Benchmarks for lance-graph (not published)" +description = "Benchmarks for lance-csr (not published)" [dependencies] -lance-graph = { path = "../lance-graph", version = "0.2.0" } +lance-csr = { path = "../lance-csr", version = "0.2.0" } lance = "13.0.0" arrow-array = "58" arrow-schema = "58" diff --git a/crates/lance-graph-benches/README.md b/crates/lance-csr-benches/README.md similarity index 78% rename from crates/lance-graph-benches/README.md rename to crates/lance-csr-benches/README.md index ce63a57..2e83cf8 100644 --- a/crates/lance-graph-benches/README.md +++ b/crates/lance-csr-benches/README.md @@ -1,6 +1,6 @@ # Lance Graph Benchmarks -Performance benchmarks for the `lance-graph` crate. +Performance benchmarks for the `lance-csr` crate. ## Running Benchmarks @@ -8,10 +8,10 @@ From the repository root: ```bash # Run all benchmarks -cargo bench -p lance-graph-benches +cargo bench -p lance-csr-benches # Run specific benchmark -cargo bench -p lance-graph-benches --bench graph_execution +cargo bench -p lance-csr-benches --bench graph_execution ``` ## Benchmarks diff --git a/crates/lance-graph-benches/benches/graph_execution.rs b/crates/lance-csr-benches/benches/graph_execution.rs similarity index 99% rename from crates/lance-graph-benches/benches/graph_execution.rs rename to crates/lance-csr-benches/benches/graph_execution.rs index 2b6831b..b1197c1 100644 --- a/crates/lance-graph-benches/benches/graph_execution.rs +++ b/crates/lance-csr-benches/benches/graph_execution.rs @@ -22,7 +22,7 @@ use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use futures::TryStreamExt; use lance::dataset::{Dataset, WriteMode, WriteParams}; -use lance_graph::{CypherQuery, GraphConfig}; +use lance_csr::{CypherQuery, GraphConfig}; use tempfile::TempDir; fn create_people_batch() -> RecordBatch { diff --git a/crates/lance-graph-catalog/Cargo.toml b/crates/lance-csr-catalog/Cargo.toml similarity index 80% rename from crates/lance-graph-catalog/Cargo.toml rename to crates/lance-csr-catalog/Cargo.toml index e984663..ba98c1d 100644 --- a/crates/lance-graph-catalog/Cargo.toml +++ b/crates/lance-csr-catalog/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "lance-graph-catalog" +name = "lance-csr-catalog" version = "0.2.0" edition = "2021" license = "Apache-2.0" -authors = ["Lance Devs "] -repository = "https://github.com/lancedb/lance-graph" +authors = ["The lance-csr Authors"] +repository = "https://github.com/hyperdimensionalcomputingai/lance-csr" readme = "README.md" description = "Catalog and namespace utilities for Lance graph" keywords = ["lance", "graph", "catalog", "namespace"] diff --git a/crates/lance-csr-catalog/LICENSE b/crates/lance-csr-catalog/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/crates/lance-csr-catalog/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/lance-csr-catalog/NOTICE b/crates/lance-csr-catalog/NOTICE new file mode 120000 index 0000000..fb376cf --- /dev/null +++ b/crates/lance-csr-catalog/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/crates/lance-graph-catalog/README.md b/crates/lance-csr-catalog/README.md similarity index 100% rename from crates/lance-graph-catalog/README.md rename to crates/lance-csr-catalog/README.md diff --git a/crates/lance-graph-catalog/src/lib.rs b/crates/lance-csr-catalog/src/lib.rs similarity index 100% rename from crates/lance-graph-catalog/src/lib.rs rename to crates/lance-csr-catalog/src/lib.rs diff --git a/crates/lance-graph-catalog/src/namespace/directory.rs b/crates/lance-csr-catalog/src/namespace/directory.rs similarity index 100% rename from crates/lance-graph-catalog/src/namespace/directory.rs rename to crates/lance-csr-catalog/src/namespace/directory.rs diff --git a/crates/lance-graph-catalog/src/namespace/mod.rs b/crates/lance-csr-catalog/src/namespace/mod.rs similarity index 100% rename from crates/lance-graph-catalog/src/namespace/mod.rs rename to crates/lance-csr-catalog/src/namespace/mod.rs diff --git a/crates/lance-graph-catalog/src/source_catalog.rs b/crates/lance-csr-catalog/src/source_catalog.rs similarity index 100% rename from crates/lance-graph-catalog/src/source_catalog.rs rename to crates/lance-csr-catalog/src/source_catalog.rs diff --git a/crates/lance-graph-python/Cargo.lock b/crates/lance-csr-python/Cargo.lock similarity index 99% rename from crates/lance-graph-python/Cargo.lock rename to crates/lance-csr-python/Cargo.lock index 248dee7..627603c 100644 --- a/crates/lance-graph-python/Cargo.lock +++ b/crates/lance-csr-python/Cargo.lock @@ -3830,6 +3830,63 @@ dependencies = [ "url", ] +[[package]] +name = "lance-csr" +version = "0.2.0" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-schema", + "async-trait", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-sql", + "futures", + "lance", + "lance-csr-catalog", + "lance-linalg", + "lance-namespace", + "lance-table", + "nom 7.1.3", + "serde", + "serde_json", + "snafu 0.8.9", +] + +[[package]] +name = "lance-csr-catalog" +version = "0.2.0" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion", + "lance-namespace", + "serde", + "serde_json", + "snafu 0.8.9", +] + +[[package]] +name = "lance-csr-python" +version = "0.2.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", + "datafusion", + "futures", + "lance", + "lance-csr", + "pyo3", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "lance-datafusion" version = "13.0.0" @@ -3959,63 +4016,6 @@ dependencies = [ "serde", ] -[[package]] -name = "lance-graph" -version = "0.2.0" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-schema", - "async-trait", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-sql", - "futures", - "lance", - "lance-graph-catalog", - "lance-linalg", - "lance-namespace", - "lance-table", - "nom 7.1.3", - "serde", - "serde_json", - "snafu 0.8.9", -] - -[[package]] -name = "lance-graph-catalog" -version = "0.2.0" -dependencies = [ - "arrow-schema", - "async-trait", - "datafusion", - "lance-namespace", - "serde", - "serde_json", - "snafu 0.8.9", -] - -[[package]] -name = "lance-graph-python" -version = "0.2.0" -dependencies = [ - "arrow", - "arrow-array", - "arrow-ipc", - "arrow-schema", - "datafusion", - "futures", - "lance", - "lance-graph", - "pyo3", - "serde", - "serde_json", - "tokio", -] - [[package]] name = "lance-index" version = "13.0.0" diff --git a/crates/lance-graph-python/Cargo.toml b/crates/lance-csr-python/Cargo.toml similarity index 82% rename from crates/lance-graph-python/Cargo.toml rename to crates/lance-csr-python/Cargo.toml index 3244cb3..f8bcee3 100644 --- a/crates/lance-graph-python/Cargo.toml +++ b/crates/lance-csr-python/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "lance-graph-python" +name = "lance-csr-python" version = "0.2.0" edition = "2021" -authors = ["Lance Devs "] +authors = ["The lance-csr Authors"] publish = false [lib] @@ -17,7 +17,7 @@ arrow-ipc = "58" datafusion = { version = "54", default-features = false } futures = "0.3" lance = "13.0.0" -lance-graph = { path = "../lance-graph" } +lance-csr = { path = "../lance-csr" } serde = { version = "1", features = ["derive"] } serde_json = "1" pyo3 = { version = "0.28", features = ["extension-module", "abi3-py312", "py-clone"] } diff --git a/crates/lance-graph-python/src/executor.rs b/crates/lance-csr-python/src/executor.rs similarity index 100% rename from crates/lance-graph-python/src/executor.rs rename to crates/lance-csr-python/src/executor.rs diff --git a/crates/lance-graph-python/src/graph.rs b/crates/lance-csr-python/src/graph.rs similarity index 99% rename from crates/lance-graph-python/src/graph.rs rename to crates/lance-csr-python/src/graph.rs index 778d688..c50acbc 100644 --- a/crates/lance-graph-python/src/graph.rs +++ b/crates/lance-csr-python/src/graph.rs @@ -28,12 +28,12 @@ use futures::StreamExt; use lance::datafusion::LanceTableProvider; use lance::dataset::builder::DatasetBuilder; use lance::dataset::Dataset; -use lance_graph::csr_expand::{ +use lance_csr::csr_expand::{ edge_changes, is_supported_id_type, Adjacency, AdjacencyCatalog, CsrQueryPlanner, NodeRows, PruneCsrTargetColumns, PushCsrTargetFilters, }; -use lance_graph::key_statistics::KeyStatisticsProvider; -use lance_graph::{ +use lance_csr::key_statistics::KeyStatisticsProvider; +use lance_csr::{ ast::{DistanceMetric as RustDistanceMetric, GraphPattern, ReadingClause}, CypherQuery as RustCypherQuery, ExecutionStrategy as RustExecutionStrategy, GraphConfig as RustGraphConfig, GraphError as RustGraphError, InMemoryCatalog, @@ -374,7 +374,7 @@ impl GraphConfig { )] #[derive(Clone)] pub struct GraphConfigBuilder { - inner: lance_graph::config::GraphConfigBuilder, + inner: lance_csr::config::GraphConfigBuilder, } /// Raise a ``DeprecationWarning`` pointing at the caller's line. @@ -1188,7 +1188,7 @@ enum EngineSource { /// keep the `Arc` they started with, so they finish on their original snapshot. struct EngineState { sources: HashMap, - catalog: Arc, + catalog: Arc, context: SessionContext, /// CSR adjacencies by relationship type; empty unless the engine uses CSR. csr: HashMap, @@ -1235,7 +1235,7 @@ struct CsrEntry { /// of the base. Lookups pay for the delta on every query, and a rebuild folds it in. /// /// This threshold also bounds refresh cost: each delta refresh copies the whole -/// accumulated delta (see the design note on `lance_graph::csr_expand::Adjacency`). +/// accumulated delta (see the design note on `lance_csr::csr_expand::Adjacency`). /// Raising it makes refreshes slower as well as lookups. // shortcut: fixed threshold, chosen from python/benchmarks/csr_delta_accumulated.py; tune per workload if deltas grow large. const CSR_REBUILD_FRACTION: f64 = 0.25; @@ -1322,7 +1322,7 @@ impl EngineSource { impl EngineState { /// Mean fan-out of every relationship type with key statistics, both directions. - fn fan_out(&self) -> lance_graph::logical_plan::FanOut { + fn fan_out(&self) -> lance_csr::logical_plan::FanOut { self.key_stats .values() .filter_map(|entry| entry.fan_out.as_ref()) @@ -1613,7 +1613,7 @@ fn memtable_partitions(batch: &RecordBatch) -> Vec> { /// Distinct counts for every dataset's node-id and edge-endpoint columns. /// -/// DataFusion 54 sizes hash joins from these (see `lance_graph::key_statistics`). +/// DataFusion 54 sizes hash joins from these (see `lance_csr::key_statistics`). /// Node ids count as unique (the row count). Edge endpoints use the CSR's distinct /// source and target counts when the CSR is built, and otherwise one scan of the two /// endpoint columns; only integer endpoints are counted. Reused while a dataset is diff --git a/crates/lance-graph-python/src/lib.rs b/crates/lance-csr-python/src/lib.rs similarity index 100% rename from crates/lance-graph-python/src/lib.rs rename to crates/lance-csr-python/src/lib.rs diff --git a/crates/lance-graph-python/src/namespace.rs b/crates/lance-csr-python/src/namespace.rs similarity index 93% rename from crates/lance-graph-python/src/namespace.rs rename to crates/lance-csr-python/src/namespace.rs index 8fce54d..80e97fa 100644 --- a/crates/lance-graph-python/src/namespace.rs +++ b/crates/lance-csr-python/src/namespace.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use lance_graph::DirNamespace; +use lance_csr::DirNamespace; use pyo3::prelude::*; #[pyclass(name = "DirNamespace", module = "lance_csr.graph")] diff --git a/crates/lance-graph/CONTRIBUTING.md b/crates/lance-csr/CONTRIBUTING.md similarity index 62% rename from crates/lance-graph/CONTRIBUTING.md rename to crates/lance-csr/CONTRIBUTING.md index 395cd64..70fcca5 100644 --- a/crates/lance-graph/CONTRIBUTING.md +++ b/crates/lance-csr/CONTRIBUTING.md @@ -4,11 +4,11 @@ Contributing (Rust crate) Before pushing or opening a PR, please verify Rust code formatting matches CI: ```bash -cargo fmt --manifest-path crates/lance-graph/Cargo.toml -- --check +cargo fmt --manifest-path crates/lance-csr/Cargo.toml -- --check ``` If the check fails, format the code locally: ```bash -cargo fmt --manifest-path crates/lance-graph/Cargo.toml +cargo fmt --manifest-path crates/lance-csr/Cargo.toml ``` diff --git a/crates/lance-graph/Cargo.toml b/crates/lance-csr/Cargo.toml similarity index 85% rename from crates/lance-graph/Cargo.toml rename to crates/lance-csr/Cargo.toml index b054f60..d577c42 100644 --- a/crates/lance-graph/Cargo.toml +++ b/crates/lance-csr/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "lance-graph" +name = "lance-csr" version = "0.2.0" edition = "2021" license = "Apache-2.0" -authors = ["Lance Devs "] -repository = "https://github.com/lancedb/lance-graph" +authors = ["The lance-csr Authors"] +repository = "https://github.com/hyperdimensionalcomputingai/lance-csr" readme = "README.md" description = "Graph query engine for Lance datasets with Cypher support" keywords = ["lance", "graph", "cypher", "query", "datafusion"] @@ -31,7 +31,7 @@ datafusion-expr = "54" datafusion-sql = "54" datafusion-functions-aggregate = "54" futures = "0.3" -lance-graph-catalog = { path = "../lance-graph-catalog", version = "0.2.0" } +lance-csr-catalog = { path = "../lance-csr-catalog", version = "0.2.0" } lance = "13.0.0" lance-linalg = "13.0.0" lance-namespace = "13.0.0" diff --git a/crates/lance-csr/LICENSE b/crates/lance-csr/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/crates/lance-csr/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/lance-csr/NOTICE b/crates/lance-csr/NOTICE new file mode 120000 index 0000000..fb376cf --- /dev/null +++ b/crates/lance-csr/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/crates/lance-graph/README.md b/crates/lance-csr/README.md similarity index 92% rename from crates/lance-graph/README.md rename to crates/lance-csr/README.md index 5fecb3a..ea3856c 100644 --- a/crates/lance-graph/README.md +++ b/crates/lance-csr/README.md @@ -20,7 +20,7 @@ use std::sync::Arc; use arrow_array::{ArrayRef, Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use lance_graph::{CypherQuery, ExecutionStrategy, GraphConfig}; +use lance_csr::{CypherQuery, ExecutionStrategy, GraphConfig}; let config = GraphConfig::builder() .node_table("Person", "person_id") @@ -60,7 +60,7 @@ The query expects a `HashMap` keyed by the labels and relat Graph mappings are declared with `GraphConfig::builder()`: ```rust -use lance_graph::{GraphConfig, NodeMapping, RelationshipMapping}; +use lance_csr::{GraphConfig, NodeMapping, RelationshipMapping}; let config = GraphConfig::builder() .node_table("Person", "person_id") @@ -116,11 +116,11 @@ Basic aggregations like `COUNT` are supported. Optional matches and subqueries a - `config` – Graph configuration types and builders. - `query` – High level `CypherQuery` API and runtime. - `error` – `GraphError` and result helpers. -- `namespace` – Namespace helpers (re-exported from `lance-graph-catalog`). -- `source_catalog` – Catalog helpers for looking up table metadata (re-exported from `lance-graph-catalog`). +- `namespace` – Namespace helpers (re-exported from `lance-csr-catalog`). +- `source_catalog` – Catalog helpers for looking up table metadata (re-exported from `lance-csr-catalog`). -`lance-graph` re-exports the catalog and namespace types from the `lance-graph-catalog` crate for -API compatibility. You can depend on `lance-graph-catalog` directly if you only need catalog or +`lance-csr` re-exports the catalog and namespace types from the `lance-csr-catalog` crate for +API compatibility. You can depend on `lance-csr-catalog` directly if you only need catalog or namespace utilities. ## Error Handling @@ -130,7 +130,7 @@ Most APIs return `Result`. Errors include parsing failures, missi ## Testing ```bash -cargo test -p lance-graph +cargo test -p lance-csr ``` ## Benchmarks diff --git a/crates/lance-graph/src/ast.rs b/crates/lance-csr/src/ast.rs similarity index 100% rename from crates/lance-graph/src/ast.rs rename to crates/lance-csr/src/ast.rs diff --git a/crates/lance-graph/src/case_insensitive.rs b/crates/lance-csr/src/case_insensitive.rs similarity index 98% rename from crates/lance-graph/src/case_insensitive.rs rename to crates/lance-csr/src/case_insensitive.rs index 048a3c7..1ad14bd 100644 --- a/crates/lance-graph/src/case_insensitive.rs +++ b/crates/lance-csr/src/case_insensitive.rs @@ -17,7 +17,7 @@ use std::hash::{Hash, Hasher}; /// # Examples /// /// ``` -/// use lance_graph::case_insensitive::CaseInsensitiveStr; +/// use lance_csr::case_insensitive::CaseInsensitiveStr; /// /// let a = CaseInsensitiveStr::new("Person"); /// let b = CaseInsensitiveStr::new("person"); @@ -97,7 +97,7 @@ pub type CaseInsensitiveMap = HashMap; /// # Examples /// /// ``` -/// use lance_graph::case_insensitive::qualify_column; +/// use lance_csr::case_insensitive::qualify_column; /// /// assert_eq!(qualify_column("Person", "Name"), "person__name"); /// assert_eq!(qualify_column("p", "fullName"), "p__fullname"); @@ -131,7 +131,7 @@ pub fn qualify_column(alias: &str, property: &str) -> String { /// /// ``` /// use std::collections::HashMap; -/// use lance_graph::case_insensitive::CaseInsensitiveLookup; +/// use lance_csr::case_insensitive::CaseInsensitiveLookup; /// /// let mut map = HashMap::new(); /// map.insert("Person".to_string(), 1); diff --git a/crates/lance-graph/src/config.rs b/crates/lance-csr/src/config.rs similarity index 100% rename from crates/lance-graph/src/config.rs rename to crates/lance-csr/src/config.rs diff --git a/crates/lance-graph/src/csr_expand.rs b/crates/lance-csr/src/csr_expand.rs similarity index 100% rename from crates/lance-graph/src/csr_expand.rs rename to crates/lance-csr/src/csr_expand.rs diff --git a/crates/lance-graph/src/csr_index.rs b/crates/lance-csr/src/csr_index.rs similarity index 100% rename from crates/lance-graph/src/csr_index.rs rename to crates/lance-csr/src/csr_index.rs diff --git a/crates/lance-graph/src/datafusion_planner/analysis.rs b/crates/lance-csr/src/datafusion_planner/analysis.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/analysis.rs rename to crates/lance-csr/src/datafusion_planner/analysis.rs diff --git a/crates/lance-graph/src/datafusion_planner/builder/aggregate_ops.rs b/crates/lance-csr/src/datafusion_planner/builder/aggregate_ops.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/builder/aggregate_ops.rs rename to crates/lance-csr/src/datafusion_planner/builder/aggregate_ops.rs diff --git a/crates/lance-graph/src/datafusion_planner/builder/basic_ops.rs b/crates/lance-csr/src/datafusion_planner/builder/basic_ops.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/builder/basic_ops.rs rename to crates/lance-csr/src/datafusion_planner/builder/basic_ops.rs diff --git a/crates/lance-graph/src/datafusion_planner/builder/comprehension_ops.rs b/crates/lance-csr/src/datafusion_planner/builder/comprehension_ops.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/builder/comprehension_ops.rs rename to crates/lance-csr/src/datafusion_planner/builder/comprehension_ops.rs diff --git a/crates/lance-graph/src/datafusion_planner/builder/expand_ops.rs b/crates/lance-csr/src/datafusion_planner/builder/expand_ops.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/builder/expand_ops.rs rename to crates/lance-csr/src/datafusion_planner/builder/expand_ops.rs diff --git a/crates/lance-graph/src/datafusion_planner/builder/helpers.rs b/crates/lance-csr/src/datafusion_planner/builder/helpers.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/builder/helpers.rs rename to crates/lance-csr/src/datafusion_planner/builder/helpers.rs diff --git a/crates/lance-graph/src/datafusion_planner/builder/join_builder.rs b/crates/lance-csr/src/datafusion_planner/builder/join_builder.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/builder/join_builder.rs rename to crates/lance-csr/src/datafusion_planner/builder/join_builder.rs diff --git a/crates/lance-graph/src/datafusion_planner/builder/mod.rs b/crates/lance-csr/src/datafusion_planner/builder/mod.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/builder/mod.rs rename to crates/lance-csr/src/datafusion_planner/builder/mod.rs diff --git a/crates/lance-graph/src/datafusion_planner/config_helpers.rs b/crates/lance-csr/src/datafusion_planner/config_helpers.rs similarity index 99% rename from crates/lance-graph/src/datafusion_planner/config_helpers.rs rename to crates/lance-csr/src/datafusion_planner/config_helpers.rs index af29dad..9f4fcfb 100644 --- a/crates/lance-graph/src/datafusion_planner/config_helpers.rs +++ b/crates/lance-csr/src/datafusion_planner/config_helpers.rs @@ -10,7 +10,7 @@ use super::analysis::PlanningContext; use super::DataFusionPlanner; use crate::config::{NodeMapping, RelationshipMapping}; use crate::error::Result; -use lance_graph_catalog::GraphSourceCatalog; +use lance_csr_catalog::GraphSourceCatalog; use std::sync::Arc; impl DataFusionPlanner { diff --git a/crates/lance-graph/src/datafusion_planner/cypher_udf.rs b/crates/lance-csr/src/datafusion_planner/cypher_udf.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/cypher_udf.rs rename to crates/lance-csr/src/datafusion_planner/cypher_udf.rs diff --git a/crates/lance-graph/src/datafusion_planner/expression.rs b/crates/lance-csr/src/datafusion_planner/expression.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/expression.rs rename to crates/lance-csr/src/datafusion_planner/expression.rs diff --git a/crates/lance-graph/src/datafusion_planner/join_ops.rs b/crates/lance-csr/src/datafusion_planner/join_ops.rs similarity index 99% rename from crates/lance-graph/src/datafusion_planner/join_ops.rs rename to crates/lance-csr/src/datafusion_planner/join_ops.rs index da9ed1c..f96c187 100644 --- a/crates/lance-graph/src/datafusion_planner/join_ops.rs +++ b/crates/lance-csr/src/datafusion_planner/join_ops.rs @@ -16,7 +16,7 @@ use crate::error::Result; use datafusion::logical_expr::{ col, BinaryExpr, Expr, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, TableSource, }; -use lance_graph_catalog::GraphSourceCatalog; +use lance_csr_catalog::GraphSourceCatalog; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/src/datafusion_planner/mod.rs b/crates/lance-csr/src/datafusion_planner/mod.rs similarity index 99% rename from crates/lance-graph/src/datafusion_planner/mod.rs rename to crates/lance-csr/src/datafusion_planner/mod.rs index 6c8d8ab..15bd354 100644 --- a/crates/lance-graph/src/datafusion_planner/mod.rs +++ b/crates/lance-csr/src/datafusion_planner/mod.rs @@ -34,7 +34,7 @@ use crate::config::GraphConfig; use crate::error::Result; use crate::logical_plan::LogicalOperator; use datafusion::logical_expr::LogicalPlan; -use lance_graph_catalog::GraphSourceCatalog; +use lance_csr_catalog::GraphSourceCatalog; use std::sync::Arc; /// Planner abstraction for graph-to-physical planning diff --git a/crates/lance-graph/src/datafusion_planner/scan_ops.rs b/crates/lance-csr/src/datafusion_planner/scan_ops.rs similarity index 99% rename from crates/lance-graph/src/datafusion_planner/scan_ops.rs rename to crates/lance-csr/src/datafusion_planner/scan_ops.rs index 33434f8..83de637 100644 --- a/crates/lance-graph/src/datafusion_planner/scan_ops.rs +++ b/crates/lance-csr/src/datafusion_planner/scan_ops.rs @@ -107,7 +107,7 @@ impl DataFusionPlanner { // No catalog attached - create empty source fallback for flexibility // This allows planners created with DataFusionPlanner::new() to work // without requiring a catalog, though they won't have actual data sources - let empty_source = Arc::new(lance_graph_catalog::SimpleTableSource::empty()); + let empty_source = Arc::new(lance_csr_catalog::SimpleTableSource::empty()); let normalized_label = label.to_lowercase(); let builder = LogicalPlanBuilder::scan(&normalized_label, empty_source, None).map_err(|e| { diff --git a/crates/lance-graph/src/datafusion_planner/test_fixtures.rs b/crates/lance-csr/src/datafusion_planner/test_fixtures.rs similarity index 92% rename from crates/lance-graph/src/datafusion_planner/test_fixtures.rs rename to crates/lance-csr/src/datafusion_planner/test_fixtures.rs index 90ef19e..90bda75 100644 --- a/crates/lance-graph/src/datafusion_planner/test_fixtures.rs +++ b/crates/lance-csr/src/datafusion_planner/test_fixtures.rs @@ -6,7 +6,7 @@ use crate::config::GraphConfig; use crate::logical_plan::LogicalOperator; use arrow_schema::{DataType, Field, Schema}; -use lance_graph_catalog::{InMemoryCatalog, SimpleTableSource}; +use lance_csr_catalog::{InMemoryCatalog, SimpleTableSource}; use std::sync::Arc; pub fn person_schema() -> Arc { @@ -17,7 +17,7 @@ pub fn person_schema() -> Arc { ])) } -pub fn make_catalog() -> Arc { +pub fn make_catalog() -> Arc { let person_src = Arc::new(SimpleTableSource::new(person_schema())); let knows_schema = Arc::new(Schema::new(vec![ Field::new("src_person_id", DataType::Int64, false), diff --git a/crates/lance-graph/src/datafusion_planner/udf.rs b/crates/lance-csr/src/datafusion_planner/udf.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/udf.rs rename to crates/lance-csr/src/datafusion_planner/udf.rs diff --git a/crates/lance-graph/src/datafusion_planner/vector_ops.rs b/crates/lance-csr/src/datafusion_planner/vector_ops.rs similarity index 100% rename from crates/lance-graph/src/datafusion_planner/vector_ops.rs rename to crates/lance-csr/src/datafusion_planner/vector_ops.rs diff --git a/crates/lance-graph/src/error.rs b/crates/lance-csr/src/error.rs similarity index 100% rename from crates/lance-graph/src/error.rs rename to crates/lance-csr/src/error.rs diff --git a/crates/lance-graph/src/key_statistics.rs b/crates/lance-csr/src/key_statistics.rs similarity index 100% rename from crates/lance-graph/src/key_statistics.rs rename to crates/lance-csr/src/key_statistics.rs diff --git a/crates/lance-graph/src/lance_vector_search.rs b/crates/lance-csr/src/lance_vector_search.rs similarity index 99% rename from crates/lance-graph/src/lance_vector_search.rs rename to crates/lance-csr/src/lance_vector_search.rs index a50ca9a..b74cff3 100644 --- a/crates/lance-graph/src/lance_vector_search.rs +++ b/crates/lance-csr/src/lance_vector_search.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -//! Lance Vector Search API for lance-graph +//! Lance Vector Search API for lance-csr //! //! This module provides a flexible API for vector similarity search that can work with: //! - In-memory RecordBatches (brute-force search) @@ -16,8 +16,8 @@ //! # Example //! //! ```ignore -//! use lance_graph::lance_vector_search::VectorSearch; -//! use lance_graph::ast::DistanceMetric; +//! use lance_csr::lance_vector_search::VectorSearch; +//! use lance_csr::ast::DistanceMetric; //! //! // Step 1: Run Cypher query to get candidates //! let candidates = query.execute(datasets, None).await?; diff --git a/crates/lance-graph/src/lib.rs b/crates/lance-csr/src/lib.rs similarity index 92% rename from crates/lance-graph/src/lib.rs rename to crates/lance-csr/src/lib.rs index 37083e7..fbaaca3 100644 --- a/crates/lance-graph/src/lib.rs +++ b/crates/lance-csr/src/lib.rs @@ -18,7 +18,7 @@ //! # Example //! //! ```no_run -//! use lance_graph::{CypherQuery, GraphConfig, Result}; +//! use lance_csr::{CypherQuery, GraphConfig, Result}; //! //! # fn example() -> Result<()> { //! let config = GraphConfig::builder() @@ -61,9 +61,7 @@ pub const MAX_VARIABLE_LENGTH_HOPS: u32 = 20; pub use config::{GraphConfig, NodeMapping, RelationshipMapping}; pub use csr_index::{CsrIndex, CsrIndexBuilder}; pub use error::{GraphError, Result}; -pub use lance_graph_catalog::{ - DirNamespace, GraphSourceCatalog, InMemoryCatalog, SimpleTableSource, -}; +pub use lance_csr_catalog::{DirNamespace, GraphSourceCatalog, InMemoryCatalog, SimpleTableSource}; pub use lance_vector_search::VectorSearch; pub use query::{CypherQuery, ExecutionStrategy, SqlDialect}; pub use sql_query::SqlQuery; diff --git a/crates/lance-graph/src/logical_plan.rs b/crates/lance-csr/src/logical_plan.rs similarity index 100% rename from crates/lance-graph/src/logical_plan.rs rename to crates/lance-csr/src/logical_plan.rs diff --git a/crates/lance-graph/src/parameter_substitution.rs b/crates/lance-csr/src/parameter_substitution.rs similarity index 100% rename from crates/lance-graph/src/parameter_substitution.rs rename to crates/lance-csr/src/parameter_substitution.rs diff --git a/crates/lance-graph/src/parser.rs b/crates/lance-csr/src/parser.rs similarity index 100% rename from crates/lance-graph/src/parser.rs rename to crates/lance-csr/src/parser.rs diff --git a/crates/lance-graph/src/pattern_alternatives.rs b/crates/lance-csr/src/pattern_alternatives.rs similarity index 100% rename from crates/lance-graph/src/pattern_alternatives.rs rename to crates/lance-csr/src/pattern_alternatives.rs diff --git a/crates/lance-graph/src/pattern_order.rs b/crates/lance-csr/src/pattern_order.rs similarity index 100% rename from crates/lance-graph/src/pattern_order.rs rename to crates/lance-csr/src/pattern_order.rs diff --git a/crates/lance-graph/src/query.rs b/crates/lance-csr/src/query.rs similarity index 98% rename from crates/lance-graph/src/query.rs rename to crates/lance-csr/src/query.rs index 3b43c20..c26e68a 100644 --- a/crates/lance-graph/src/query.rs +++ b/crates/lance-csr/src/query.rs @@ -16,7 +16,7 @@ use datafusion_sql::unparser::dialect::{ CustomDialect, DefaultDialect, MySqlDialect, PostgreSqlDialect, SqliteDialect, }; use datafusion_sql::unparser::Unparser; -use lance_graph_catalog::DirNamespace; +use lance_csr_catalog::DirNamespace; use lance_namespace::models::DescribeTableRequest; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -235,7 +235,7 @@ impl CypherQuery { /// ```ignore /// use std::collections::HashMap; /// use arrow::record_batch::RecordBatch; - /// use lance_graph::query::CypherQuery; + /// use lance_csr::query::CypherQuery; /// /// // Create in-memory datasets /// let mut datasets = HashMap::new(); @@ -330,7 +330,7 @@ impl CypherQuery { /// ```ignore /// use std::collections::HashMap; /// use arrow::record_batch::RecordBatch; - /// use lance_graph::query::CypherQuery; + /// use lance_csr::query::CypherQuery; /// /// // Create in-memory datasets /// let mut datasets = HashMap::new(); @@ -443,7 +443,7 @@ impl CypherQuery { /// ```ignore /// use datafusion::execution::context::SessionContext; /// use datafusion::prelude::CsvReadOptions; - /// use lance_graph::{CypherQuery, GraphConfig}; + /// use lance_csr::{CypherQuery, GraphConfig}; /// /// // Step 1: Create GraphConfig /// let config = GraphConfig::builder() @@ -471,7 +471,7 @@ impl CypherQuery { ctx: datafusion::execution::context::SessionContext, ) -> Result { use datafusion::datasource::DefaultTableSource; - use lance_graph_catalog::InMemoryCatalog; + use lance_csr_catalog::InMemoryCatalog; use std::sync::Arc; let config = self.require_config()?; @@ -538,8 +538,8 @@ impl CypherQuery { /// ```ignore /// use std::sync::Arc; /// use datafusion::execution::context::SessionContext; - /// use lance_graph::InMemoryCatalog; - /// use lance_graph::query::CypherQuery; + /// use lance_csr::InMemoryCatalog; + /// use lance_csr::query::CypherQuery; /// /// // Create custom catalog /// let catalog = InMemoryCatalog::new() @@ -556,7 +556,7 @@ impl CypherQuery { /// ``` pub async fn execute_with_catalog_and_context( &self, - catalog: std::sync::Arc, + catalog: std::sync::Arc, ctx: datafusion::execution::context::SessionContext, ) -> Result { use arrow::compute::concat_batches; @@ -595,7 +595,7 @@ impl CypherQuery { /// operators (scans and filters stream; joins, sorts and aggregates may buffer). pub async fn execute_stream_with_catalog_and_context( &self, - catalog: std::sync::Arc, + catalog: std::sync::Arc, ctx: datafusion::execution::context::SessionContext, ) -> Result { let df = self @@ -612,7 +612,7 @@ impl CypherQuery { /// Explain the plan with an explicit catalog and session context. pub async fn explain_with_catalog_and_context( &self, - catalog: std::sync::Arc, + catalog: std::sync::Arc, ctx: datafusion::execution::context::SessionContext, ) -> Result { self.explain_internal(catalog, ctx).await @@ -621,7 +621,7 @@ impl CypherQuery { /// Plan the query (phases 1-3) and hand it to DataFusion (phase 4) without executing. async fn dataframe_with_catalog_and_context( &self, - catalog: std::sync::Arc, + catalog: std::sync::Arc, ctx: datafusion::execution::context::SessionContext, ) -> Result { let (_logical_plan, df_logical_plan) = self.create_logical_plans(catalog)?; @@ -669,12 +669,12 @@ impl CypherQuery { &self, datasets: HashMap, ) -> Result<( - lance_graph_catalog::InMemoryCatalog, + lance_csr_catalog::InMemoryCatalog, datafusion::execution::context::SessionContext, )> { use datafusion::datasource::{DefaultTableSource, MemTable}; use datafusion::execution::context::SessionContext; - use lance_graph_catalog::InMemoryCatalog; + use lance_csr_catalog::InMemoryCatalog; use std::sync::Arc; if datasets.is_empty() { @@ -731,13 +731,13 @@ impl CypherQuery { &self, namespace: std::sync::Arc, ) -> Result<( - lance_graph_catalog::InMemoryCatalog, + lance_csr_catalog::InMemoryCatalog, datafusion::execution::context::SessionContext, )> { use datafusion::datasource::{DefaultTableSource, TableProvider}; use datafusion::execution::context::SessionContext; use lance::datafusion::LanceTableProvider; - use lance_graph_catalog::InMemoryCatalog; + use lance_csr_catalog::InMemoryCatalog; use std::sync::Arc; let config = self.require_config()?; @@ -852,7 +852,7 @@ impl CypherQuery { /// Internal helper to explain the query execution plan with explicit catalog and session context async fn explain_internal( &self, - catalog: std::sync::Arc, + catalog: std::sync::Arc, ctx: datafusion::execution::context::SessionContext, ) -> Result { // Create all plans (phases 1-4) @@ -869,7 +869,7 @@ impl CypherQuery { /// DataFusion logical planning) without creating the physical plan. fn create_logical_plans( &self, - catalog: std::sync::Arc, + catalog: std::sync::Arc, ) -> Result<( crate::logical_plan::LogicalOperator, datafusion::logical_expr::LogicalPlan, @@ -905,7 +905,7 @@ impl CypherQuery { /// Helper to create all plans (graph logical, DataFusion logical, physical) async fn create_plans( &self, - catalog: std::sync::Arc, + catalog: std::sync::Arc, ctx: &datafusion::execution::context::SessionContext, ) -> Result<( crate::logical_plan::LogicalOperator, @@ -1168,8 +1168,8 @@ impl CypherQuery { /// /// # Example /// ```ignore - /// use lance_graph::{CypherQuery, VectorSearch}; - /// use lance_graph::ast::DistanceMetric; + /// use lance_csr::{CypherQuery, VectorSearch}; + /// use lance_csr::ast::DistanceMetric; /// /// let results = query /// .execute_with_vector_rerank( diff --git a/crates/lance-graph/src/semantic.rs b/crates/lance-csr/src/semantic.rs similarity index 100% rename from crates/lance-graph/src/semantic.rs rename to crates/lance-csr/src/semantic.rs diff --git a/crates/lance-graph/src/shortest_path.rs b/crates/lance-csr/src/shortest_path.rs similarity index 100% rename from crates/lance-graph/src/shortest_path.rs rename to crates/lance-csr/src/shortest_path.rs diff --git a/crates/lance-graph/src/spark_dialect.rs b/crates/lance-csr/src/spark_dialect.rs similarity index 100% rename from crates/lance-graph/src/spark_dialect.rs rename to crates/lance-csr/src/spark_dialect.rs diff --git a/crates/lance-graph/src/sql_query.rs b/crates/lance-csr/src/sql_query.rs similarity index 99% rename from crates/lance-graph/src/sql_query.rs rename to crates/lance-csr/src/sql_query.rs index 9cf82f7..691dedc 100644 --- a/crates/lance-graph/src/sql_query.rs +++ b/crates/lance-csr/src/sql_query.rs @@ -23,11 +23,11 @@ use std::sync::Arc; /// # Example /// /// ```no_run -/// use lance_graph::SqlQuery; +/// use lance_csr::SqlQuery; /// use arrow_array::RecordBatch; /// use std::collections::HashMap; /// -/// # async fn example() -> lance_graph::Result<()> { +/// # async fn example() -> lance_csr::Result<()> { /// let mut datasets: HashMap = HashMap::new(); /// // datasets.insert("person".to_string(), person_batch); /// diff --git a/crates/lance-graph/tests/test_case_insensitivity.rs b/crates/lance-csr/tests/test_case_insensitivity.rs similarity index 99% rename from crates/lance-graph/tests/test_case_insensitivity.rs rename to crates/lance-csr/tests/test_case_insensitivity.rs index 8fd59f8..bc5ac1a 100644 --- a/crates/lance-graph/tests/test_case_insensitivity.rs +++ b/crates/lance-csr/tests/test_case_insensitivity.rs @@ -5,7 +5,7 @@ use arrow_array::{BooleanArray, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use lance_graph::{CypherQuery, ExecutionStrategy, GraphConfig}; +use lance_csr::{CypherQuery, ExecutionStrategy, GraphConfig}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_complex_return_clauses.rs b/crates/lance-csr/tests/test_complex_return_clauses.rs similarity index 97% rename from crates/lance-graph/tests/test_complex_return_clauses.rs rename to crates/lance-csr/tests/test_complex_return_clauses.rs index 24e53e3..0e79eb6 100644 --- a/crates/lance-graph/tests/test_complex_return_clauses.rs +++ b/crates/lance-csr/tests/test_complex_return_clauses.rs @@ -1,8 +1,8 @@ use arrow_array::{Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use lance_arrow::SchemaExt; -use lance_graph::config::GraphConfig; -use lance_graph::{CypherQuery, ExecutionStrategy, NodeMapping}; +use lance_csr::config::GraphConfig; +use lance_csr::{CypherQuery, ExecutionStrategy, NodeMapping}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_datafusion_pipeline.rs b/crates/lance-csr/tests/test_datafusion_pipeline.rs similarity index 99% rename from crates/lance-graph/tests/test_datafusion_pipeline.rs rename to crates/lance-csr/tests/test_datafusion_pipeline.rs index 2b2041f..554b1c9 100644 --- a/crates/lance-graph/tests/test_datafusion_pipeline.rs +++ b/crates/lance-csr/tests/test_datafusion_pipeline.rs @@ -1,7 +1,7 @@ use arrow_array::{Array, Float64Array, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use lance_graph::config::GraphConfig; -use lance_graph::{CypherQuery, ExecutionStrategy}; +use lance_csr::config::GraphConfig; +use lance_csr::{CypherQuery, ExecutionStrategy}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_datafusion_scenarios.rs b/crates/lance-csr/tests/test_datafusion_scenarios.rs similarity index 99% rename from crates/lance-graph/tests/test_datafusion_scenarios.rs rename to crates/lance-csr/tests/test_datafusion_scenarios.rs index b77ae44..3e5de71 100644 --- a/crates/lance-graph/tests/test_datafusion_scenarios.rs +++ b/crates/lance-csr/tests/test_datafusion_scenarios.rs @@ -1,7 +1,7 @@ use arrow_array::{BooleanArray, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use lance_graph::config::GraphConfig; -use lance_graph::{CypherQuery, ExecutionStrategy}; +use lance_csr::config::GraphConfig; +use lance_csr::{CypherQuery, ExecutionStrategy}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_datafusion_varlength_complex.rs b/crates/lance-csr/tests/test_datafusion_varlength_complex.rs similarity index 99% rename from crates/lance-graph/tests/test_datafusion_varlength_complex.rs rename to crates/lance-csr/tests/test_datafusion_varlength_complex.rs index cbe51f3..852c1ad 100644 --- a/crates/lance-graph/tests/test_datafusion_varlength_complex.rs +++ b/crates/lance-csr/tests/test_datafusion_varlength_complex.rs @@ -1,7 +1,7 @@ use arrow_array::{Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use lance_graph::config::GraphConfig; -use lance_graph::{CypherQuery, ExecutionStrategy}; +use lance_csr::config::GraphConfig; +use lance_csr::{CypherQuery, ExecutionStrategy}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_datafusion_vector_search.rs b/crates/lance-csr/tests/test_datafusion_vector_search.rs similarity index 99% rename from crates/lance-graph/tests/test_datafusion_vector_search.rs rename to crates/lance-csr/tests/test_datafusion_vector_search.rs index b8d18a7..c87eccc 100644 --- a/crates/lance-graph/tests/test_datafusion_vector_search.rs +++ b/crates/lance-csr/tests/test_datafusion_vector_search.rs @@ -5,8 +5,8 @@ use arrow_array::{Array, FixedSizeListArray, Float32Array, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, FieldRef, Schema}; -use lance_graph::config::GraphConfig; -use lance_graph::{CypherQuery, ExecutionStrategy, Result}; +use lance_csr::config::GraphConfig; +use lance_csr::{CypherQuery, ExecutionStrategy, Result}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_datafusion_with_context.rs b/crates/lance-csr/tests/test_datafusion_with_context.rs similarity index 99% rename from crates/lance-graph/tests/test_datafusion_with_context.rs rename to crates/lance-csr/tests/test_datafusion_with_context.rs index c57acfe..5ae2793 100644 --- a/crates/lance-graph/tests/test_datafusion_with_context.rs +++ b/crates/lance-csr/tests/test_datafusion_with_context.rs @@ -1,6 +1,6 @@ use datafusion::execution::context::SessionContext; -use lance_graph::config::GraphConfig; -use lance_graph::query::CypherQuery; +use lance_csr::config::GraphConfig; +use lance_csr::query::CypherQuery; #[tokio::test] async fn test_execute_with_context_csv_simple() { diff --git a/crates/lance-graph/tests/test_explain_output.rs b/crates/lance-csr/tests/test_explain_output.rs similarity index 99% rename from crates/lance-graph/tests/test_explain_output.rs rename to crates/lance-csr/tests/test_explain_output.rs index 07c4580..3803f84 100644 --- a/crates/lance-graph/tests/test_explain_output.rs +++ b/crates/lance-csr/tests/test_explain_output.rs @@ -1,7 +1,7 @@ use arrow_array::{Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use lance_graph::config::GraphConfig; -use lance_graph::query::CypherQuery; +use lance_csr::config::GraphConfig; +use lance_csr::query::CypherQuery; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_lance_vector_search.rs b/crates/lance-csr/tests/test_lance_vector_search.rs similarity index 98% rename from crates/lance-graph/tests/test_lance_vector_search.rs rename to crates/lance-csr/tests/test_lance_vector_search.rs index 3f6d1e5..1d94804 100644 --- a/crates/lance-graph/tests/test_lance_vector_search.rs +++ b/crates/lance-csr/tests/test_lance_vector_search.rs @@ -5,9 +5,9 @@ use arrow_array::{Array, FixedSizeListArray, Float32Array, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, FieldRef, Schema}; -use lance_graph::ast::DistanceMetric; -use lance_graph::config::GraphConfig; -use lance_graph::{CypherQuery, Result, VectorSearch}; +use lance_csr::ast::DistanceMetric; +use lance_csr::config::GraphConfig; +use lance_csr::{CypherQuery, Result, VectorSearch}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_sql_query.rs b/crates/lance-csr/tests/test_sql_query.rs similarity index 99% rename from crates/lance-graph/tests/test_sql_query.rs rename to crates/lance-csr/tests/test_sql_query.rs index 09baad9..685350e 100644 --- a/crates/lance-graph/tests/test_sql_query.rs +++ b/crates/lance-csr/tests/test_sql_query.rs @@ -5,7 +5,7 @@ use arrow_array::{Float64Array, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use lance_graph::SqlQuery; +use lance_csr::SqlQuery; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_to_spark_sql.rs b/crates/lance-csr/tests/test_to_spark_sql.rs similarity index 99% rename from crates/lance-graph/tests/test_to_spark_sql.rs rename to crates/lance-csr/tests/test_to_spark_sql.rs index 443c582..2800427 100644 --- a/crates/lance-graph/tests/test_to_spark_sql.rs +++ b/crates/lance-csr/tests/test_to_spark_sql.rs @@ -8,7 +8,7 @@ use arrow::array::{Int32Array, StringArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; -use lance_graph::{CypherQuery, GraphConfig, SqlDialect}; +use lance_csr::{CypherQuery, GraphConfig, SqlDialect}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/lance-graph/tests/test_to_sql.rs b/crates/lance-csr/tests/test_to_sql.rs similarity index 99% rename from crates/lance-graph/tests/test_to_sql.rs rename to crates/lance-csr/tests/test_to_sql.rs index 90947fe..3ab5d8f 100644 --- a/crates/lance-graph/tests/test_to_sql.rs +++ b/crates/lance-csr/tests/test_to_sql.rs @@ -8,7 +8,7 @@ use arrow::array::{Int32Array, StringArray}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; -use lance_graph::{CypherQuery, GraphConfig}; +use lance_csr::{CypherQuery, GraphConfig}; use std::collections::HashMap; use std::sync::Arc; diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 1741bdd..70a0376 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -34,7 +34,7 @@ the internals. The CSR index is the project's distinguishing feature, so it gets ## Accuracy comes first - **Document what exists on `main`.** Read the code, tests and docstrings before writing. The Python API is - defined in `crates/lance-graph-python/src/graph.rs`; the Cypher surface in `crates/lance-graph/src/` + defined in `crates/lance-csr-python/src/graph.rs`; the Cypher surface in `crates/lance-csr/src/` (`parser.rs`, `semantic.rs`, `logical_plan.rs`). Never describe planned features as shipped. - **Every Python example runs.** `check_snippets.py` executes each page's `python` blocks in order, in one namespace, in a temporary directory. Run it against a local build before you commit: diff --git a/docs/execution.md b/docs/execution.md index f96bd1a..7039694 100644 --- a/docs/execution.md +++ b/docs/execution.md @@ -11,7 +11,7 @@ flowchart LR B --> C["5–6
DataFusion plan, execute"] --> R["Arrow batches"] ``` -Source paths below are in `crates/lance-graph/src/`. +Source paths below are in `crates/lance-csr/src/`. 1. **Parse** (`parser.rs`). The query text becomes an abstract syntax tree. Syntax errors raise `ValueError` here. diff --git a/docs/project_structure.md b/docs/project_structure.md index 1382766..11cd931 100644 --- a/docs/project_structure.md +++ b/docs/project_structure.md @@ -1,17 +1,17 @@ # Project structure -lance-csr is a Cargo workspace with Python bindings. The Rust crates keep the `lance-graph*` names of the project -it was forked from. +lance-csr is a Cargo workspace with Python bindings. The Rust crates are named `lance-csr*`; the engine and +catalog crates are published on crates.io. ``` . -├── Cargo.toml # Workspace: lance-graph, lance-graph-catalog, lance-graph-benches +├── Cargo.toml # Workspace: lance-csr, lance-csr-catalog, lance-csr-benches ├── crates/ -│ ├── lance-graph/ # The engine: Cypher parser, semantic analysis, logical planner, +│ ├── lance-csr/ # The engine: Cypher parser, semantic analysis, logical planner, │ │ # DataFusion planner, CSR index and traversal operators -│ ├── lance-graph-catalog/ # Graph source catalog and DirNamespace -│ ├── lance-graph-python/ # PyO3 bindings (built by maturin; outside the workspace) -│ └── lance-graph-benches/ # Criterion benchmarks (not published) +│ ├── lance-csr-catalog/ # Graph source catalog and DirNamespace +│ ├── lance-csr-python/ # PyO3 bindings (built by maturin; outside the workspace) +│ └── lance-csr-benches/ # Criterion benchmarks (not published) ├── python/ │ ├── pyproject.toml # The lance-csr package, built with maturin │ ├── python/lance_csr/ # Python package that wraps the bindings @@ -22,7 +22,7 @@ it was forked from. └── docs/ # Documentation ``` -A query goes through four stages, all in `crates/lance-graph/src/`: +A query goes through four stages, all in `crates/lance-csr/src/`: 1. **Parse** (`parser.rs`) the Cypher text into an AST (`ast.rs`). 2. **Analyze** (`semantic.rs`) variables and scopes, and substitute parameters. diff --git a/python/DEVELOPMENT.md b/python/DEVELOPMENT.md index 885a34c..da2bc69 100644 --- a/python/DEVELOPMENT.md +++ b/python/DEVELOPMENT.md @@ -12,7 +12,7 @@ Dependencies are managed with uv and pinned in `uv.lock`. The package is built with [maturin](https://github.com/PyO3/maturin): `pyproject.toml` points it at the bindings crate in -`crates/lance-graph-python/`, so run these commands from this `python/` directory. +`crates/lance-csr-python/`, so run these commands from this `python/` directory. ```shell cd python @@ -70,10 +70,10 @@ cargo clippy --workspace --all-targets -- -D warnings `python/benchmarks/` holds scripts for the engine's own costs: catalog reuse, bounded-memory streaming, CSR expansion and incremental CSR maintenance. Build with `uv run --no-sync maturin develop --release` before running them. The -Rust Criterion benchmarks live in the `lance-graph-benches` crate: +Rust Criterion benchmarks live in the `lance-csr-benches` crate: ```shell -cargo bench -p lance-graph-benches --bench graph_execution -- --warm-up-time 1 +cargo bench -p lance-csr-benches --bench graph_execution -- --warm-up-time 1 ``` The Graph Data Council (GDC), formerly known as the Linked Data Benchmark Council (LDBC), maintains the diff --git a/python/Makefile b/python/Makefile index 6951799..39e89a4 100644 --- a/python/Makefile +++ b/python/Makefile @@ -31,4 +31,4 @@ wheel: $(RUN) maturin build --release --out dist --sdist clean: - rm -rf .venv dist ../crates/lance-graph-python/target + rm -rf .venv dist ../crates/lance-csr-python/target diff --git a/python/README.md b/python/README.md index 99bd547..7a7ffc1 100644 --- a/python/README.md +++ b/python/README.md @@ -223,7 +223,7 @@ Re-run `make install` after changing Rust code; Python changes apply immediately ## Repository layout -- `crates/lance-graph-python/src/` – PyO3 bridge that exposes graph APIs to Python +- `crates/lance-csr-python/src/` – PyO3 bridge that exposes graph APIs to Python - `python/python/lance_csr/` – pure-Python wrapper and `__init__` - `python/benchmarks/` – catalog-reuse and bounded-memory benchmarks - `python/python/tests/` – graph-centric functional tests diff --git a/python/pyproject.toml b/python/pyproject.toml index bb2c8fa..25ba503 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ python-source = "python" module-name = "lance_csr._internal" python-packages = ["lance_csr"] -manifest-path = "../crates/lance-graph-python/Cargo.toml" +manifest-path = "../crates/lance-csr-python/Cargo.toml" [build-system] requires = ["maturin>=1.4"] diff --git a/python/python/lance_csr/__init__.py b/python/python/lance_csr/__init__.py index 196aa72..9d4d52d 100644 --- a/python/python/lance_csr/__init__.py +++ b/python/python/lance_csr/__init__.py @@ -1,4 +1,4 @@ -"""Python bindings for the ``lance-graph`` crate. +"""Python bindings for the ``lance-csr`` crate. Provides two query interfaces: From 9f2892716e879bcc5051fd17f83504d45a631ba6 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Mon, 26 Oct 2026 08:42:55 -0700 Subject: [PATCH 2/2] feat: expose CypherEngine as a Rust API and bump to 0.3.0 Move the cached engine (snapshot, CSR adjacencies, node indexes, join-key statistics and refresh) from the PyO3 bindings into the lance-csr crate as an async CypherEngine. The Python class is now a thin wrapper with the same API and exception types. refresh() now holds an async lock for the whole operation, so overlapping refreshes can no longer publish an older snapshot after a newer one (fixes #12). Add engine tests, a deterministic refresh race test, a runnable embedded_query example that CI runs, and a "Using from Rust" docs page whose snippets are included from that example. Bump the crates and the Python package to 0.3.0. --- .bumpversion.toml | 2 +- .github/workflows/docs.yml | 2 + .github/workflows/rust-test.yml | 3 + Cargo.lock | 6 +- README.md | 4 + crates/lance-csr-benches/Cargo.toml | 4 +- crates/lance-csr-catalog/Cargo.toml | 2 +- crates/lance-csr-python/Cargo.lock | 6 +- crates/lance-csr-python/Cargo.toml | 2 +- crates/lance-csr-python/src/graph.rs | 836 +--------------- crates/lance-csr/Cargo.toml | 6 +- crates/lance-csr/README.md | 63 +- crates/lance-csr/examples/embedded_query.rs | 121 +++ crates/lance-csr/src/engine.rs | 1003 +++++++++++++++++++ crates/lance-csr/src/lib.rs | 26 +- crates/lance-csr/tests/test_engine.rs | 178 ++++ docs/AGENTS.md | 10 +- docs/api-reference.md | 6 +- docs/benchmarks.md | 3 +- docs/getting-started.md | 5 + docs/index.md | 5 +- docs/rust.md | 130 +++ python/pyproject.toml | 2 +- python/uv.lock | 2 +- zensical.toml | 5 + 25 files changed, 1605 insertions(+), 827 deletions(-) create mode 100644 crates/lance-csr/examples/embedded_query.rs create mode 100644 crates/lance-csr/src/engine.rs create mode 100644 crates/lance-csr/tests/test_engine.rs create mode 100644 docs/rust.md diff --git a/.bumpversion.toml b/.bumpversion.toml index dedef9a..789c46b 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.2.0" +current_version = "0.3.0" parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(-(?Palpha|beta|rc)\\.(?P\\d+))?" serialize = [ "{major}.{minor}.{patch}-{pre_label}.{pre_n}", diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9d7ae49..a43c8d9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,6 +7,8 @@ on: paths: - docs/** - zensical.toml + # Pages include this example with --8<--. + - crates/lance-csr/examples/** - .github/workflows/docs.yml concurrency: diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 35f4201..fe668d1 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -38,5 +38,8 @@ jobs: run: cargo test --workspace --no-run - name: Run tests run: cargo test --workspace + # The Rust guide includes this example; run it so the documented code works. + - name: Run embedded example + run: cargo run -p lance-csr --example embedded_query - name: Check benchmarks run: cargo check -p lance-csr-benches --benches diff --git a/Cargo.lock b/Cargo.lock index 1b45076..d92f820 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3947,7 +3947,7 @@ dependencies = [ [[package]] name = "lance-csr" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ahash", "arrow", @@ -3976,7 +3976,7 @@ dependencies = [ [[package]] name = "lance-csr-benches" -version = "0.2.0" +version = "0.3.0" dependencies = [ "arrow-array", "arrow-schema", @@ -3990,7 +3990,7 @@ dependencies = [ [[package]] name = "lance-csr-catalog" -version = "0.2.0" +version = "0.3.0" dependencies = [ "arrow-schema", "async-trait", diff --git a/README.md b/README.md index 92f6ae7..a081a11 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,10 @@ uv add lance-csr Prebuilt wheels cover Linux x86-64 and native Apple Silicon (CPython 3.12+); no Rust toolchain is needed. See [`python/README.md`](python/README.md) for details and usage. +To use the engine from Rust, add the crate with `cargo add lance-csr` and see +[`crates/lance-csr/README.md`](crates/lance-csr/README.md) and the +[Rust guide](https://lance-csr.pages.dev/rust/). + ## Build from source You need: diff --git a/crates/lance-csr-benches/Cargo.toml b/crates/lance-csr-benches/Cargo.toml index 3c03181..f0f4e94 100644 --- a/crates/lance-csr-benches/Cargo.toml +++ b/crates/lance-csr-benches/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "lance-csr-benches" -version = "0.2.0" +version = "0.3.0" edition = "2021" publish = false description = "Benchmarks for lance-csr (not published)" [dependencies] -lance-csr = { path = "../lance-csr", version = "0.2.0" } +lance-csr = { path = "../lance-csr", version = "0.3.0" } lance = "13.0.0" arrow-array = "58" arrow-schema = "58" diff --git a/crates/lance-csr-catalog/Cargo.toml b/crates/lance-csr-catalog/Cargo.toml index ba98c1d..c15ef7b 100644 --- a/crates/lance-csr-catalog/Cargo.toml +++ b/crates/lance-csr-catalog/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lance-csr-catalog" -version = "0.2.0" +version = "0.3.0" edition = "2021" license = "Apache-2.0" authors = ["The lance-csr Authors"] diff --git a/crates/lance-csr-python/Cargo.lock b/crates/lance-csr-python/Cargo.lock index 627603c..693777c 100644 --- a/crates/lance-csr-python/Cargo.lock +++ b/crates/lance-csr-python/Cargo.lock @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "lance-csr" -version = "0.2.0" +version = "0.3.0" dependencies = [ "ahash", "arrow", @@ -3858,7 +3858,7 @@ dependencies = [ [[package]] name = "lance-csr-catalog" -version = "0.2.0" +version = "0.3.0" dependencies = [ "arrow-schema", "async-trait", @@ -3871,7 +3871,7 @@ dependencies = [ [[package]] name = "lance-csr-python" -version = "0.2.0" +version = "0.3.0" dependencies = [ "arrow", "arrow-array", diff --git a/crates/lance-csr-python/Cargo.toml b/crates/lance-csr-python/Cargo.toml index f8bcee3..b67264b 100644 --- a/crates/lance-csr-python/Cargo.toml +++ b/crates/lance-csr-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lance-csr-python" -version = "0.2.0" +version = "0.3.0" edition = "2021" authors = ["The lance-csr Authors"] publish = false diff --git a/crates/lance-csr-python/src/graph.rs b/crates/lance-csr-python/src/graph.rs index c50acbc..c08e7b3 100644 --- a/crates/lance-csr-python/src/graph.rs +++ b/crates/lance-csr-python/src/graph.rs @@ -15,29 +15,21 @@ //! Graph query functionality for Lance datasets use std::collections::HashMap; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use arrow::compute::concat_batches; use arrow::ffi_stream::ArrowArrayStreamReader; use arrow_array::{RecordBatch, RecordBatchReader}; use arrow_schema::Schema; -use datafusion::datasource::{DefaultTableSource, MemTable, TableProvider}; +use datafusion::datasource::MemTable; use datafusion::execution::context::SessionContext; -use datafusion::execution::session_state::SessionStateBuilder; use futures::StreamExt; -use lance::datafusion::LanceTableProvider; -use lance::dataset::builder::DatasetBuilder; -use lance::dataset::Dataset; -use lance_csr::csr_expand::{ - edge_changes, is_supported_id_type, Adjacency, AdjacencyCatalog, CsrQueryPlanner, NodeRows, - PruneCsrTargetColumns, PushCsrTargetFilters, -}; -use lance_csr::key_statistics::KeyStatisticsProvider; use lance_csr::{ ast::{DistanceMetric as RustDistanceMetric, GraphPattern, ReadingClause}, + engine::{CypherEngine as RustCypherEngine, EngineSource}, CypherQuery as RustCypherQuery, ExecutionStrategy as RustExecutionStrategy, - GraphConfig as RustGraphConfig, GraphError as RustGraphError, InMemoryCatalog, - SqlDialect as RustSqlDialect, SqlQuery as RustSqlQuery, VectorSearch as RustVectorSearch, + GraphConfig as RustGraphConfig, GraphError as RustGraphError, SqlDialect as RustSqlDialect, + SqlQuery as RustSqlQuery, VectorSearch as RustVectorSearch, }; use pyo3::{ exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError}, @@ -1164,215 +1156,6 @@ fn record_batch_to_python_table( Ok(table.unbind()) } -/// One input dataset as registered with a [`CypherEngine`]. -#[derive(Clone)] -enum EngineSource { - /// On-disk Lance dataset, read lazily through `LanceTableProvider`. - /// - /// `dataset` is pinned to one version; only `refresh()` replaces it. `uri` and - /// `storage_options` are kept so refresh can reopen the same location with the - /// same credentials. - Lance { - uri: String, - storage_options: HashMap, - dataset: Arc, - }, - /// In-memory Arrow data supplied by the caller. Treated as immutable: replacing - /// it requires a new engine. - Arrow(RecordBatch), -} - -/// Everything a query needs, built once and shared read-only by concurrent queries. -/// -/// `refresh()` builds a complete replacement and swaps it in; queries already running -/// keep the `Arc` they started with, so they finish on their original snapshot. -struct EngineState { - sources: HashMap, - catalog: Arc, - context: SessionContext, - /// CSR adjacencies by relationship type; empty unless the engine uses CSR. - csr: HashMap, - /// Node id -> row indexes by node label, for the fused target lookup; same lifecycle. - node_index: HashMap, - /// Join-key distinct counts by dataset name, reported to DataFusion's join planner. - key_stats: HashMap, -} - -/// Distinct counts for one dataset's node-id and edge-endpoint columns. -#[derive(Clone)] -struct KeyStatsEntry { - /// Dataset the counts describe; reused while it is the same snapshot. - source: EngineSource, - /// Actual column name -> distinct non-null values. - distinct: HashMap, - /// For a relationship dataset: mean fan-out by direction (outgoing, incoming). - fan_out: Option<(String, f64, f64)>, -} - -/// Id -> row index for one node table, plus the source it came from. -#[derive(Clone)] -struct NodeIndexEntry { - source: EngineSource, - rows: Arc, - build_ms: f64, -} - -/// Both-direction adjacencies for one relationship type, plus the source they came from. -#[derive(Clone)] -struct CsrEntry { - /// Dataset the adjacencies reflect; reused while it is the same snapshot. - source: EngineSource, - outgoing: Arc, - incoming: Arc, - /// Wall time of the last build or delta application. - build_ms: f64, - /// How the last refresh produced this entry: build, rebuild, delta or reused. - update: &'static str, -} - -/// Rebuild instead of applying a delta once pending changes (appended edges + -/// tombstones, accumulated since the last full build) would exceed this fraction -/// of the base. Lookups pay for the delta on every query, and a rebuild folds it in. -/// -/// This threshold also bounds refresh cost: each delta refresh copies the whole -/// accumulated delta (see the design note on `lance_csr::csr_expand::Adjacency`). -/// Raising it makes refreshes slower as well as lookups. -// shortcut: fixed threshold, chosen from python/benchmarks/csr_delta_accumulated.py; tune per workload if deltas grow large. -const CSR_REBUILD_FRACTION: f64 = 0.25; - -/// Apply the edge changes between `old`'s snapshot and `source` as a delta. -/// -/// `Ok(None)` means rebuild: not both Lance snapshots, the diff is not meaningful -/// or failed (e.g. old files already cleaned up), or the delta would be too large. -fn try_apply_csr_delta( - py: Python, - old: &CsrEntry, - source: &EngineSource, - src: &str, - dst: &str, -) -> PyResult> { - let ( - EngineSource::Lance { - dataset: old_ds, .. - }, - EngineSource::Lance { - dataset: new_ds, .. - }, - ) = (&old.source, source) - else { - return Ok(None); - }; - let started = std::time::Instant::now(); - let Ok(Some(changes)) = RT.block_on(Some(py), edge_changes(old_ds, new_ds, src, dst))? else { - return Ok(None); - }; - let elapsed = || started.elapsed().as_secs_f64() * 1e3; - if changes.is_empty() { - return Ok(Some(CsrEntry { - source: source.clone(), - build_ms: elapsed(), - update: "reused", - ..old.clone() - })); - } - let base = old.outgoing.base_edges() as f64; - let pending = old.outgoing.delta_edges() + old.outgoing.tombstones() + changes.len() as u64; - if pending as f64 > CSR_REBUILD_FRACTION * base { - return Ok(None); - } - let flip = |edges: &[(i64, i64)]| edges.iter().map(|&(a, b)| (b, a)).collect::>(); - let outgoing = old.outgoing.with_changes(&changes.added, &changes.removed); - let incoming = old - .incoming - .with_changes(&flip(&changes.added), &flip(&changes.removed)); - Ok(Some(CsrEntry { - source: source.clone(), - outgoing: Arc::new(outgoing), - incoming: Arc::new(incoming), - build_ms: elapsed(), - update: "delta", - })) -} - -impl EngineSource { - /// Pinned Lance version, or `None` for PyArrow input. - fn lance_version(&self) -> Option { - match self { - Self::Lance { dataset, .. } => Some(dataset.manifest().version), - Self::Arrow(_) => None, - } - } - - /// True when both refer to the same immutable data (same pinned Lance handle or - /// same Arrow batch), so anything derived from one is valid for the other. - fn same_data(&self, other: &Self) -> bool { - match (self, other) { - (Self::Lance { dataset: a, .. }, Self::Lance { dataset: b, .. }) => Arc::ptr_eq(a, b), - (Self::Arrow(a), Self::Arrow(b)) => { - a.num_columns() == b.num_columns() - && a.columns() - .iter() - .zip(b.columns()) - .all(|(x, y)| x.to_data().ptr_eq(&y.to_data())) - } - _ => false, - } - } -} - -impl EngineState { - /// Mean fan-out of every relationship type with key statistics, both directions. - fn fan_out(&self) -> lance_csr::logical_plan::FanOut { - self.key_stats - .values() - .filter_map(|entry| entry.fan_out.as_ref()) - .flat_map(|(rel_type, outgoing, incoming)| { - [ - ((rel_type.clone(), true), *outgoing), - ((rel_type.clone(), false), *incoming), - ] - }) - .collect() - } - - fn adjacency_catalog(&self) -> AdjacencyCatalog { - let mut catalog = AdjacencyCatalog::default(); - for (rel_type, entry) in &self.csr { - catalog.insert(rel_type, true, entry.outgoing.clone()); - catalog.insert(rel_type, false, entry.incoming.clone()); - } - for (label, entry) in &self.node_index { - catalog.insert_nodes(label, entry.rows.clone()); - } - catalog - } -} - -/// Open a Lance dataset in Rust. `version = None` means the latest version. -fn open_lance_dataset( - py: Python, - uri: &str, - version: Option, - storage_options: &HashMap, -) -> PyResult { - let mut builder = DatasetBuilder::from_uri(uri).with_storage_options(storage_options.clone()); - if let Some(version) = version { - builder = builder.with_version(version); - } - RT.block_on(Some(py), builder.load())?.map_err(|e| { - PyRuntimeError::new_err(format!("Failed to open Lance dataset {}: {}", uri, e)) - }) -} - -/// Same version number and manifest commit time means the same snapshot. -/// -/// The timestamp distinguishes a table deleted and recreated at the same location, -/// which restarts version numbering at 1. -fn same_lance_snapshot(a: &Dataset, b: &Dataset) -> bool { - a.manifest().version == b.manifest().version - && a.manifest().timestamp_nanos == b.manifest().timestamp_nanos -} - /// Convert one Python input value into an engine source without reading its rows. fn python_value_to_source( py: Python, @@ -1400,449 +1183,20 @@ fn python_value_to_source( .extract::>>()? .unwrap_or_default(); // Open the exact version the caller passed so the engine never silently jumps to latest. - let dataset = open_lance_dataset(py, &uri, Some(version), &storage_options)?; - Ok(EngineSource::Lance { - uri, - storage_options, - dataset: Arc::new(dataset), - }) -} - -/// Check that every configured id/endpoint column exists in its dataset. -/// -/// Labels and columns compare case-insensitively, matching the planner. Mappings -/// without a dataset are left for query time, as before, so unused labels stay legal. -fn validate_sources( - config: &RustGraphConfig, - schemas: &HashMap, -) -> PyResult<()> { - let schema_for = |label: &str| { - schemas - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(label)) - .map(|(_, schema)| schema) - }; - let check = |label: &str, column: &str| -> PyResult<()> { - if let Some(schema) = schema_for(label) { - if !schema - .fields() - .iter() - .any(|f| f.name().eq_ignore_ascii_case(column)) - { - return Err(PyValueError::new_err(format!( - "Dataset for '{}' has no column '{}' required by the graph configuration", - label, column - ))); - } - } - Ok(()) - }; - for mapping in config.node_mappings.values() { - check(&mapping.label, &mapping.id_field)?; - } - for mapping in config.relationship_mappings.values() { - check(&mapping.relationship_type, &mapping.source_id_field)?; - check(&mapping.relationship_type, &mapping.target_id_field)?; - } - Ok(()) -} - -/// Validate sources and register them in a fresh catalog and session context. -fn build_engine_state( - py: Python, - config: &RustGraphConfig, - sources: HashMap, - use_csr: bool, - previous: Option<&EngineState>, -) -> PyResult { - // The CSR planner only changes plans that contain CSR nodes, so install it always. - let ctx = SessionContext::new_with_state( - SessionStateBuilder::new() - .with_default_features() - .with_query_planner(Arc::new(CsrQueryPlanner)) - // Move target-only filters into CSR nodes first, so pruning can then drop - // the filter columns from the take. - .with_optimizer_rule(Arc::new(PushCsrTargetFilters)) - .with_optimizer_rule(Arc::new(PruneCsrTargetColumns)) - .build(), - ); - let mut schemas = HashMap::new(); - let mut providers: HashMap> = HashMap::new(); - - for (name, source) in &sources { - let provider: Arc = match source { - // No _rowid/_rowaddr columns, so Lance and Arrow inputs expose the same schema. - EngineSource::Lance { dataset, .. } => { - Arc::new(LanceTableProvider::new(dataset.clone(), false, false)) - } - EngineSource::Arrow(batch) => Arc::new( - MemTable::try_new(batch.schema(), memtable_partitions(batch)).map_err(|e| { - PyRuntimeError::new_err(format!( - "Failed to create MemTable for {}: {}", - name, e - )) - })?, - ), - }; - schemas.insert(name.clone(), provider.schema()); - // Registered unwrapped first: the CSR and key-statistics builds below scan it. - ctx.register_table(name.to_lowercase(), provider.clone()) - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to register table {}: {}", name, e)) - })?; - providers.insert(name.clone(), provider); - } - - validate_sources(config, &schemas)?; - let (csr, node_index) = if use_csr { - ( - build_csr_entries(py, config, &sources, &schemas, &ctx, previous)?, - build_node_index(py, config, &sources, previous)?, - ) - } else { - (HashMap::new(), HashMap::new()) - }; - let key_stats = build_key_statistics(py, config, &sources, &schemas, &ctx, &csr, previous)?; - - let mut catalog = InMemoryCatalog::new(); - for (name, provider) in providers { - // Re-register with join-key statistics, so plans see them; see `key_statistics`. - let provider: Arc = match key_stats.get(&name) { - Some(entry) => Arc::new(KeyStatisticsProvider::new(provider, entry.distinct.clone())), - None => provider, - }; - let table = name.to_lowercase(); - ctx.deregister_table(table.as_str()) - .and_then(|_| ctx.register_table(table.as_str(), provider.clone())) - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to register table {}: {}", name, e)) - })?; - - // Every dataset is registered as both a node and a relationship source; the - // GraphConfig and query pattern decide which interpretation the planner uses. - let table_source = Arc::new(DefaultTableSource::new(provider)); - catalog = catalog - .with_node_source(&name, table_source.clone()) - .with_relationship_source(&name, table_source); - } - Ok(EngineState { - sources, - catalog: Arc::new(catalog), - context: ctx, - csr, - node_index, - key_stats, - }) + RT.block_on( + Some(py), + EngineSource::open_lance(uri, Some(version), storage_options), + )? + .map_err(graph_error_to_pyerr) } -/// Build (or reuse) an id -> row index for every node label with unique integer ids. -/// -/// Reused while the node dataset is the same snapshot; edge changes never touch it. -/// Labels whose ids are not integers or not unique get no entry, so their targets -/// keep the hash join. -fn build_node_index( - py: Python, - config: &RustGraphConfig, - sources: &HashMap, - previous: Option<&EngineState>, -) -> PyResult> { - let mut entries = HashMap::new(); - for mapping in config.node_mappings.values() { - let label = &mapping.label; - let Some(source) = sources - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(label)) - .map(|(_, source)| source) - else { - continue; - }; - if let Some(old) = previous.and_then(|p| p.node_index.get(label)) { - if old.source.same_data(source) { - entries.insert(label.clone(), old.clone()); - continue; - } - } - let started = std::time::Instant::now(); - let rows = match source { - EngineSource::Lance { dataset, .. } => RT.block_on( - Some(py), - NodeRows::from_lance(dataset.clone(), &mapping.id_field), - )?, - EngineSource::Arrow(batch) => NodeRows::from_batch(batch.clone(), &mapping.id_field), - } - .map_err(graph_error_to_pyerr)?; - if let Some(rows) = rows { - entries.insert( - label.clone(), - NodeIndexEntry { - source: source.clone(), - rows: Arc::new(rows), - build_ms: started.elapsed().as_secs_f64() * 1e3, - }, - ); - } +/// Convert a Python ``params`` dict into JSON values for the engine. +fn python_params(params: Option<&Bound<'_, PyDict>>) -> PyResult> { + let mut parameters = HashMap::new(); + for (key, value) in params.into_iter().flat_map(|p| p.iter()) { + parameters.insert(key.extract::()?, python_to_json(&value)?); } - Ok(entries) -} - -/// Rows per in-memory batch, DataFusion's default `batch_size`. -const MEMTABLE_BATCH_ROWS: usize = 8192; - -/// Split one in-memory table into zero-copy slices spread over one partition per core. -/// -/// A `MemTable` holding a single large batch is one partition with one batch, and -/// `RoundRobinBatch` repartitioning moves whole batches, so every operator that does -/// not hash-repartition (such as `CsrExpandExec`) would run on one thread over all of -/// it. Slicing shares the input's buffers; nothing is copied. -fn memtable_partitions(batch: &RecordBatch) -> Vec> { - let partitions = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1); - let mut out = vec![Vec::new(); partitions]; - let rows = batch.num_rows(); - if rows == 0 { - return vec![vec![batch.clone()]]; - } - for (i, offset) in (0..rows).step_by(MEMTABLE_BATCH_ROWS).enumerate() { - let len = MEMTABLE_BATCH_ROWS.min(rows - offset); - out[i % partitions].push(batch.slice(offset, len)); - } - out.retain(|p| !p.is_empty()); - out -} - -/// Distinct counts for every dataset's node-id and edge-endpoint columns. -/// -/// DataFusion 54 sizes hash joins from these (see `lance_csr::key_statistics`). -/// Node ids count as unique (the row count). Edge endpoints use the CSR's distinct -/// source and target counts when the CSR is built, and otherwise one scan of the two -/// endpoint columns; only integer endpoints are counted. Reused while a dataset is -/// the same snapshot, so an unchanged `refresh()` costs nothing. -// shortcut: the fallback scan reads every edge endpoint at construction and on each changed refresh (about one CSR build's scan); switch to an approximate counter (HyperLogLog) or Lance column statistics if that shows up in startup time. -fn build_key_statistics( - py: Python, - config: &RustGraphConfig, - sources: &HashMap, - schemas: &HashMap, - ctx: &SessionContext, - csr: &HashMap, - previous: Option<&EngineState>, -) -> PyResult> { - let mut entries = HashMap::new(); - for (name, source) in sources { - if let Some(old) = previous.and_then(|p| p.key_stats.get(name)) { - if old.source.same_data(source) { - entries.insert(name.clone(), old.clone()); - continue; - } - } - let column = |wanted: &str| { - schemas[name] - .fields() - .iter() - .find(|f| f.name().eq_ignore_ascii_case(wanted)) - .cloned() - }; - let rows = || -> PyResult { - Ok(match source { - EngineSource::Arrow(batch) => batch.num_rows(), - EngineSource::Lance { dataset, .. } => RT - .block_on(Some(py), dataset.count_rows(None))? - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to count rows of {}: {}", name, e)) - })?, - }) - }; - let mut distinct = HashMap::new(); - let mut to_count = Vec::new(); - // Relationship type and endpoint columns, for the fan-out once counts are known. - let mut endpoints = None; - for mapping in config.node_mappings.values() { - if !mapping.label.eq_ignore_ascii_case(name) { - continue; - } - if let Some(id) = column(&mapping.id_field) { - distinct.insert(id.name().clone(), rows()?); - } - } - for mapping in config.relationship_mappings.values() { - if !mapping.relationship_type.eq_ignore_ascii_case(name) { - continue; - } - let (Some(src), Some(dst)) = ( - column(&mapping.source_id_field), - column(&mapping.target_id_field), - ) else { - continue; - }; - endpoints = Some(( - mapping.relationship_type.to_lowercase(), - src.name().clone(), - dst.name().clone(), - )); - match csr.get(&mapping.relationship_type) { - // Built from this same snapshot in this refresh, so its counts are current. - Some(entry) => { - distinct.insert(src.name().clone(), entry.outgoing.num_sources()); - distinct.insert(dst.name().clone(), entry.incoming.num_sources()); - } - None => to_count.extend( - [src, dst] - .into_iter() - .filter(|f| is_supported_id_type(f.data_type())) - .map(|f| f.name().clone()), - ), - } - } - to_count.retain(|c| !distinct.contains_key(c)); - if !to_count.is_empty() { - let table = name.to_lowercase(); - let counts = RT - .block_on(Some(py), async { - let df = ctx.table(table.as_str()).await?; - let names: Vec<&str> = to_count.iter().map(String::as_str).collect(); - let mut stream = df.select_columns(&names)?.execute_stream().await?; - let mut seen: Vec> = - vec![Default::default(); names.len()]; - while let Some(batch) = stream.next().await { - let batch = batch?; - for (i, values) in seen.iter_mut().enumerate() { - let column = arrow::compute::cast( - batch.column(i), - &arrow_schema::DataType::Int64, - )?; - let column = column - .as_any() - .downcast_ref::() - .expect("cast to Int64"); - values.extend(column.iter().flatten()); - } - } - Ok::<_, datafusion::error::DataFusionError>( - seen.into_iter().map(|s| s.len()).collect::>(), - ) - })? - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to count join keys of {}: {}", name, e)) - })?; - distinct.extend(to_count.into_iter().zip(counts)); - } - let fan_out = match endpoints { - Some((rel_type, src, dst)) if distinct.contains_key(&src) => { - let edges = rows()? as f64; - let per = |c: &str| edges / distinct.get(c).copied().unwrap_or(1).max(1) as f64; - Some((rel_type, per(&src), per(&dst))) - } - _ => None, - }; - if !distinct.is_empty() { - entries.insert( - name.clone(), - KeyStatsEntry { - source: source.clone(), - distinct, - fan_out, - }, - ); - } - } - Ok(entries) -} - -/// Build (or reuse) CSR adjacencies for every relationship type with integer endpoints. -/// -/// An entry from `previous` is reused when its relationship dataset is the same -/// snapshot; node-table changes never invalidate it, since the CSR stores raw -/// target ids. Relationship types with non-integer endpoints get no entry and -/// queries over them use the join path. -fn build_csr_entries( - py: Python, - config: &RustGraphConfig, - sources: &HashMap, - schemas: &HashMap, - ctx: &SessionContext, - previous: Option<&EngineState>, -) -> PyResult> { - let mut entries = HashMap::new(); - for mapping in config.relationship_mappings.values() { - let rel_type = &mapping.relationship_type; - let Some((name, source)) = sources - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(rel_type)) - else { - continue; - }; - if let Some(old) = previous.and_then(|p| p.csr.get(rel_type)) { - if old.source.same_data(source) { - entries.insert(rel_type.clone(), old.clone()); - continue; - } - } - let field = |column: &str| { - schemas[name] - .fields() - .iter() - .find(|f| f.name().eq_ignore_ascii_case(column)) - .cloned() - }; - let (Some(src), Some(dst)) = ( - field(&mapping.source_id_field), - field(&mapping.target_id_field), - ) else { - continue; - }; - if !is_supported_id_type(src.data_type()) || !is_supported_id_type(dst.data_type()) { - continue; - } - - let previous_entry = previous.and_then(|p| p.csr.get(rel_type)); - if let Some(old) = previous_entry { - if let Some(entry) = try_apply_csr_delta(py, old, source, src.name(), dst.name())? { - entries.insert(rel_type.clone(), entry); - continue; - } - } - - let started = std::time::Instant::now(); - let table = name.to_lowercase(); - let (outgoing, incoming) = RT.block_on(Some(py), async { - // Two projected scans of the endpoint columns, one per direction. - let scan = || async { - ctx.table(table.as_str()) - .await? - .select_columns(&[src.name().as_str(), dst.name().as_str()])? - .execute_stream() - .await - }; - let scan_err = |e: datafusion::error::DataFusionError| { - PyRuntimeError::new_err(format!("Failed to scan edges for CSR: {}", e)) - }; - let outgoing = - Adjacency::build(scan().await.map_err(scan_err)?, src.name(), dst.name()) - .await - .map_err(graph_error_to_pyerr)?; - let incoming = - Adjacency::build(scan().await.map_err(scan_err)?, dst.name(), src.name()) - .await - .map_err(graph_error_to_pyerr)?; - Ok::<_, PyErr>((outgoing, incoming)) - })??; - entries.insert( - rel_type.clone(), - CsrEntry { - source: source.clone(), - outgoing: Arc::new(outgoing), - incoming: Arc::new(incoming), - build_ms: started.elapsed().as_secs_f64() * 1e3, - update: if previous_entry.is_some() { - "rebuild" - } else { - "build" - }, - }, - ); - } - Ok(entries) + Ok(parameters) } /// Blocking adapter from a DataFusion stream to Arrow's `RecordBatchReader`. @@ -1907,41 +1261,7 @@ impl RecordBatchReader for BlockingBatchReader { /// >>> result3 = engine.execute("MATCH (p:Person) RETURN count(*)") #[pyclass(name = "CypherEngine", module = "lance_csr.graph")] pub struct CypherEngine { - config: RustGraphConfig, - state: RwLock>, - use_csr: bool, -} - -impl CypherEngine { - /// The current snapshot. Cloning the `Arc` lets a query outlive a concurrent refresh. - fn snapshot(&self) -> Arc { - self.state - .read() - .expect("engine state lock poisoned") - .clone() - } - - fn parse( - &self, - query: &str, - params: Option<&Bound<'_, PyDict>>, - state: &EngineState, - ) -> PyResult { - let mut parameters = HashMap::new(); - for (key, value) in params.into_iter().flat_map(|p| p.iter()) { - parameters.insert(key.extract::()?, python_to_json(&value)?); - } - let query = RustCypherQuery::new(query) - .map_err(graph_error_to_pyerr)? - .with_parameters(parameters) - .with_config(self.config.clone()) - .with_fan_out(state.fan_out()); - Ok(if self.use_csr { - query.with_adjacency(state.adjacency_catalog()) - } else { - query - }) - } + inner: RustCypherEngine, } #[pymethods] @@ -1972,21 +1292,19 @@ impl CypherEngine { datasets: &Bound<'_, PyDict>, use_csr: bool, ) -> PyResult { - if datasets.is_empty() { - return Err(PyValueError::new_err("No input datasets provided")); - } let mut sources = HashMap::new(); for (key, value) in datasets.iter() { let name: String = key.extract()?; let source = python_value_to_source(py, &name, &value)?; sources.insert(name, source); } - let state = build_engine_state(py, &config.inner, sources, use_csr, None)?; - Ok(Self { - config: config.inner.clone(), - state: RwLock::new(Arc::new(state)), - use_csr, - }) + let inner = RT + .block_on( + Some(py), + RustCypherEngine::new(config.inner.clone(), sources, use_csr), + )? + .map_err(graph_error_to_pyerr)?; + Ok(Self { inner }) } /// Execute a Cypher query and return the full result as a ``pyarrow.Table``. @@ -2015,16 +1333,10 @@ impl CypherEngine { query: &str, params: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - let state = self.snapshot(); - let cypher_query = self.parse(query, params, &state)?; + let params = python_params(params)?; let result_batch = RT - .block_on( - Some(py), - cypher_query - .execute_with_catalog_and_context(state.catalog.clone(), state.context.clone()), - )? + .block_on(Some(py), self.inner.execute(query, params))? .map_err(graph_error_to_pyerr)?; - record_batch_to_python_table(py, &result_batch) } @@ -2043,16 +1355,9 @@ impl CypherEngine { ) -> PyResult> { use arrow::pyarrow::IntoPyArrow; - let state = self.snapshot(); - let cypher_query = self.parse(query, params, &state)?; + let params = python_params(params)?; let stream = RT - .block_on( - Some(py), - cypher_query.execute_stream_with_catalog_and_context( - state.catalog.clone(), - state.context.clone(), - ), - )? + .block_on(Some(py), self.inner.execute_stream(query, params))? .map_err(graph_error_to_pyerr)?; let reader: Box = Box::new(BlockingBatchReader { schema: stream.schema(), @@ -2069,14 +1374,9 @@ impl CypherEngine { query: &str, params: Option<&Bound<'_, PyDict>>, ) -> PyResult { - let state = self.snapshot(); - let cypher_query = self.parse(query, params, &state)?; - RT.block_on( - Some(py), - cypher_query - .explain_with_catalog_and_context(state.catalog.clone(), state.context.clone()), - )? - .map_err(graph_error_to_pyerr) + let params = python_params(params)?; + RT.block_on(Some(py), self.inner.explain(query, params))? + .map_err(graph_error_to_pyerr) } /// Move every Lance dataset to its latest version. @@ -2084,26 +1384,11 @@ impl CypherEngine { /// Datasets whose version and commit time are unchanged keep their open handle. /// The replacement is validated in full before it is swapped in; on error nothing /// changes and the previous snapshot stays usable. Queries already running finish - /// on the snapshot they started with. PyArrow inputs are unchanged. + /// on the snapshot they started with. PyArrow inputs are unchanged. Concurrent calls + /// from several threads run one at a time. fn refresh(&self, py: Python) -> PyResult<()> { - let current = self.snapshot(); - let mut sources = current.sources.clone(); - for source in sources.values_mut() { - if let EngineSource::Lance { - uri, - storage_options, - dataset, - } = source - { - let latest = open_lance_dataset(py, uri, None, storage_options)?; - if !same_lance_snapshot(&latest, dataset) { - *dataset = Arc::new(latest); - } - } - } - let state = build_engine_state(py, &self.config, sources, self.use_csr, Some(¤t))?; - *self.state.write().expect("engine state lock poisoned") = Arc::new(state); - Ok(()) + RT.block_on(Some(py), self.inner.refresh())? + .map_err(graph_error_to_pyerr) } /// Node id -> row index statistics per node label (experimental, ``use_csr=True``). @@ -2112,10 +1397,10 @@ impl CypherEngine { /// missing when its ids are not unique integers; its targets then use a join. fn node_index_stats(&self, py: Python) -> PyResult> { let stats = PyDict::new(py); - for (label, entry) in &self.snapshot().node_index { + for (label, entry) in self.inner.node_index_stats() { let item = PyDict::new(py); - item.set_item("rows", entry.rows.num_rows())?; - item.set_item("version", entry.source.lance_version())?; + item.set_item("rows", entry.rows)?; + item.set_item("version", entry.version)?; item.set_item("build_ms", entry.build_ms)?; stats.set_item(label, item)?; } @@ -2135,13 +1420,13 @@ impl CypherEngine { /// An identical dict after ``refresh()`` means the snapshot itself was unchanged. fn csr_stats(&self, py: Python) -> PyResult> { let stats = PyDict::new(py); - for (rel_type, entry) in &self.snapshot().csr { + for (rel_type, entry) in self.inner.csr_stats() { let item = PyDict::new(py); - item.set_item("edges", entry.outgoing.num_edges())?; - item.set_item("delta_edges", entry.outgoing.delta_edges())?; - item.set_item("tombstones", entry.outgoing.tombstones())?; + item.set_item("edges", entry.edges)?; + item.set_item("delta_edges", entry.delta_edges)?; + item.set_item("tombstones", entry.tombstones)?; item.set_item("update", entry.update)?; - item.set_item("version", entry.source.lance_version())?; + item.set_item("version", entry.version)?; item.set_item("build_ms", entry.build_ms)?; stats.set_item(rel_type, item)?; } @@ -2150,22 +1435,13 @@ impl CypherEngine { /// Pinned version of each Lance dataset, keyed by dataset name. fn dataset_versions(&self) -> HashMap { - self.snapshot() - .sources - .iter() - .filter_map(|(name, source)| match source { - EngineSource::Lance { dataset, .. } => { - Some((name.clone(), dataset.manifest().version)) - } - EngineSource::Arrow(_) => None, - }) - .collect() + self.inner.dataset_versions() } /// Get the graph configuration fn config(&self) -> GraphConfig { GraphConfig { - inner: self.config.clone(), + inner: self.inner.config().clone(), } } @@ -2191,28 +1467,22 @@ impl CypherEngine { query: &str, vector_search: &VectorSearch, ) -> PyResult> { - let state = self.snapshot(); - let cypher_query = self.parse(query, None, &state)?; - let vs = vector_search.inner.clone(); - - // Execute query to get candidates, then apply vector reranking let result_batch = RT - .block_on(Some(py), async move { - let candidates = cypher_query - .execute_with_catalog_and_context(state.catalog.clone(), state.context.clone()) - .await?; - vs.search(&candidates).await - })? + .block_on( + Some(py), + self.inner + .execute_with_vector_rerank(query, &vector_search.inner), + )? .map_err(graph_error_to_pyerr)?; - record_batch_to_python_table(py, &result_batch) } fn __repr__(&self) -> String { + let config = self.inner.config(); format!( "CypherEngine(nodes={}, relationships={})", - self.config.node_mappings.len(), - self.config.relationship_mappings.len() + config.node_mappings.len(), + config.relationship_mappings.len() ) } } diff --git a/crates/lance-csr/Cargo.toml b/crates/lance-csr/Cargo.toml index d577c42..7c70898 100644 --- a/crates/lance-csr/Cargo.toml +++ b/crates/lance-csr/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lance-csr" -version = "0.2.0" +version = "0.3.0" edition = "2021" license = "Apache-2.0" authors = ["The lance-csr Authors"] @@ -31,7 +31,7 @@ datafusion-expr = "54" datafusion-sql = "54" datafusion-functions-aggregate = "54" futures = "0.3" -lance-csr-catalog = { path = "../lance-csr-catalog", version = "0.2.0" } +lance-csr-catalog = { path = "../lance-csr-catalog", version = "0.3.0" } lance = "13.0.0" lance-linalg = "13.0.0" lance-namespace = "13.0.0" @@ -45,4 +45,4 @@ snafu = "0.8" futures = "0.3" lance-arrow = "13.0.0" tempfile = "3" -tokio = { version = "1.37", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1.37", features = ["macros", "rt-multi-thread", "time"] } diff --git a/crates/lance-csr/README.md b/crates/lance-csr/README.md index ea3856c..f7a7e7c 100644 --- a/crates/lance-csr/README.md +++ b/crates/lance-csr/README.md @@ -1,6 +1,9 @@ -# Lance Graph Query Engine +# lance-csr -A graph query engine for Lance datasets with Cypher syntax support. This crate enables querying Lance's columnar datasets using familiar graph query patterns, interpreting tabular data as property graphs. +A Cypher graph query engine for [Lance](https://lancedb.github.io/lance/) datasets, built on Apache DataFusion, +with a CSR (compressed sparse row) adjacency index for fast multi-hop traversal. It interprets Lance datasets or +Arrow tables as property graphs. The [lance-csr Python package](https://pypi.org/project/lance-csr/) wraps this +crate. ## Features @@ -8,11 +11,44 @@ A graph query engine for Lance datasets with Cypher syntax support. This crate e - Graph configuration for mapping Lance tables to nodes and relationships - Semantic validation with typed `GraphError` diagnostics - Pluggable execution strategies (DataFusion planner by default, Lance Native placeholder) -- Async query execution that returns Arrow `RecordBatch` results +- `CypherEngine`: register datasets once, optionally build the CSR, and serve many queries +- Lance datasets read lazily and pinned to a version until `refresh()` +- Async query execution that returns Arrow `RecordBatch` results or a stream - JSON-serializable parameter binding for reusable query templates - Logical plan debugging via `CypherQuery::explain` -## Quick Start +## Quick start + +```rust +use std::collections::HashMap; +use lance_csr::{CypherEngine, EngineSource, GraphConfig}; + +let config = GraphConfig::builder() + .node_table("Person", "id") + .rel_table("KNOWS", "Person", "Person", "src_id", "dst_id") + .build()?; +let sources = HashMap::from([ + ("Person".to_string(), EngineSource::open_lance("person.lance", None, HashMap::new()).await?), + ("KNOWS".to_string(), EngineSource::open_lance("knows.lance", None, HashMap::new()).await?), +]); +let engine = CypherEngine::new(config, sources, true).await?; +let batch = engine + .execute("MATCH (a:Person)-[:KNOWS]->(b) RETURN a.name, b.name", HashMap::new()) + .await?; +``` + +`examples/embedded_query.rs` is a complete program that writes two Lance datasets, queries them, and picks up a +new write with `refresh()`: + +```bash +cargo run -p lance-csr --example embedded_query +``` + +The [Rust guide](https://lance-csr.pages.dev/rust/) covers the API, and the rest of the +[documentation](https://lance-csr.pages.dev/) covers the graph schema, supported Cypher and the CSR index. +Building the crate needs `protoc` for Lance's dependencies. + +For one query over in-memory batches, `CypherQuery` works without an engine: ```rust use std::collections::HashMap; @@ -96,15 +132,10 @@ All queries use the DataFusion planner for optimization and execution. A builder (`CypherQueryBuilder`) is also available for constructing queries programmatically without parsing text. -## Supported Cypher Surface - -- Node patterns `(:Label)` with optional variables. -- Relationship patterns with fixed direction and type, including multi-hop paths. -- Property comparisons against literal values with `AND`/`OR`/`NOT`/`EXISTS`. -- RETURN lists of property accesses, optional `DISTINCT`, `ORDER BY`, `SKIP` (offset), and `LIMIT`. -- Positional and named parameters (e.g. `$min_age`). +## Supported Cypher -Basic aggregations like `COUNT` are supported. Optional matches and subqueries are parsed but not executed yet. +See [Supported Cypher](https://lance-csr.pages.dev/cypher/#supported-cypher) in the documentation for the +clauses, patterns, expressions and functions the engine runs. ## Crate Layout @@ -114,7 +145,9 @@ Basic aggregations like `COUNT` are supported. Optional matches and subqueries a - `logical_plan` – Builders for graph logical plans. - `datafusion_planner` – DataFusion-based execution planning. - `config` – Graph configuration types and builders. +- `engine` – `CypherEngine`: cached snapshot, CSR adjacencies, `refresh()`. - `query` – High level `CypherQuery` API and runtime. +- `csr_index`, `csr_expand` – The CSR adjacency index and the traversal operators that use it. - `error` – `GraphError` and result helpers. - `namespace` – Namespace helpers (re-exported from `lance-csr-catalog`). - `source_catalog` – Catalog helpers for looking up table metadata (re-exported from `lance-csr-catalog`). @@ -135,7 +168,9 @@ cargo test -p lance-csr ## Benchmarks -See the repository root `README.md` for benchmark setup, run commands, and report locations. +All published numbers were measured through the Python package; see the +[benchmarks page](https://lance-csr.pages.dev/benchmarks/). Criterion micro-benchmarks live in the unpublished +`lance-csr-benches` crate. ## Python Bindings @@ -146,4 +181,4 @@ See the Python package docs for setup and development: ## License -Apache-2.0. See the top-level LICENSE file for details. +Apache-2.0. See `LICENSE` and `NOTICE`. diff --git a/crates/lance-csr/examples/embedded_query.rs b/crates/lance-csr/examples/embedded_query.rs new file mode 100644 index 0000000..d3a0f8d --- /dev/null +++ b/crates/lance-csr/examples/embedded_query.rs @@ -0,0 +1,121 @@ +//! Run Cypher over Lance datasets from Rust, with no Python. +//! +//! Writes a small social graph as two Lance datasets in a temporary directory, opens +//! them in a CSR-backed engine, runs a two-hop query, then appends an edge and picks +//! it up with `refresh()`. The documentation includes this file section by section +//! (the `--8<--` markers), so keep it compiling and its printed output in step. +//! +//! ```bash +//! cargo run -p lance-csr --example embedded_query +//! ``` + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::util::pretty::print_batches; +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use lance::dataset::{Dataset, WriteMode, WriteParams}; +use lance_csr::{CypherEngine, EngineSource, GraphConfig}; + +fn knows(src: Vec, dst: Vec) -> lance_csr::Result { + let schema = Arc::new(Schema::new(vec![ + Field::new("src_id", DataType::Int64, false), + Field::new("dst_id", DataType::Int64, false), + ])); + let columns = vec![ + Arc::new(Int64Array::from(src)) as _, + Arc::new(Int64Array::from(dst)) as _, + ]; + Ok(RecordBatch::try_new(schema, columns)?) +} + +async fn write_lance(batch: RecordBatch, uri: &str, mode: WriteMode) -> lance_csr::Result<()> { + let schema = batch.schema(); + let params = WriteParams { + mode, + ..Default::default() + }; + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + uri, + Some(params), + ) + .await?; + Ok(()) +} + +#[tokio::main] +async fn main() -> lance_csr::Result<()> { + let dir = tempfile::tempdir().expect("create a temporary directory"); + let person_uri = dir.path().join("person.lance").display().to_string(); + let knows_uri = dir.path().join("knows.lance").display().to_string(); + + let person = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4])), + Arc::new(StringArray::from(vec!["Alice", "Bob", "Carol", "David"])), + ], + )?; + write_lance(person, &person_uri, WriteMode::Create).await?; + // Alice knows Bob; Bob knows Carol and David; Carol knows David. + write_lance( + knows(vec![1, 2, 2, 3], vec![2, 3, 4, 4])?, + &knows_uri, + WriteMode::Create, + ) + .await?; + + // --8<-- [start:engine] + let config = GraphConfig::builder() + .node_table("Person", "id") + .rel_table("KNOWS", "Person", "Person", "src_id", "dst_id") + .build()?; + // Each dataset is pinned to the version opened here until `engine.refresh()`. + let sources = HashMap::from([ + ( + "Person".to_string(), + EngineSource::open_lance(person_uri, None, HashMap::new()).await?, + ), + ( + "KNOWS".to_string(), + EngineSource::open_lance(knows_uri.clone(), None, HashMap::new()).await?, + ), + ]); + let engine = CypherEngine::new(config, sources, true).await?; + // --8<-- [end:engine] + + // --8<-- [start:query] + let query = "MATCH (a:Person {name: $name})-[:KNOWS]->()-[:KNOWS]->(c:Person) + RETURN c.name AS friend_of_friend ORDER BY friend_of_friend"; + let params = || HashMap::from([("name".to_string(), serde_json::json!("Alice"))]); + let result = engine.execute(query, params()).await?; + print_batches(&[result])?; + // +------------------+ + // | friend_of_friend | + // +------------------+ + // | Carol | + // | David | + // +------------------+ + // --8<-- [end:query] + + // --8<-- [start:refresh] + // Bob now knows Alice too. The engine doesn't see the write until refresh(). + write_lance(knows(vec![2], vec![1])?, &knows_uri, WriteMode::Append).await?; + engine.refresh().await?; + let result = engine.execute(query, params()).await?; + print_batches(&[result])?; // Alice, Carol, David + + let stats = &engine.csr_stats()["KNOWS"]; + println!( + "{} edges, update={}, version={:?}", + stats.edges, stats.update, stats.version + ); + // 5 edges, update=delta, version=Some(2) + // --8<-- [end:refresh] + Ok(()) +} diff --git a/crates/lance-csr/src/engine.rs b/crates/lance-csr/src/engine.rs new file mode 100644 index 0000000..f87eb51 --- /dev/null +++ b/crates/lance-csr/src/engine.rs @@ -0,0 +1,1003 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! A Cypher engine that registers its datasets once and serves many queries. +//! +//! [`CypherEngine`] holds a snapshot of every input: the DataFusion context, the +//! catalog, join-key statistics and, with `use_csr`, the CSR adjacencies and node +//! indexes. Queries share the snapshot read-only; [`CypherEngine::refresh`] builds a +//! complete replacement and swaps it in. +//! +//! ```no_run +//! use std::collections::HashMap; +//! use lance_csr::engine::{CypherEngine, EngineSource}; +//! use lance_csr::GraphConfig; +//! +//! # async fn example() -> lance_csr::Result<()> { +//! let config = GraphConfig::builder() +//! .node_table("Person", "id") +//! .rel_table("KNOWS", "Person", "Person", "src_id", "dst_id") +//! .build()?; +//! let sources = HashMap::from([ +//! ("Person".to_string(), EngineSource::open_lance("data/person.lance", None, HashMap::new()).await?), +//! ("KNOWS".to_string(), EngineSource::open_lance("data/knows.lance", None, HashMap::new()).await?), +//! ]); +//! let engine = CypherEngine::new(config, sources, true).await?; +//! let batch = engine +//! .execute("MATCH (a:Person)-[:KNOWS]->(b) RETURN a.name, b.name", HashMap::new()) +//! .await?; +//! # Ok(()) +//! # } +//! ``` + +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, RwLock}; + +use arrow_array::RecordBatch; +use arrow_schema::SchemaRef; +use datafusion::datasource::{DefaultTableSource, MemTable, TableProvider}; +use datafusion::execution::context::SessionContext; +use datafusion::execution::session_state::SessionStateBuilder; +use datafusion::execution::SendableRecordBatchStream; +use futures::StreamExt; +use lance::datafusion::LanceTableProvider; +use lance::dataset::builder::DatasetBuilder; +use lance::dataset::Dataset; +use serde_json::Value as JsonValue; + +use crate::config::GraphConfig; +use crate::csr_expand::{ + edge_changes, is_supported_id_type, Adjacency, AdjacencyCatalog, CsrQueryPlanner, NodeRows, + PruneCsrTargetColumns, PushCsrTargetFilters, +}; +use crate::error::{GraphError, Result}; +use crate::key_statistics::KeyStatisticsProvider; +use crate::lance_vector_search::VectorSearch; +use crate::logical_plan::FanOut; +use crate::query::CypherQuery; +use crate::{GraphSourceCatalog, InMemoryCatalog}; + +/// One input dataset as registered with a [`CypherEngine`]. +#[derive(Clone)] +pub enum EngineSource { + /// On-disk Lance dataset, read lazily through `LanceTableProvider`. + /// + /// `dataset` is pinned to one version; only `refresh()` replaces it. `uri` and + /// `storage_options` are kept so refresh can reopen the same location with the + /// same credentials. + Lance { + uri: String, + storage_options: HashMap, + dataset: Arc, + }, + /// In-memory Arrow data supplied by the caller. Treated as immutable: replacing + /// it requires a new engine. + Arrow(RecordBatch), +} + +impl EngineSource { + /// Open a Lance dataset at `uri`, pinned to `version` (`None` means the latest). + /// + /// `storage_options` are kept so [`CypherEngine::refresh`] can reopen the same + /// location with the same credentials. + pub async fn open_lance( + uri: impl Into, + version: Option, + storage_options: HashMap, + ) -> Result { + let uri = uri.into(); + let dataset = open_lance_dataset(&uri, version, &storage_options).await?; + Ok(Self::Lance { + uri, + storage_options, + dataset: Arc::new(dataset), + }) + } + + /// Pinned Lance version, or `None` for Arrow input. + pub fn lance_version(&self) -> Option { + match self { + Self::Lance { dataset, .. } => Some(dataset.manifest().version), + Self::Arrow(_) => None, + } + } + + /// True when both refer to the same immutable data (same pinned Lance handle or + /// same Arrow batch), so anything derived from one is valid for the other. + fn same_data(&self, other: &Self) -> bool { + match (self, other) { + (Self::Lance { dataset: a, .. }, Self::Lance { dataset: b, .. }) => Arc::ptr_eq(a, b), + (Self::Arrow(a), Self::Arrow(b)) => { + a.num_columns() == b.num_columns() + && a.columns() + .iter() + .zip(b.columns()) + .all(|(x, y)| x.to_data().ptr_eq(&y.to_data())) + } + _ => false, + } + } + + /// Row count: free for Arrow input, from Lance metadata otherwise. + async fn num_rows(&self, name: &str) -> Result { + Ok(match self { + Self::Arrow(batch) => batch.num_rows(), + Self::Lance { dataset, .. } => dataset + .count_rows(None) + .await + .map_err(|e| exec_error(format!("Failed to count rows of {}: {}", name, e)))?, + }) + } +} + +/// CSR adjacency statistics for one relationship type; see [`CypherEngine::csr_stats`]. +#[derive(Debug, Clone, PartialEq)] +pub struct CsrStats { + /// Live indexed edges (null endpoints are skipped). + pub edges: u64, + /// Appended edges held outside the base CSR since its last full build. + pub delta_edges: u64, + /// Removed edges held outside the base CSR since its last full build. + pub tombstones: u64, + /// How the last refresh produced it: `build` (first), `rebuild` (full), `delta` + /// (changes applied) or `reused` (no edge change). + pub update: &'static str, + /// Relationship dataset version it reflects (`None` for Arrow input). + pub version: Option, + /// Wall time of that build or delta, both directions. + pub build_ms: f64, +} + +/// Node id -> row index statistics for one node label; see [`CypherEngine::node_index_stats`]. +#[derive(Debug, Clone, PartialEq)] +pub struct NodeIndexStats { + /// Indexed node rows. + pub rows: usize, + /// Node dataset version it reflects (`None` for Arrow input). + pub version: Option, + /// Wall time of the build. + pub build_ms: f64, +} + +/// Everything a query needs, built once and shared read-only by concurrent queries. +/// +/// `refresh()` builds a complete replacement and swaps it in; queries already running +/// keep the `Arc` they started with, so they finish on their original snapshot. +struct EngineState { + sources: HashMap, + catalog: Arc, + context: SessionContext, + /// CSR adjacencies by relationship type; empty unless the engine uses CSR. + csr: HashMap, + /// Node id -> row indexes by node label, for the fused target lookup; same lifecycle. + node_index: HashMap, + /// Join-key distinct counts by dataset name, reported to DataFusion's join planner. + key_stats: HashMap, +} + +/// Distinct counts for one dataset's node-id and edge-endpoint columns. +#[derive(Clone)] +struct KeyStatsEntry { + /// Dataset the counts describe; reused while it is the same snapshot. + source: EngineSource, + /// Actual column name -> distinct non-null values. + distinct: HashMap, + /// For a relationship dataset: mean fan-out by direction (outgoing, incoming). + fan_out: Option<(String, f64, f64)>, +} + +/// Id -> row index for one node table, plus the source it came from. +#[derive(Clone)] +struct NodeIndexEntry { + source: EngineSource, + rows: Arc, + build_ms: f64, +} + +/// Both-direction adjacencies for one relationship type, plus the source they came from. +#[derive(Clone)] +struct CsrEntry { + /// Dataset the adjacencies reflect; reused while it is the same snapshot. + source: EngineSource, + outgoing: Arc, + incoming: Arc, + /// Wall time of the last build or delta application. + build_ms: f64, + /// How the last refresh produced this entry: build, rebuild, delta or reused. + update: &'static str, +} + +/// Rebuild instead of applying a delta once pending changes (appended edges + +/// tombstones, accumulated since the last full build) would exceed this fraction +/// of the base. Lookups pay for the delta on every query, and a rebuild folds it in. +/// +/// This threshold also bounds refresh cost: each delta refresh copies the whole +/// accumulated delta (see the design note on `crate::csr_expand::Adjacency`). +/// Raising it makes refreshes slower as well as lookups. +// shortcut: fixed threshold, chosen from python/benchmarks/csr_delta_accumulated.py; tune per workload if deltas grow large. +const CSR_REBUILD_FRACTION: f64 = 0.25; + +/// Apply the edge changes between `old`'s snapshot and `source` as a delta. +/// +/// `None` means rebuild: not both Lance snapshots, the diff is not meaningful +/// or failed (e.g. old files already cleaned up), or the delta would be too large. +async fn try_apply_csr_delta( + old: &CsrEntry, + source: &EngineSource, + src: &str, + dst: &str, +) -> Option { + let ( + EngineSource::Lance { + dataset: old_ds, .. + }, + EngineSource::Lance { + dataset: new_ds, .. + }, + ) = (&old.source, source) + else { + return None; + }; + let started = std::time::Instant::now(); + let Ok(Some(changes)) = edge_changes(old_ds, new_ds, src, dst).await else { + return None; + }; + let elapsed = || started.elapsed().as_secs_f64() * 1e3; + if changes.is_empty() { + return Some(CsrEntry { + source: source.clone(), + build_ms: elapsed(), + update: "reused", + ..old.clone() + }); + } + let base = old.outgoing.base_edges() as f64; + let pending = old.outgoing.delta_edges() + old.outgoing.tombstones() + changes.len() as u64; + if pending as f64 > CSR_REBUILD_FRACTION * base { + return None; + } + let flip = |edges: &[(i64, i64)]| edges.iter().map(|&(a, b)| (b, a)).collect::>(); + let outgoing = old.outgoing.with_changes(&changes.added, &changes.removed); + let incoming = old + .incoming + .with_changes(&flip(&changes.added), &flip(&changes.removed)); + Some(CsrEntry { + source: source.clone(), + outgoing: Arc::new(outgoing), + incoming: Arc::new(incoming), + build_ms: elapsed(), + update: "delta", + }) +} + +impl EngineState { + /// Mean fan-out of every relationship type with key statistics, both directions. + fn fan_out(&self) -> FanOut { + self.key_stats + .values() + .filter_map(|entry| entry.fan_out.as_ref()) + .flat_map(|(rel_type, outgoing, incoming)| { + [ + ((rel_type.clone(), true), *outgoing), + ((rel_type.clone(), false), *incoming), + ] + }) + .collect() + } + + fn adjacency_catalog(&self) -> AdjacencyCatalog { + let mut catalog = AdjacencyCatalog::default(); + for (rel_type, entry) in &self.csr { + catalog.insert(rel_type, true, entry.outgoing.clone()); + catalog.insert(rel_type, false, entry.incoming.clone()); + } + for (label, entry) in &self.node_index { + catalog.insert_nodes(label, entry.rows.clone()); + } + catalog + } +} + +fn exec_error(message: String) -> GraphError { + GraphError::ExecutionError { + message, + location: snafu::Location::new(file!(), line!(), column!()), + } +} + +fn config_error(message: String) -> GraphError { + GraphError::ConfigError { + message, + location: snafu::Location::new(file!(), line!(), column!()), + } +} + +/// Open a Lance dataset. `version = None` means the latest version. +async fn open_lance_dataset( + uri: &str, + version: Option, + storage_options: &HashMap, +) -> Result { + let mut builder = DatasetBuilder::from_uri(uri).with_storage_options(storage_options.clone()); + if let Some(version) = version { + builder = builder.with_version(version); + } + builder + .load() + .await + .map_err(|e| exec_error(format!("Failed to open Lance dataset {}: {}", uri, e))) +} + +/// Same version number and manifest commit time means the same snapshot. +/// +/// The timestamp distinguishes a table deleted and recreated at the same location, +/// which restarts version numbering at 1. +fn same_lance_snapshot(a: &Dataset, b: &Dataset) -> bool { + a.manifest().version == b.manifest().version + && a.manifest().timestamp_nanos == b.manifest().timestamp_nanos +} + +/// Check that every configured id/endpoint column exists in its dataset. +/// +/// Labels and columns compare case-insensitively, matching the planner. Mappings +/// without a dataset are left for query time, as before, so unused labels stay legal. +fn validate_sources(config: &GraphConfig, schemas: &HashMap) -> Result<()> { + let schema_for = |label: &str| { + schemas + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(label)) + .map(|(_, schema)| schema) + }; + let check = |label: &str, column: &str| -> Result<()> { + if let Some(schema) = schema_for(label) { + if !schema + .fields() + .iter() + .any(|f| f.name().eq_ignore_ascii_case(column)) + { + return Err(config_error(format!( + "Dataset for '{}' has no column '{}' required by the graph configuration", + label, column + ))); + } + } + Ok(()) + }; + for mapping in config.node_mappings.values() { + check(&mapping.label, &mapping.id_field)?; + } + for mapping in config.relationship_mappings.values() { + check(&mapping.relationship_type, &mapping.source_id_field)?; + check(&mapping.relationship_type, &mapping.target_id_field)?; + } + Ok(()) +} + +/// Validate sources and register them in a fresh catalog and session context. +async fn build_engine_state( + config: &GraphConfig, + sources: HashMap, + use_csr: bool, + previous: Option<&EngineState>, +) -> Result { + // The CSR planner only changes plans that contain CSR nodes, so install it always. + let ctx = SessionContext::new_with_state( + SessionStateBuilder::new() + .with_default_features() + .with_query_planner(Arc::new(CsrQueryPlanner)) + // Move target-only filters into CSR nodes first, so pruning can then drop + // the filter columns from the take. + .with_optimizer_rule(Arc::new(PushCsrTargetFilters)) + .with_optimizer_rule(Arc::new(PruneCsrTargetColumns)) + .build(), + ); + let mut schemas = HashMap::new(); + let mut providers: HashMap> = HashMap::new(); + + for (name, source) in &sources { + let provider: Arc = match source { + // No _rowid/_rowaddr columns, so Lance and Arrow inputs expose the same schema. + EngineSource::Lance { dataset, .. } => { + Arc::new(LanceTableProvider::new(dataset.clone(), false, false)) + } + EngineSource::Arrow(batch) => Arc::new( + MemTable::try_new(batch.schema(), memtable_partitions(batch)).map_err(|e| { + exec_error(format!("Failed to create MemTable for {}: {}", name, e)) + })?, + ), + }; + schemas.insert(name.clone(), provider.schema()); + // Registered unwrapped first: the CSR and key-statistics builds below scan it. + ctx.register_table(name.to_lowercase(), provider.clone()) + .map_err(|e| exec_error(format!("Failed to register table {}: {}", name, e)))?; + providers.insert(name.clone(), provider); + } + + validate_sources(config, &schemas)?; + let (csr, node_index) = if use_csr { + ( + build_csr_entries(config, &sources, &schemas, &ctx, previous).await?, + build_node_index(config, &sources, previous).await?, + ) + } else { + (HashMap::new(), HashMap::new()) + }; + let key_stats = build_key_statistics(config, &sources, &schemas, &ctx, &csr, previous).await?; + + let mut catalog = InMemoryCatalog::new(); + for (name, provider) in providers { + // Re-register with join-key statistics, so plans see them; see `key_statistics`. + let provider: Arc = match key_stats.get(&name) { + Some(entry) => Arc::new(KeyStatisticsProvider::new(provider, entry.distinct.clone())), + None => provider, + }; + let table = name.to_lowercase(); + ctx.deregister_table(table.as_str()) + .and_then(|_| ctx.register_table(table.as_str(), provider.clone())) + .map_err(|e| exec_error(format!("Failed to register table {}: {}", name, e)))?; + + // Every dataset is registered as both a node and a relationship source; the + // GraphConfig and query pattern decide which interpretation the planner uses. + let table_source = Arc::new(DefaultTableSource::new(provider)); + catalog = catalog + .with_node_source(&name, table_source.clone()) + .with_relationship_source(&name, table_source); + } + Ok(EngineState { + sources, + catalog: Arc::new(catalog), + context: ctx, + csr, + node_index, + key_stats, + }) +} + +/// Build (or reuse) an id -> row index for every node label with unique integer ids. +/// +/// Reused while the node dataset is the same snapshot; edge changes never touch it. +/// Labels whose ids are not integers or not unique get no entry, so their targets +/// keep the hash join. +async fn build_node_index( + config: &GraphConfig, + sources: &HashMap, + previous: Option<&EngineState>, +) -> Result> { + let mut entries = HashMap::new(); + for mapping in config.node_mappings.values() { + let label = &mapping.label; + let Some(source) = sources + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(label)) + .map(|(_, source)| source) + else { + continue; + }; + if let Some(old) = previous.and_then(|p| p.node_index.get(label)) { + if old.source.same_data(source) { + entries.insert(label.clone(), old.clone()); + continue; + } + } + let started = std::time::Instant::now(); + let rows = match source { + EngineSource::Lance { dataset, .. } => { + NodeRows::from_lance(dataset.clone(), &mapping.id_field).await? + } + EngineSource::Arrow(batch) => NodeRows::from_batch(batch.clone(), &mapping.id_field)?, + }; + if let Some(rows) = rows { + entries.insert( + label.clone(), + NodeIndexEntry { + source: source.clone(), + rows: Arc::new(rows), + build_ms: started.elapsed().as_secs_f64() * 1e3, + }, + ); + } + } + Ok(entries) +} + +/// Rows per in-memory batch, DataFusion's default `batch_size`. +const MEMTABLE_BATCH_ROWS: usize = 8192; + +/// Split one in-memory table into zero-copy slices spread over one partition per core. +/// +/// A `MemTable` holding a single large batch is one partition with one batch, and +/// `RoundRobinBatch` repartitioning moves whole batches, so every operator that does +/// not hash-repartition (such as `CsrExpandExec`) would run on one thread over all of +/// it. Slicing shares the input's buffers; nothing is copied. +fn memtable_partitions(batch: &RecordBatch) -> Vec> { + let partitions = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + let mut out = vec![Vec::new(); partitions]; + let rows = batch.num_rows(); + if rows == 0 { + return vec![vec![batch.clone()]]; + } + for (i, offset) in (0..rows).step_by(MEMTABLE_BATCH_ROWS).enumerate() { + let len = MEMTABLE_BATCH_ROWS.min(rows - offset); + out[i % partitions].push(batch.slice(offset, len)); + } + out.retain(|p| !p.is_empty()); + out +} + +/// Distinct counts for every dataset's node-id and edge-endpoint columns. +/// +/// DataFusion 54 sizes hash joins from these (see `crate::key_statistics`). +/// Node ids count as unique (the row count). Edge endpoints use the CSR's distinct +/// source and target counts when the CSR is built, and otherwise one scan of the two +/// endpoint columns; only integer endpoints are counted. Reused while a dataset is +/// the same snapshot, so an unchanged `refresh()` costs nothing. +// shortcut: the fallback scan reads every edge endpoint at construction and on each changed refresh (about one CSR build's scan); switch to an approximate counter (HyperLogLog) or Lance column statistics if that shows up in startup time. +async fn build_key_statistics( + config: &GraphConfig, + sources: &HashMap, + schemas: &HashMap, + ctx: &SessionContext, + csr: &HashMap, + previous: Option<&EngineState>, +) -> Result> { + let mut entries = HashMap::new(); + for (name, source) in sources { + if let Some(old) = previous.and_then(|p| p.key_stats.get(name)) { + if old.source.same_data(source) { + entries.insert(name.clone(), old.clone()); + continue; + } + } + let column = |wanted: &str| { + schemas[name] + .fields() + .iter() + .find(|f| f.name().eq_ignore_ascii_case(wanted)) + .cloned() + }; + let mut distinct = HashMap::new(); + let mut to_count = Vec::new(); + // Relationship type and endpoint columns, for the fan-out once counts are known. + let mut endpoints = None; + for mapping in config.node_mappings.values() { + if !mapping.label.eq_ignore_ascii_case(name) { + continue; + } + if let Some(id) = column(&mapping.id_field) { + distinct.insert(id.name().clone(), source.num_rows(name).await?); + } + } + for mapping in config.relationship_mappings.values() { + if !mapping.relationship_type.eq_ignore_ascii_case(name) { + continue; + } + let (Some(src), Some(dst)) = ( + column(&mapping.source_id_field), + column(&mapping.target_id_field), + ) else { + continue; + }; + endpoints = Some(( + mapping.relationship_type.to_lowercase(), + src.name().clone(), + dst.name().clone(), + )); + match csr.get(&mapping.relationship_type) { + // Built from this same snapshot in this refresh, so its counts are current. + Some(entry) => { + distinct.insert(src.name().clone(), entry.outgoing.num_sources()); + distinct.insert(dst.name().clone(), entry.incoming.num_sources()); + } + None => to_count.extend( + [src, dst] + .into_iter() + .filter(|f| is_supported_id_type(f.data_type())) + .map(|f| f.name().clone()), + ), + } + } + to_count.retain(|c| !distinct.contains_key(c)); + if !to_count.is_empty() { + let table = name.to_lowercase(); + let counts = async { + let df = ctx.table(table.as_str()).await?; + let names: Vec<&str> = to_count.iter().map(String::as_str).collect(); + let mut stream = df.select_columns(&names)?.execute_stream().await?; + let mut seen: Vec> = vec![Default::default(); names.len()]; + while let Some(batch) = stream.next().await { + let batch = batch?; + for (i, values) in seen.iter_mut().enumerate() { + let column = + arrow::compute::cast(batch.column(i), &arrow_schema::DataType::Int64)?; + let column = column + .as_any() + .downcast_ref::() + .expect("cast to Int64"); + values.extend(column.iter().flatten()); + } + } + Ok::<_, datafusion::error::DataFusionError>( + seen.into_iter().map(|s| s.len()).collect::>(), + ) + } + .await + .map_err(|e| exec_error(format!("Failed to count join keys of {}: {}", name, e)))?; + distinct.extend(to_count.into_iter().zip(counts)); + } + let fan_out = match endpoints { + Some((rel_type, src, dst)) if distinct.contains_key(&src) => { + let edges = source.num_rows(name).await? as f64; + let per = |c: &str| edges / distinct.get(c).copied().unwrap_or(1).max(1) as f64; + Some((rel_type, per(&src), per(&dst))) + } + _ => None, + }; + if !distinct.is_empty() { + entries.insert( + name.clone(), + KeyStatsEntry { + source: source.clone(), + distinct, + fan_out, + }, + ); + } + } + Ok(entries) +} + +/// Build (or reuse) CSR adjacencies for every relationship type with integer endpoints. +/// +/// An entry from `previous` is reused when its relationship dataset is the same +/// snapshot; node-table changes never invalidate it, since the CSR stores raw +/// target ids. Relationship types with non-integer endpoints get no entry and +/// queries over them use the join path. +async fn build_csr_entries( + config: &GraphConfig, + sources: &HashMap, + schemas: &HashMap, + ctx: &SessionContext, + previous: Option<&EngineState>, +) -> Result> { + let mut entries = HashMap::new(); + for mapping in config.relationship_mappings.values() { + let rel_type = &mapping.relationship_type; + let Some((name, source)) = sources + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(rel_type)) + else { + continue; + }; + if let Some(old) = previous.and_then(|p| p.csr.get(rel_type)) { + if old.source.same_data(source) { + entries.insert(rel_type.clone(), old.clone()); + continue; + } + } + let field = |column: &str| { + schemas[name] + .fields() + .iter() + .find(|f| f.name().eq_ignore_ascii_case(column)) + .cloned() + }; + let (Some(src), Some(dst)) = ( + field(&mapping.source_id_field), + field(&mapping.target_id_field), + ) else { + continue; + }; + if !is_supported_id_type(src.data_type()) || !is_supported_id_type(dst.data_type()) { + continue; + } + + let previous_entry = previous.and_then(|p| p.csr.get(rel_type)); + if let Some(old) = previous_entry { + if let Some(entry) = try_apply_csr_delta(old, source, src.name(), dst.name()).await { + entries.insert(rel_type.clone(), entry); + continue; + } + } + + let started = std::time::Instant::now(); + let table = name.to_lowercase(); + // Two projected scans of the endpoint columns, one per direction. + let scan = || async { + ctx.table(table.as_str()) + .await? + .select_columns(&[src.name().as_str(), dst.name().as_str()])? + .execute_stream() + .await + .map_err(|e| exec_error(format!("Failed to scan edges for CSR: {}", e))) + }; + let outgoing = Adjacency::build(scan().await?, src.name(), dst.name()).await?; + let incoming = Adjacency::build(scan().await?, dst.name(), src.name()).await?; + entries.insert( + rel_type.clone(), + CsrEntry { + source: source.clone(), + outgoing: Arc::new(outgoing), + incoming: Arc::new(incoming), + build_ms: started.elapsed().as_secs_f64() * 1e3, + update: if previous_entry.is_some() { + "rebuild" + } else { + "build" + }, + }, + ); + } + Ok(entries) +} + +/// Cypher query engine with a cached catalog for efficient multi-query execution. +/// +/// The engine registers each dataset once and reuses the catalog and DataFusion +/// context across queries. +/// +/// Lance datasets are read lazily from storage, pinned to the version they were +/// opened at: writes made later (by this or another process) are not visible until +/// [`refresh`](Self::refresh). Each dataset is pinned separately; versions across +/// tables are not a transactional multi-table snapshot. Arrow batches are held in +/// memory and never change; build a new engine to replace them. +/// +/// Building the CSR (`use_csr`) is CPU-bound work inside an async function: in an +/// application with its own async runtime, construct and refresh the engine on a +/// blocking thread (for example `tokio::task::spawn_blocking` plus `block_on`). +pub struct CypherEngine { + config: GraphConfig, + state: RwLock>, + use_csr: bool, + /// Held for a whole refresh, so a slower refresh can't swap in older data over a newer one. + refresh_lock: futures::lock::Mutex<()>, +} + +impl CypherEngine { + /// Register `sources` (dataset name -> data) and build the snapshot. + /// + /// Lance datasets are not read here, except for the CSR, node-index and + /// join-key statistics builds. With `use_csr`, relationship types with integer + /// endpoints get CSR adjacencies and node labels with unique integer ids get a + /// node index. + /// + /// # Errors + /// [`GraphError::ConfigError`] if `sources` is empty or a configured id/endpoint + /// column is missing; [`GraphError::ExecutionError`] if a dataset cannot be + /// registered or scanned. + pub async fn new( + config: GraphConfig, + sources: HashMap, + use_csr: bool, + ) -> Result { + if sources.is_empty() { + return Err(config_error("No input datasets provided".to_string())); + } + let state = build_engine_state(&config, sources, use_csr, None).await?; + Ok(Self { + config, + state: RwLock::new(Arc::new(state)), + use_csr, + refresh_lock: Default::default(), + }) + } + + /// The current snapshot. Cloning the `Arc` lets a query outlive a concurrent refresh. + fn snapshot(&self) -> Arc { + self.state + .read() + .expect("engine state lock poisoned") + .clone() + } + + fn parse( + &self, + query: &str, + params: HashMap, + state: &EngineState, + ) -> Result { + let query = CypherQuery::new(query)? + .with_parameters(params) + .with_config(self.config.clone()) + .with_fan_out(state.fan_out()); + Ok(if self.use_csr { + query.with_adjacency(state.adjacency_catalog()) + } else { + query + }) + } + + /// Execute a Cypher query and return the whole result as one batch. + /// + /// `params` gives values for `$name` parameters in the query. Use + /// [`execute_stream`](Self::execute_stream) for large outputs. + pub async fn execute( + &self, + query: &str, + params: HashMap, + ) -> Result { + let state = self.snapshot(); + self.parse(query, params, &state)? + .execute_with_catalog_and_context(state.catalog.clone(), state.context.clone()) + .await + } + + /// Execute a Cypher query and stream the result. + /// + /// Batches are computed as the stream is polled. Scans, filters and projections + /// stream with bounded memory; joins, sorts and aggregations may still buffer. + /// The stream keeps what it reads alive, so a concurrent `refresh()` does not + /// affect it. Planning errors return here; execution errors arrive in the stream. + /// Dropping the stream cancels the remaining work. + pub async fn execute_stream( + &self, + query: &str, + params: HashMap, + ) -> Result { + let state = self.snapshot(); + self.parse(query, params, &state)? + .execute_stream_with_catalog_and_context(state.catalog.clone(), state.context.clone()) + .await + } + + /// Return the query plan, including the physical plan with pushed-down scans. + pub async fn explain(&self, query: &str, params: HashMap) -> Result { + let state = self.snapshot(); + self.parse(query, params, &state)? + .explain_with_catalog_and_context(state.catalog.clone(), state.context.clone()) + .await + } + + /// Execute a Cypher query, then rerank its rows by vector similarity. + pub async fn execute_with_vector_rerank( + &self, + query: &str, + vector_search: &VectorSearch, + ) -> Result { + let candidates = self.execute(query, HashMap::new()).await?; + vector_search.search(&candidates).await + } + + /// Move every Lance dataset to its latest version. + /// + /// Datasets whose version and commit time are unchanged keep their open handle. + /// The replacement is validated in full before it is swapped in; on error nothing + /// changes and the previous snapshot stays usable. Queries already running finish + /// on the snapshot they started with. Arrow inputs are unchanged. Concurrent calls + /// run one at a time. + pub async fn refresh(&self) -> Result<()> { + let _refreshing = self.refresh_lock.lock().await; + let current = self.snapshot(); + let mut sources = current.sources.clone(); + for source in sources.values_mut() { + if let EngineSource::Lance { + uri, + storage_options, + dataset, + } = source + { + let latest = open_lance_dataset(uri, None, storage_options).await?; + if !same_lance_snapshot(&latest, dataset) { + *dataset = Arc::new(latest); + } + } + } + let state = build_engine_state(&self.config, sources, self.use_csr, Some(¤t)).await?; + *self.state.write().expect("engine state lock poisoned") = Arc::new(state); + Ok(()) + } + + /// CSR adjacency statistics per relationship type (experimental, `use_csr`). + /// + /// An identical map after `refresh()` means the snapshot itself was unchanged. + pub fn csr_stats(&self) -> HashMap { + self.snapshot() + .csr + .iter() + .map(|(rel_type, entry)| { + let stats = CsrStats { + edges: entry.outgoing.num_edges(), + delta_edges: entry.outgoing.delta_edges(), + tombstones: entry.outgoing.tombstones(), + update: entry.update, + version: entry.source.lance_version(), + build_ms: entry.build_ms, + }; + (rel_type.clone(), stats) + }) + .collect() + } + + /// Node id -> row index statistics per node label (experimental, `use_csr`). + /// + /// A label is missing when its ids are not unique integers; its targets then use + /// a join. + pub fn node_index_stats(&self) -> HashMap { + self.snapshot() + .node_index + .iter() + .map(|(label, entry)| { + let stats = NodeIndexStats { + rows: entry.rows.num_rows(), + version: entry.source.lance_version(), + build_ms: entry.build_ms, + }; + (label.clone(), stats) + }) + .collect() + } + + /// Pinned version of each Lance dataset, keyed by dataset name. + pub fn dataset_versions(&self) -> HashMap { + self.snapshot() + .sources + .iter() + .filter_map(|(name, source)| Some((name.clone(), source.lance_version()?))) + .collect() + } + + /// The graph configuration the engine was built with. + pub fn config(&self) -> &GraphConfig { + &self.config + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int64Array, RecordBatchIterator}; + use arrow_schema::{DataType, Field, Schema}; + use lance::dataset::{WriteMode, WriteParams}; + + async fn write_people(uri: &str, ids: Vec, mode: WriteMode) { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(ids))]).unwrap(); + let params = WriteParams { + mode, + ..Default::default() + }; + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + uri, + Some(params), + ) + .await + .unwrap(); + } + + /// A refresh that starts while another is in flight waits for it, then publishes + /// the newest version: an older snapshot can never land after a newer one. + #[tokio::test] + async fn refresh_waits_for_an_in_flight_refresh() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("person.lance").display().to_string(); + write_people(&uri, vec![1], WriteMode::Create).await; + let config = GraphConfig::builder() + .node_table("Person", "id") + .build() + .unwrap(); + let source = EngineSource::open_lance(uri.clone(), None, HashMap::new()) + .await + .unwrap(); + let sources = HashMap::from([("Person".to_string(), source)]); + let engine = Arc::new(CypherEngine::new(config, sources, true).await.unwrap()); + + // Stand in for refresh A, still building its replacement. + let in_flight = engine.refresh_lock.lock().await; + let b = tokio::spawn({ + let engine = engine.clone(); + async move { engine.refresh().await } + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(!b.is_finished(), "refresh B ran while A was in flight"); + + // A newer version lands before A finishes; B must publish it, not anything older. + write_people(&uri, vec![2], WriteMode::Append).await; + drop(in_flight); + b.await.unwrap().unwrap(); + assert_eq!(engine.dataset_versions()["Person"], 2); + assert_eq!(engine.node_index_stats()["Person"].rows, 2); + } +} diff --git a/crates/lance-csr/src/lib.rs b/crates/lance-csr/src/lib.rs index fbaaca3..2481444 100644 --- a/crates/lance-csr/src/lib.rs +++ b/crates/lance-csr/src/lib.rs @@ -14,23 +14,31 @@ //! - Property graph interpretation of Lance datasets //! - Translation to optimized SQL via DataFusion //! - Support for nodes, relationships, and properties +//! - A CSR (compressed sparse row) adjacency index for multi-hop traversal //! //! # Example //! +//! [`CypherEngine`] registers datasets once, optionally builds the CSR, and serves +//! many queries. See [`engine`] for the full API and `examples/embedded_query.rs` +//! for a runnable program. +//! //! ```no_run -//! use lance_csr::{CypherQuery, GraphConfig, Result}; +//! use std::collections::HashMap; +//! use lance_csr::{CypherEngine, EngineSource, GraphConfig, Result}; //! -//! # fn example() -> Result<()> { +//! # async fn example() -> Result<()> { //! let config = GraphConfig::builder() //! .node_table("Person", "person_id") //! .rel_table("KNOWS", "Person", "Person", "src_person_id", "dst_person_id") //! .build()?; -//! -//! let query = CypherQuery::new("MATCH (p:Person) WHERE p.age > 30 RETURN p.name")? -//! .with_config(config); -//! -//! // Execute against a dataset (would need actual dataset integration) -//! // let results = query.execute(&dataset).await?; +//! let sources = HashMap::from([ +//! ("Person".to_string(), EngineSource::open_lance("person.lance", None, HashMap::new()).await?), +//! ("KNOWS".to_string(), EngineSource::open_lance("knows.lance", None, HashMap::new()).await?), +//! ]); +//! let engine = CypherEngine::new(config, sources, true).await?; +//! let batch = engine +//! .execute("MATCH (p:Person) WHERE p.age > 30 RETURN p.name", HashMap::new()) +//! .await?; //! # Ok(()) //! # } //! ``` @@ -41,6 +49,7 @@ pub mod config; pub mod csr_expand; pub mod csr_index; pub mod datafusion_planner; +pub mod engine; pub mod error; pub mod key_statistics; pub mod lance_vector_search; @@ -60,6 +69,7 @@ pub const MAX_VARIABLE_LENGTH_HOPS: u32 = 20; pub use config::{GraphConfig, NodeMapping, RelationshipMapping}; pub use csr_index::{CsrIndex, CsrIndexBuilder}; +pub use engine::{CypherEngine, EngineSource}; pub use error::{GraphError, Result}; pub use lance_csr_catalog::{DirNamespace, GraphSourceCatalog, InMemoryCatalog, SimpleTableSource}; pub use lance_vector_search::VectorSearch; diff --git a/crates/lance-csr/tests/test_engine.rs b/crates/lance-csr/tests/test_engine.rs new file mode 100644 index 0000000..fe18e76 --- /dev/null +++ b/crates/lance-csr/tests/test_engine.rs @@ -0,0 +1,178 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::{Array, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance::dataset::{Dataset, WriteMode, WriteParams}; +use lance_csr::{CypherEngine, EngineSource, GraphConfig, GraphError}; + +fn config() -> GraphConfig { + GraphConfig::builder() + .node_table("Person", "id") + .rel_table("KNOWS", "Person", "Person", "src_id", "dst_id") + .build() + .unwrap() +} + +fn people() -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4])), + Arc::new(StringArray::from(vec!["Alice", "Bob", "Carol", "David"])), + ], + ) + .unwrap() +} + +fn knows(edges: &[(i64, i64)]) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("src_id", DataType::Int64, false), + Field::new("dst_id", DataType::Int64, false), + ])); + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from_iter_values(edges.iter().map(|e| e.0))), + Arc::new(Int64Array::from_iter_values(edges.iter().map(|e| e.1))), + ], + ) + .unwrap() +} + +fn names(batch: &RecordBatch) -> Vec { + let column = batch + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + (0..column.len()) + .map(|i| column.value(i).to_string()) + .collect() +} + +async fn write(batch: RecordBatch, uri: &str, mode: WriteMode) { + let schema = batch.schema(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let params = WriteParams { + mode, + ..Default::default() + }; + Dataset::write(reader, uri, Some(params)).await.unwrap(); +} + +const TWO_HOPS: &str = + "MATCH (a:Person {name: 'Alice'})-[:KNOWS]->()-[:KNOWS]->(c:Person) RETURN c.name AS name ORDER BY name"; + +#[tokio::test] +async fn arrow_input_with_csr() { + let sources = HashMap::from([ + ("Person".to_string(), EngineSource::Arrow(people())), + ( + "KNOWS".to_string(), + EngineSource::Arrow(knows(&[(1, 2), (2, 3), (2, 4)])), + ), + ]); + let engine = CypherEngine::new(config(), sources, true).await.unwrap(); + + let batch = engine.execute(TWO_HOPS, HashMap::new()).await.unwrap(); + assert_eq!(names(&batch), ["Carol", "David"]); + + let plan = engine.explain(TWO_HOPS, HashMap::new()).await.unwrap(); + assert!(plan.contains("CsrExpandExec"), "{plan}"); + + let csr = &engine.csr_stats()["KNOWS"]; + assert_eq!((csr.edges, csr.update, csr.version), (3, "build", None)); + assert_eq!(engine.node_index_stats()["Person"].rows, 4); + assert!(engine.dataset_versions().is_empty()); +} + +#[tokio::test] +async fn parameters_and_streaming() { + let sources = HashMap::from([ + ("Person".to_string(), EngineSource::Arrow(people())), + ("KNOWS".to_string(), EngineSource::Arrow(knows(&[(1, 2)]))), + ]); + let engine = CypherEngine::new(config(), sources, false).await.unwrap(); + let query = "MATCH (p:Person) WHERE p.id >= $min RETURN p.name AS name ORDER BY name"; + let params = HashMap::from([("min".to_string(), serde_json::json!(3))]); + + let batches: Vec = engine + .execute_stream(query, params) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let streamed: Vec = batches.iter().flat_map(names).collect(); + assert_eq!(streamed, ["Carol", "David"]); + assert!(engine.csr_stats().is_empty()); +} + +#[tokio::test] +async fn lance_input_is_pinned_until_refresh() { + let dir = tempfile::tempdir().unwrap(); + let person_uri = dir.path().join("person.lance"); + let knows_uri = dir.path().join("knows.lance"); + let (person_uri, knows_uri) = (person_uri.to_str().unwrap(), knows_uri.to_str().unwrap()); + write(people(), person_uri, WriteMode::Create).await; + write(knows(&[(1, 2), (2, 3)]), knows_uri, WriteMode::Create).await; + + let sources = HashMap::from([ + ( + "Person".to_string(), + EngineSource::open_lance(person_uri, None, HashMap::new()) + .await + .unwrap(), + ), + ( + "KNOWS".to_string(), + EngineSource::open_lance(knows_uri, None, HashMap::new()) + .await + .unwrap(), + ), + ]); + let engine = CypherEngine::new(config(), sources, true).await.unwrap(); + assert_eq!( + names(&engine.execute(TWO_HOPS, HashMap::new()).await.unwrap()), + ["Carol"] + ); + + // A write after construction stays invisible until refresh(). + write(knows(&[(2, 4)]), knows_uri, WriteMode::Append).await; + assert_eq!( + names(&engine.execute(TWO_HOPS, HashMap::new()).await.unwrap()), + ["Carol"] + ); + + engine.refresh().await.unwrap(); + assert_eq!( + names(&engine.execute(TWO_HOPS, HashMap::new()).await.unwrap()), + ["Carol", "David"] + ); + let csr = &engine.csr_stats()["KNOWS"]; + assert_eq!((csr.edges, csr.version), (3, Some(2))); + assert_eq!(engine.dataset_versions()["Person"], 1); +} + +#[tokio::test] +async fn construction_errors_are_config_errors() { + let empty = CypherEngine::new(config(), HashMap::new(), false).await; + assert!(matches!(empty, Err(GraphError::ConfigError { .. }))); + + let sources = HashMap::from([ + ("Person".to_string(), EngineSource::Arrow(people())), + ("KNOWS".to_string(), EngineSource::Arrow(people())), + ]); + let missing = CypherEngine::new(config(), sources, false).await; + match missing { + Err(GraphError::ConfigError { message, .. }) => assert!(message.contains("src_id")), + other => panic!("expected a ConfigError, got {:?}", other.err()), + } +} diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 70a0376..584da48 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -7,7 +7,7 @@ Read them before you change a page. A reader should be able to find out what lance-csr is, install it, run a query, and learn every user-facing option without reading the source. Write for a Python developer who knows a little Cypher and nothing about -the internals. The CSR index is the project's distinguishing feature, so it gets the most care. +the internals; `rust.md` covers the same engine for Rust developers. The CSR index is the project's distinguishing feature, so it gets the most care. ## How the site is built @@ -34,7 +34,8 @@ the internals. The CSR index is the project's distinguishing feature, so it gets ## Accuracy comes first - **Document what exists on `main`.** Read the code, tests and docstrings before writing. The Python API is - defined in `crates/lance-csr-python/src/graph.rs`; the Cypher surface in `crates/lance-csr/src/` + defined in `crates/lance-csr-python/src/graph.rs`, on top of the Rust `CypherEngine` in + `crates/lance-csr/src/engine.rs`; the Cypher surface in `crates/lance-csr/src/` (`parser.rs`, `semantic.rs`, `logical_plan.rs`). Never describe planned features as shipped. - **Every Python example runs.** `check_snippets.py` executes each page's `python` blocks in order, in one namespace, in a temporary directory. Run it against a local build before you commit: @@ -46,6 +47,11 @@ the internals. The CSR index is the project's distinguishing feature, so it gets Put the output in a trailing comment (`print(x) # [{'name': 'Bob'}]`) and check it matches what the run printed. Use `ORDER BY` so results are deterministic, and don't show output that varies (timings, dict or label order) without saying so. +- **Every Rust example compiles and runs.** Rust code on a page is never typed into the Markdown: it lives in + `crates/lance-csr/examples/embedded_query.rs` between `// --8<-- [start:name]` and `// --8<-- [end:name]` + markers, and the page includes it with `--8<-- "crates/lance-csr/examples/embedded_query.rs:name"` inside a + `rust` fence. CI builds and runs the example, and the strict build fails if a file or section is missing. + Keep the output comments in the example in step with what it prints. - A snippet that can't run on its own (it needs LanceDB, a remote store or an ANN index) gets a `` line directly above its fence. Use this sparingly. - **Every user-facing knob is documented**: constructor arguments, keyword arguments, builder methods, enum diff --git a/docs/api-reference.md b/docs/api-reference.md index 81ffd03..b528333 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -16,6 +16,9 @@ code. The guides link here for exact signatures and link back for examples. `lance_csr.__version__` is the package version. +This page covers the Python API. For Rust, see [Using from Rust](rust.md) and the +[API documentation on docs.rs](https://docs.rs/lance-csr). + ## Datasets Methods that take `datasets` take a `dict[str, object]` mapping each node label or relationship type (or SQL @@ -112,7 +115,8 @@ be opened. `#!python refresh() -> None` : Moves every Lance dataset to its latest version and updates the CSR, node indexes and statistics. - All-or-nothing; running queries keep their snapshot. See [Lance datasets](lance.md#refresh). + All-or-nothing; running queries keep their snapshot, and concurrent calls run one at a time. See + [Lance datasets](lance.md#refresh). `#!python dataset_versions() -> dict[str, int]` : The pinned version of each Lance dataset. PyArrow inputs are not listed. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index dc00bc3..c29f57d 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -11,7 +11,8 @@ The benchmark runners and raw results live in two sibling repositories, next to every number on this page can be reproduced from there. lance-csr runs with `use_csr=True` in two modes: *in memory* holds every table as Arrow in memory; *lazy* reads the Lance datasets from disk as queries need them (see [Lance datasets](lance.md)). Neo4j runs as a server in a Docker VM; the other engines run embedded in the Python -process. +process. All numbers were measured through the Python package; the Rust crate runs the same engine and has no +separate benchmark results. The tables are generated from the repositories' committed results by `docs/sync_benchmarks.py`, which pins each repository to a commit. diff --git a/docs/getting-started.md b/docs/getting-started.md index a9efa78..c94720d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,6 +31,10 @@ build from the source distribution, which needs Rust and `protoc` (see [Contribu The package depends on `pyarrow>=14` and `pylance` 13.x, which uv installs for you. +!!! tip "Using Rust?" + + The same engine is on crates.io: run `cargo add lance-csr` and see [Using from Rust](rust.md). + !!! note "Docs track the `main` branch" These pages describe the code on `main`. The `node_table` and `rel_table` builder methods arrived in @@ -160,5 +164,6 @@ than one query against the same data. - [Graph schema](graph-schema.md): every option of `GraphConfig`. - [Querying with Cypher](cypher.md): parameters, streaming, `explain()` and the supported Cypher. - [Lance datasets](lance.md): query data on disk without loading it, and pick up new writes with `refresh()`. +- [Using from Rust](rust.md): the same engine as a Rust library. - The [`examples/`](https://github.com/hyperdimensionalcomputingai/lance-csr/tree/main/examples) directory has runnable scripts. diff --git a/docs/index.md b/docs/index.md index 766206a..c3e620c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,8 +11,8 @@ title: Home **lance-csr** is an open source [openCypher](https://opencypher.org/) query engine for graphs stored as [Lance](https://lancedb.github.io/lance/) datasets or Arrow tables. It is written in Rust on -[Apache DataFusion](https://datafusion.apache.org/), ships as a Python package, and runs on one machine with no -server to operate. Its distinguishing feature is a **compressed sparse row (CSR) adjacency index** that turns +[Apache DataFusion](https://datafusion.apache.org/), ships as a Python package and a Rust crate, and runs on one machine with +no server to operate. Its distinguishing feature is a **compressed sparse row (CSR) adjacency index** that turns each relationship hop from a hash join into an array lookup, which is what makes multi-hop traversals fast. ```python @@ -148,3 +148,4 @@ without it, but the API around it may change. Prebuilt wheels cover Linux x86-64 [:fontawesome-brands-github: GitHub](https://github.com/hyperdimensionalcomputingai/lance-csr){ .md-button } [:fontawesome-brands-python: PyPI](https://pypi.org/project/lance-csr/){ .md-button } +[:fontawesome-brands-rust: crates.io](https://crates.io/crates/lance-csr){ .md-button } diff --git a/docs/rust.md b/docs/rust.md new file mode 100644 index 0000000..eeede83 --- /dev/null +++ b/docs/rust.md @@ -0,0 +1,130 @@ +# Using lance-csr from Rust + +lance-csr is a Rust library first. The Python package is a thin wrapper around the same engine, so everything +on the other pages (the [graph schema](graph-schema.md), the [supported Cypher](cypher.md#supported-cypher), +the [CSR index](csr.md) and [version pinning](lance.md)) applies to Rust too. This page covers what's +different: installing the crate, the async API, and the Rust types. + +Every Rust snippet on this page comes from +[`examples/embedded_query.rs`](https://github.com/hyperdimensionalcomputingai/lance-csr/blob/main/crates/lance-csr/examples/embedded_query.rs), +which CI compiles and runs. Run it yourself from a checkout: + +```bash +cargo run -p lance-csr --example embedded_query +``` + +## Install + +The engine is published on crates.io as `lance-csr`: + +```bash +cargo add lance-csr +``` + +Building it needs `protoc`, the Protocol Buffers compiler, for Lance's dependencies (see +[Contributing](contributing.md)). The engine is async, so your program also needs a runtime such as +[Tokio](https://tokio.rs/). + +!!! warning "Use the same Lance, DataFusion and Arrow versions" + + lance-csr 0.3 builds on `lance` 13, `datafusion` 54 and `arrow` 58. If your program also uses those crates, + for example to open datasets or read results, depend on the same major versions. Otherwise their types + (`Dataset`, `RecordBatch`) won't match lance-csr's. + +## Build an engine + +`CypherEngine` is the same engine as in Python. It registers your datasets once, optionally builds the CSR, +and serves many queries: + +```rust +--8<-- "crates/lance-csr/examples/embedded_query.rs:engine" +``` + +Each input is an `EngineSource`: + +`EngineSource::open_lance(uri, version, storage_options)` +: Opens a Lance dataset, pinned to `version` (`None` means the latest). The dataset is read lazily as + queries need it, and `storage_options` are kept so `refresh()` can reopen it with the same credentials. + See [Lance datasets](lance.md). + +`EngineSource::Arrow(batch)` +: An in-memory Arrow `RecordBatch`. It never changes; build a new engine to replace it. + +The third argument of `CypherEngine::new` is `use_csr`. With `true`, relationship types with integer endpoints +get [CSR adjacencies](csr.md) and node labels with unique integer ids get a node index, exactly as with +`use_csr=True` in Python. + +## Run queries + +`execute` returns the whole result as one `RecordBatch`. Parameters are a `HashMap` of `serde_json::Value`s, +one per `$name` in the query; pass `HashMap::new()` when there are none: + +```rust +--8<-- "crates/lance-csr/examples/embedded_query.rs:query" +``` + +The other query methods take the same arguments: + +| Method | Returns | +| --- | --- | +| `execute(query, params)` | `RecordBatch`, the whole result | +| `execute_stream(query, params)` | DataFusion `SendableRecordBatchStream`, computed as you poll it; see [streaming](lance.md#streaming-large-results) | +| `explain(query, params)` | `String`, the logical and physical plans | +| `execute_with_vector_rerank(query, &vector_search)` | `RecordBatch`, reranked by a [`VectorSearch`](vector-search.md) | + +Each query runs on the snapshot that was current when it started. + +## Pick up new writes + +`refresh()` moves every Lance dataset to its latest version and updates the CSR, applying the new edges as a +delta when it can (see [Keeping the CSR current](csr.md#keeping-the-csr-current)): + +```rust +--8<-- "crates/lance-csr/examples/embedded_query.rs:refresh" +``` + +`csr_stats()` and `node_index_stats()` return the same fields as in Python, as `CsrStats` and `NodeIndexStats` +structs keyed by relationship type and node label. `dataset_versions()` gives the pinned version of each Lance +dataset. + +## Errors + +Every method returns `lance_csr::Result`, whose error is `GraphError`. The Python package raises a +different exception depending on the variant: + +| `GraphError` variant | Meaning | Python exception | +| --- | --- | --- | +| `ParseError` | The Cypher text doesn't parse | `ValueError` | +| `ConfigError` | Bad graph configuration, no datasets, or a configured column is missing | `ValueError` | +| `PlanError`, `InvalidPattern` | The query can't be planned against the configuration | `ValueError` | +| `UnsupportedFeature` | Valid Cypher that lance-csr doesn't support | `NotImplementedError` | +| `ExecutionError`, `DataFusion`, `LanceCore`, `Arrow` | Opening, scanning or executing failed | `RuntimeError` | + +## Async runtimes and cancellation + +- Building the CSR in `new()` and `refresh()` is CPU-heavy work inside an async function, so it holds a + runtime worker thread until it finishes. In an application with its own runtime (a desktop or web backend), + run those two calls with `tokio::task::spawn_blocking` so other tasks keep running. +- Dropping a query's future, or the stream from `execute_stream`, cancels the rest of that query. There are no + built-in timeouts; wrap a call in `tokio::time::timeout` if you need one. + +## Python and Rust side by side + +| Python | Rust | +| --- | --- | +| `CypherEngine(config, datasets, use_csr=True)` | `CypherEngine::new(config, sources, true).await` | +| `lance.dataset(uri)` in `datasets` | `EngineSource::open_lance(uri, None, options).await` | +| a PyArrow table in `datasets` | `EngineSource::Arrow(batch)` | +| `engine.execute(query, params)` → `pyarrow.Table` | `engine.execute(query, params).await` → `RecordBatch` | +| `engine.execute_batches(...)` → `RecordBatchReader` | `engine.execute_stream(...).await` → stream | +| `engine.refresh()` | `engine.refresh().await` | +| `engine.csr_stats()` → `dict` | `engine.csr_stats()` → `HashMap` | +| `ValueError`, `RuntimeError`, ... | `GraphError` variants | + +`CypherQuery` (one query over in-memory batches), `SqlQuery` and `DirNamespace` also exist in Rust. The +[API documentation on docs.rs](https://docs.rs/lance-csr) lists every type and method. + +## Performance + +The engine code is the same whichever language calls it. The [benchmarks](benchmarks.md) were all measured +through the Python package; there are no separate Rust benchmark results. diff --git a/python/pyproject.toml b/python/pyproject.toml index 25ba503..6715e47 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lance-csr" -version = "0.2.0" +version = "0.3.0" dependencies = [ "pyarrow>=14", "pylance>=13.0.0,<14", diff --git a/python/uv.lock b/python/uv.lock index 26fda10..466cf6e 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -40,7 +40,7 @@ wheels = [ [[package]] name = "lance-csr" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "pyarrow" }, diff --git a/zensical.toml b/zensical.toml index 0ddf335..5cc7961 100644 --- a/zensical.toml +++ b/zensical.toml @@ -33,6 +33,7 @@ nav = [ { "Vector search" = "vector-search.md" }, { "SQL queries" = "sql.md" }, ] }, + { "Using from Rust" = "rust.md" }, { "How it works" = "execution.md" }, { "API reference" = "api-reference.md" }, { "Benchmarks" = "benchmarks.md" }, @@ -114,6 +115,10 @@ pymdownx.inlinehilite = {} pymdownx.keys = {} pymdownx.mark = {} pymdownx.smartsymbols = {} +# Rust examples are included from compiled files; a missing file or section fails the build. +pymdownx.snippets.base_path = ["."] +pymdownx.snippets.check_paths = true +pymdownx.snippets.dedent_subsections = true pymdownx.superfences.custom_fences = [ { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }, ]