diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 1e17e64f42..301108380c 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -311,8 +311,8 @@ Next, write the data to the table. Both `append` and `overwrite` produce the sam -!!! note inline end "Fast append" - PyIceberg defaults to the [fast append](https://iceberg.apache.org/spec/#snapshots) to minimize the amount of data written. This enables fast commit operations, reducing the possibility of conflicts. The downside of the fast append is that it creates more metadata than a merge commit. [Compaction is planned](https://github.com/apache/iceberg-python/issues/270) and will automatically rewrite all the metadata when a threshold is hit, to maintain performant reads. +!!! note inline end "Merge and fast append" + PyIceberg defaults to [merge append](https://iceberg.apache.org/spec/#snapshots), which automatically merges manifests when the configured threshold is reached to keep metadata compact and reads performant. Fast append remains available by setting the table property `commit.manifest-merge.enabled` to `False`. Fast append minimizes the metadata written during a commit, enabling faster commits and reducing the possibility of conflicts, but accumulates more manifest metadata over time. diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 54d33dd00e..4ad7beb89c 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -99,12 +99,12 @@ Iceberg tables support table properties to configure table behavior. | ------------------------------------ | ------------------- | ------------- | ----------------------------------------------------------- | | `commit.manifest.target-size-bytes` | Size in bytes | 8388608 (8MB) | Target size when merging manifest files | | `commit.manifest.min-count-to-merge` | Number of manifests | 100 | Minimum number of manifests to accumulate before merging | -| `commit.manifest-merge.enabled` | Boolean | False | Controls whether to automatically merge manifests on writes | +| `commit.manifest-merge.enabled` | Boolean | True | Controls whether to automatically merge manifests on writes | -!!! note "Fast append" - Unlike Java implementation, PyIceberg default to the [fast append](api.md#write-to-a-table) and thus `commit.manifest-merge.enabled` is set to `False` by default. +!!! note "Append modes" + PyIceberg defaults to [merge append](api.md#write-to-a-table) to keep manifest metadata compact. Set `commit.manifest-merge.enabled` to `False` on a table to use fast append, which writes less metadata during each commit but accumulates more manifests over time. diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9624eac981..f7d34ba12f 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -205,7 +205,7 @@ class TableProperties: MANIFEST_MIN_MERGE_COUNT_DEFAULT = 100 MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled" - MANIFEST_MERGE_ENABLED_DEFAULT = False + MANIFEST_MERGE_ENABLED_DEFAULT = True METADATA_PREVIOUS_VERSIONS_MAX = "write.metadata.previous-versions-max" METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT = 100 diff --git a/tests/integration/test_writes/test_partitioned_writes.py b/tests/integration/test_writes/test_partitioned_writes.py index 1d1488255f..040c0cfeea 100644 --- a/tests/integration/test_writes/test_partitioned_writes.py +++ b/tests/integration/test_writes/test_partitioned_writes.py @@ -748,8 +748,11 @@ def test_dynamic_partition_overwrite_evolve_partition(spark: SparkSession, sessi tbl.dynamic_partition_overwrite(arrow_table) result = tbl.scan().to_arrow() - assert result["place"].to_pylist() == ["Groningen", "Amsterdam", "Drachten"] - assert result["inhabitants"].to_pylist() == [238147, 921402, 44940] + assert sorted(zip(result["place"].to_pylist(), result["inhabitants"].to_pylist(), strict=True)) == [ + ("Amsterdam", 921402), + ("Drachten", 44940), + ("Groningen", 238147), + ] @pytest.mark.integration diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 30fdd76ab7..553a9ac574 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1571,7 +1571,7 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_a = _create_table( session_catalog, "default.merge_manifest_a", - {"commit.manifest-merge.enabled": "true", "commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, [], ) tbl_b = _create_table( @@ -1588,7 +1588,11 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_c = _create_table( session_catalog, "default.merge_manifest_c", - {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + { + "commit.manifest-merge.enabled": "false", + "commit.manifest.min-count-to-merge": "1", + "format-version": format_version, + }, [], ) diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 0f72b08087..91cdabcb9c 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -659,6 +659,25 @@ def summary_calls(n_files: int) -> int: ) +def test_append_snapshot_producer_defaults_to_merge_append(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _MergeAppendFiles + + append = table_v2.transaction()._append_snapshot_producer({}) + + assert type(append) is _MergeAppendFiles + + +def test_append_snapshot_producer_uses_fast_append_when_manifest_merge_disabled(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _FastAppendFiles + + transaction = table_v2.transaction() + transaction.set_properties({"commit.manifest-merge.enabled": "false"}) + + append = transaction._append_snapshot_producer({}) + + assert type(append) is _FastAppendFiles + + @pytest.fixture def overwrite_table(catalog: Catalog, arrow_table_simple: pa.Table) -> Table: catalog.create_namespace("default") @@ -737,3 +756,4 @@ def test_overwrite_rejects_explicit_delete_without_parent_snapshot( with empty.transaction() as tx: with tx.update_snapshot().overwrite() as overwrite: overwrite.delete_data_file(stale_file) +