diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java index aed6645b..c8f49e25 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java @@ -149,10 +149,20 @@ public void setUp() throws Exception { throw e; } - // Copy CLI config into container so ice CLI can talk to co-located REST server + // Copy CLI config into container so ice CLI can talk to co-located REST server. + // The s3 section lets CLI commands that read s3:// paths directly (e.g. describe-metadata on + // metadata.json) reach MinIO via its network alias instead of defaulting to AWS. File cliConfigHost = File.createTempFile("ice-docker-cli-", ".yaml"); try { - Files.write(cliConfigHost.toPath(), "uri: http://localhost:5000\n".getBytes()); + String cliConfig = + "uri: http://localhost:5000\n" + + "s3:\n" + + " endpoint: http://minio:9000\n" + + " pathStyleAccess: true\n" + + " accessKeyID: minioadmin\n" + + " secretAccessKey: minioadmin\n" + + " region: us-east-1\n"; + Files.writeString(cliConfigHost.toPath(), cliConfig); catalog.copyFileToContainer( MountableFile.forHostPath(cliConfigHost.toPath()), "/tmp/ice-cli.yaml"); } finally { diff --git a/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/input.parquet b/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/input.parquet new file mode 100644 index 00000000..028c64cf Binary files /dev/null and b/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/input.parquet differ diff --git a/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/run.sh.tmpl b/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/run.sh.tmpl new file mode 100644 index 00000000..0ad2f95a --- /dev/null +++ b/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/run.sh.tmpl @@ -0,0 +1,71 @@ +#!/bin/bash +set -e + +echo "Running Iceberg v3 table creation test..." + +# Create namespace +{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME} +echo "OK Created namespace: ${NAMESPACE_NAME}" + +# Get the full path to the input file (relative to scenario directory) +SCENARIO_DIR="{{SCENARIO_DIR}}" +INPUT_PATH="${SCENARIO_DIR}/${INPUT_FILE}" + +# Create a format-version 3 table and insert data +{{ICE_CLI}} --config {{CLI_CONFIG}} insert --create-table --format-version=3 ${TABLE_NAME} ${INPUT_PATH} +echo "OK Created table ${TABLE_NAME} with --format-version=3 and inserted ${INPUT_FILE}" + +# Verify describe output shows formatVersion 3 (Iceberg strips format-version from properties) +{{ICE_CLI}} --config {{CLI_CONFIG}} describe ${TABLE_NAME} > /tmp/v3_create_describe.txt +if ! grep -q "formatVersion: 3" /tmp/v3_create_describe.txt; then + echo "FAIL describe output missing 'formatVersion: 3'" + cat /tmp/v3_create_describe.txt + exit 1 +fi +echo "OK describe shows formatVersion: 3" + +# Locate the current metadata.json via the REST catalog (works in both local and Docker runners) +TABLE_SHORT="${TABLE_NAME##*.}" +METADATA_FILE=$(curl -sf "{{CATALOG_URI}}/v1/namespaces/${NAMESPACE_NAME}/tables/${TABLE_SHORT}" \ + | grep -o '"metadata-location": *"[^"]*"' | sed 's/.*: *"//; s/"$//') +if [ -z "${METADATA_FILE}" ]; then + echo "FAIL Could not determine metadata location for ${TABLE_NAME} via {{CATALOG_URI}}" + exit 1 +fi +echo "OK Current metadata file: ${METADATA_FILE}" + +# Verify metadata summary shows formatVersion 3 +{{ICE_CLI}} --config {{CLI_CONFIG}} describe-metadata -s "${METADATA_FILE}" > /tmp/v3_create_metadata_summary.txt +if ! grep -q "formatVersion: 3" /tmp/v3_create_metadata_summary.txt; then + echo "FAIL describe-metadata -s output missing 'formatVersion: 3'" + cat /tmp/v3_create_metadata_summary.txt + exit 1 +fi +echo "OK describe-metadata -s shows formatVersion: 3" + +# Verify v3 row lineage: snapshot carries first-row-id +{{ICE_CLI}} --config {{CLI_CONFIG}} describe-metadata --snapshots "${METADATA_FILE}" > /tmp/v3_create_metadata_snapshots.txt +if ! grep -q "first-row-id:" /tmp/v3_create_metadata_snapshots.txt; then + echo "FAIL describe-metadata --snapshots output missing 'first-row-id' (v3 row lineage)" + cat /tmp/v3_create_metadata_snapshots.txt + exit 1 +fi +echo "OK describe-metadata --snapshots shows first-row-id (row lineage)" + +# Verify v3 row lineage: manifest carries first_row_id +{{ICE_CLI}} --config {{CLI_CONFIG}} describe-metadata --manifests "${METADATA_FILE}" > /tmp/v3_create_metadata_manifests.txt +if ! grep -q "first_row_id:" /tmp/v3_create_metadata_manifests.txt; then + echo "FAIL describe-metadata --manifests output missing 'first_row_id' (v3 row lineage)" + cat /tmp/v3_create_metadata_manifests.txt + exit 1 +fi +echo "OK describe-metadata --manifests shows first_row_id (row lineage)" + +# Cleanup +{{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME} +echo "OK Deleted table: ${TABLE_NAME}" + +{{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME} +echo "OK Deleted namespace: ${NAMESPACE_NAME}" + +echo "Iceberg v3 table creation test completed successfully" diff --git a/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/scenario.yaml b/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/scenario.yaml new file mode 100644 index 00000000..a85e17e6 --- /dev/null +++ b/ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/scenario.yaml @@ -0,0 +1,10 @@ +name: "Iceberg v3 Table Creation" +description: "Tests creating a format-version 3 table via insert --create-table --format-version=3 and verifies v3 metadata (row lineage)" + +catalogConfig: + warehouse: "s3://test-bucket/warehouse" + +env: + NAMESPACE_NAME: "test_v3" + TABLE_NAME: "test_v3.t1" + INPUT_FILE: "input.parquet" diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index a1494413..617af116 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -360,7 +360,12 @@ void createTable( names = {"--sort"}, description = "Sort order, e.g. [{\"column\":\"name\", \"desc\":false, \"nullFirst\":false}]") - String sortOrderJson) + String sortOrderJson, + @CommandLine.Option( + names = {"--format-version"}, + description = "Iceberg table format version (2 or 3). Default: 2", + defaultValue = "2") + int formatVersion) throws IOException { setAWSRegion(s3Region); try (RESTCatalog catalog = loadCatalog()) { @@ -387,6 +392,7 @@ void createTable( createTableIfNotExists, useVendedCredentials, s3NoSignRequest, + formatVersion, partitions, sortOrders); } @@ -517,6 +523,12 @@ void insert( description = "Sort order, e.g. [{\"column\":\"name\", \"desc\":false, \"nullFirst\":false}]") String sortOrderJson, + @CommandLine.Option( + names = {"--format-version"}, + description = + "Iceberg table format version (2 or 3) when creating the table with -p/--create-table. Default: 2", + defaultValue = "2") + int formatVersion, @CommandLine.Option( names = {"--assume-sorted"}, description = "Skip data sorting. Assume it's already sorted.") @@ -639,6 +651,7 @@ void insert( createTableIfNotExists, useVendedCredentials, s3NoSignRequest, + formatVersion, partitions, sortOrders); } // delayed in watch mode @@ -660,6 +673,7 @@ void insert( .retryListFile(retryList) .partitionList(partitions) .sortOrderList(sortOrders) + .formatVersion(formatVersion) .threadCount( threadCount < 1 ? Runtime.getRuntime().availableProcessors() : threadCount) .commitRetries(commitRetries) diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/CreateTable.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/CreateTable.java index 422ced35..c481ab54 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/CreateTable.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/CreateTable.java @@ -64,9 +64,14 @@ public static void run( boolean ignoreAlreadyExists, boolean useVendedCredentials, boolean s3NoSignRequest, + int formatVersion, @Nullable List partitionList, @Nullable List sortOrderList) throws IOException { + if (formatVersion != 2 && formatVersion != 3) { + throw new IllegalArgumentException( + "--format-version must be 2 or 3 (got " + formatVersion + ")"); + } if (ignoreAlreadyExists && catalog.tableExists(nsTable)) { return; } @@ -105,7 +110,12 @@ public static void run( // force name-based resolution instead of position-based resolution NameMapping mapping = MappingUtil.create(initialSchema); String mappingJson = NameMappingParser.toJson(mapping); - var props = Map.of(TableProperties.DEFAULT_NAME_MAPPING, mappingJson); + var props = + Map.of( + TableProperties.DEFAULT_NAME_MAPPING, + mappingJson, + TableProperties.FORMAT_VERSION, + String.valueOf(formatVersion)); PartitionSpec partitionSpec = partitionList == null diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Describe.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Describe.java index 9e14b0f7..4ebeadb7 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Describe.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Describe.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Set; import javax.annotation.Nullable; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Snapshot; @@ -142,10 +143,13 @@ private static Table.Data gatherTableData( } boolean includeSchema = optionsSet.contains(Option.INCLUDE_SCHEMA); + Integer formatVersion = + table instanceof BaseTable bt ? bt.operations().current().formatVersion() : null; return new Table.Data( includeSchema ? table.schema().toString() : null, includeSchema ? table.spec().toString() : null, includeSchema ? table.sortOrder().toString() : null, + formatVersion, optionsSet.contains(Option.INCLUDE_PROPERTIES) ? table.properties() : null, table.location(), snapshotInfo, @@ -253,6 +257,7 @@ record Data( String schemaRaw, String partitionSpecRaw, String sortOrderRaw, + Integer formatVersion, Map properties, String location, Table.Snapshot currentSnapshot, diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DescribeMetadata.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DescribeMetadata.java index 9bf07575..f8e07d5e 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DescribeMetadata.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/DescribeMetadata.java @@ -11,6 +11,7 @@ import com.altinity.ice.internal.iceberg.io.SchemeFileIO; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.IOException; @@ -141,7 +142,8 @@ public static List extractSnapshots( snapshot.operation(), currentSnapshotId != null && snapshot.snapshotId() == currentSnapshotId, snapshot.summary(), - snapshot.manifestListLocation())); + snapshot.manifestListLocation(), + snapshot.firstRowId())); } return result; } @@ -213,6 +215,7 @@ private static List extractManifests( manifest.existingFilesCount(), manifest.deletedFilesCount(), manifest.partitionSpecId(), + manifest.firstRowId(), dataFiles.isEmpty() ? null : dataFiles)); } @@ -272,7 +275,8 @@ public record SnapshotInfo( String operation, boolean current, Map summary, - String manifestListLocation) {} + String manifestListLocation, + @JsonProperty("first-row-id") Long firstRowId) {} @JsonInclude(JsonInclude.Include.NON_NULL) public record HistoryInfo( @@ -291,6 +295,7 @@ public record ManifestInfo( Integer existingFilesCount, Integer deletedFilesCount, int partitionSpecId, + @JsonProperty("first_row_id") Long firstRowId, List dataFiles) {} @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Insert.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Insert.java index 0eb6be29..9ad2d613 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Insert.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Insert.java @@ -1041,6 +1041,7 @@ public record Options( @Nullable String retryListFile, @Nullable List partitionList, @Nullable List sortOrderList, + int formatVersion, int threadCount, @Nullable String compression, int commitRetries, @@ -1066,6 +1067,7 @@ public static final class Builder { private String retryListFile; private List partitionList = List.of(); private List sortOrderList = List.of(); + private int formatVersion = 2; private int threadCount = Runtime.getRuntime().availableProcessors(); private String compression; private int commitRetries = 10; @@ -1148,6 +1150,11 @@ public Builder sortOrderList(List sortOrderList) { return this; } + public Builder formatVersion(int formatVersion) { + this.formatVersion = formatVersion; + return this; + } + public Builder threadCount(int threadCount) { this.threadCount = threadCount; return this; @@ -1185,6 +1192,7 @@ public Options build() { retryListFile, partitionList, sortOrderList, + formatVersion, threadCount, compression, commitRetries, diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/InsertWatch.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/InsertWatch.java index 3aa9afe1..cbebe223 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/InsertWatch.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/InsertWatch.java @@ -363,6 +363,7 @@ private static void insert( true, options.useVendedCredentials(), options.s3NoSignRequest(), + options.formatVersion(), null, null); } catch (NotFoundException nfe) { diff --git a/ice/src/main/resources/META-INF/native-image/com.altinity/ice/reflect-config.json b/ice/src/main/resources/META-INF/native-image/com.altinity/ice/reflect-config.json index fbc492d6..703918b2 100644 --- a/ice/src/main/resources/META-INF/native-image/com.altinity/ice/reflect-config.json +++ b/ice/src/main/resources/META-INF/native-image/com.altinity/ice/reflect-config.json @@ -244,7 +244,7 @@ "allDeclaredFields":true, "queryAllDeclaredMethods":true, "queryAllDeclaredConstructors":true, - "methods":[{"name":"currentSnapshot","parameterTypes":[] }, {"name":"location","parameterTypes":[] }, {"name":"metrics","parameterTypes":[] }, {"name":"partitionSpecRaw","parameterTypes":[] }, {"name":"properties","parameterTypes":[] }, {"name":"schemaRaw","parameterTypes":[] }, {"name":"sortOrderRaw","parameterTypes":[] }] + "methods":[{"name":"currentSnapshot","parameterTypes":[] }, {"name":"formatVersion","parameterTypes":[] }, {"name":"location","parameterTypes":[] }, {"name":"metrics","parameterTypes":[] }, {"name":"partitionSpecRaw","parameterTypes":[] }, {"name":"properties","parameterTypes":[] }, {"name":"schemaRaw","parameterTypes":[] }, {"name":"sortOrderRaw","parameterTypes":[] }] }, { "name":"com.altinity.ice.cli.internal.cmd.Describe$Table$Error", @@ -300,7 +300,7 @@ "allDeclaredFields":true, "queryAllDeclaredMethods":true, "queryAllDeclaredConstructors":true, - "methods":[{"name":"addedFilesCount","parameterTypes":[] }, {"name":"dataFiles","parameterTypes":[] }, {"name":"deletedFilesCount","parameterTypes":[] }, {"name":"existingFilesCount","parameterTypes":[] }, {"name":"partitionSpecId","parameterTypes":[] }, {"name":"path","parameterTypes":[] }] + "methods":[{"name":"addedFilesCount","parameterTypes":[] }, {"name":"dataFiles","parameterTypes":[] }, {"name":"deletedFilesCount","parameterTypes":[] }, {"name":"existingFilesCount","parameterTypes":[] }, {"name":"firstRowId","parameterTypes":[] }, {"name":"partitionSpecId","parameterTypes":[] }, {"name":"path","parameterTypes":[] }] }, { "name":"com.altinity.ice.cli.internal.cmd.DescribeMetadata$MetadataInfo", @@ -328,7 +328,7 @@ "allDeclaredFields":true, "queryAllDeclaredMethods":true, "queryAllDeclaredConstructors":true, - "methods":[{"name":"current","parameterTypes":[] }, {"name":"manifestListLocation","parameterTypes":[] }, {"name":"operation","parameterTypes":[] }, {"name":"parentId","parameterTypes":[] }, {"name":"sequenceNumber","parameterTypes":[] }, {"name":"snapshotId","parameterTypes":[] }, {"name":"summary","parameterTypes":[] }, {"name":"timestamp","parameterTypes":[] }, {"name":"timestampMillis","parameterTypes":[] }] + "methods":[{"name":"current","parameterTypes":[] }, {"name":"firstRowId","parameterTypes":[] }, {"name":"manifestListLocation","parameterTypes":[] }, {"name":"operation","parameterTypes":[] }, {"name":"parentId","parameterTypes":[] }, {"name":"sequenceNumber","parameterTypes":[] }, {"name":"snapshotId","parameterTypes":[] }, {"name":"summary","parameterTypes":[] }, {"name":"timestamp","parameterTypes":[] }, {"name":"timestampMillis","parameterTypes":[] }] }, { "name":"com.altinity.ice.cli.internal.cmd.DescribeMetadata$SnapshotLogEntry",