Added support to create iceberg v3 table using create-table.

ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java

@@ -149,10 +149,20 @@ public void setUp() throws Exception {
       throw e;
     }
 
-    // Copy CLI config into container so ice CLI can talk to co-located REST server
+    // Copy CLI config into container so ice CLI can talk to co-located REST server.
+    // The s3 section lets CLI commands that read s3:// paths directly (e.g. describe-metadata on
+    // metadata.json) reach MinIO via its network alias instead of defaulting to AWS.
     File cliConfigHost = File.createTempFile("ice-docker-cli-", ".yaml");
     try {
-      Files.write(cliConfigHost.toPath(), "uri: http://localhost:5000\n".getBytes());
+      String cliConfig =
+          "uri: http://localhost:5000\n"
+              + "s3:\n"
+              + "  endpoint: http://minio:9000\n"
+              + "  pathStyleAccess: true\n"
+              + "  accessKeyID: minioadmin\n"
+              + "  secretAccessKey: minioadmin\n"
+              + "  region: us-east-1\n";
+      Files.writeString(cliConfigHost.toPath(), cliConfig);
       catalog.copyFileToContainer(
           MountableFile.forHostPath(cliConfigHost.toPath()), "/tmp/ice-cli.yaml");
     } finally {

ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/run.sh.tmpl

@@ -0,0 +1,71 @@
+#!/bin/bash
+set -e
+
+echo "Running Iceberg v3 table creation test..."
+
+# Create namespace
+{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME}
+echo "OK Created namespace: ${NAMESPACE_NAME}"
+
+# Get the full path to the input file (relative to scenario directory)
+SCENARIO_DIR="{{SCENARIO_DIR}}"
+INPUT_PATH="${SCENARIO_DIR}/${INPUT_FILE}"
+
+# Create a format-version 3 table and insert data
+{{ICE_CLI}} --config {{CLI_CONFIG}} insert --create-table --format-version=3 ${TABLE_NAME} ${INPUT_PATH}
+echo "OK Created table ${TABLE_NAME} with --format-version=3 and inserted ${INPUT_FILE}"
+
+# Verify describe output shows formatVersion 3 (Iceberg strips format-version from properties)
+{{ICE_CLI}} --config {{CLI_CONFIG}} describe ${TABLE_NAME} > /tmp/v3_create_describe.txt
+if ! grep -q "formatVersion: 3" /tmp/v3_create_describe.txt; then
+  echo "FAIL describe output missing 'formatVersion: 3'"
+  cat /tmp/v3_create_describe.txt
+  exit 1
+fi
+echo "OK describe shows formatVersion: 3"
+
+# Locate the current metadata.json via the REST catalog (works in both local and Docker runners)
+TABLE_SHORT="${TABLE_NAME##*.}"
+METADATA_FILE=$(curl -sf "{{CATALOG_URI}}/v1/namespaces/${NAMESPACE_NAME}/tables/${TABLE_SHORT}" \
+  | grep -o '"metadata-location": *"[^"]*"' | sed 's/.*: *"//; s/"$//')
+if [ -z "${METADATA_FILE}" ]; then
+  echo "FAIL Could not determine metadata location for ${TABLE_NAME} via {{CATALOG_URI}}"
+  exit 1
+fi
+echo "OK Current metadata file: ${METADATA_FILE}"
+
+# Verify metadata summary shows formatVersion 3
+{{ICE_CLI}} --config {{CLI_CONFIG}} describe-metadata -s "${METADATA_FILE}" > /tmp/v3_create_metadata_summary.txt
+if ! grep -q "formatVersion: 3" /tmp/v3_create_metadata_summary.txt; then
+  echo "FAIL describe-metadata -s output missing 'formatVersion: 3'"
+  cat /tmp/v3_create_metadata_summary.txt
+  exit 1
+fi
+echo "OK describe-metadata -s shows formatVersion: 3"
+
+# Verify v3 row lineage: snapshot carries first-row-id
+{{ICE_CLI}} --config {{CLI_CONFIG}} describe-metadata --snapshots "${METADATA_FILE}" > /tmp/v3_create_metadata_snapshots.txt
+if ! grep -q "first-row-id:" /tmp/v3_create_metadata_snapshots.txt; then
+  echo "FAIL describe-metadata --snapshots output missing 'first-row-id' (v3 row lineage)"
+  cat /tmp/v3_create_metadata_snapshots.txt
+  exit 1
+fi
+echo "OK describe-metadata --snapshots shows first-row-id (row lineage)"
+
+# Verify v3 row lineage: manifest carries first_row_id
+{{ICE_CLI}} --config {{CLI_CONFIG}} describe-metadata --manifests "${METADATA_FILE}" > /tmp/v3_create_metadata_manifests.txt
+if ! grep -q "first_row_id:" /tmp/v3_create_metadata_manifests.txt; then
+  echo "FAIL describe-metadata --manifests output missing 'first_row_id' (v3 row lineage)"
+  cat /tmp/v3_create_metadata_manifests.txt
+  exit 1
+fi
+echo "OK describe-metadata --manifests shows first_row_id (row lineage)"
+
+# Cleanup
+{{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME}
+echo "OK Deleted table: ${TABLE_NAME}"
+
+{{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME}
+echo "OK Deleted namespace: ${NAMESPACE_NAME}"
+
+echo "Iceberg v3 table creation test completed successfully"

ice-rest-catalog/src/test/resources/scenarios/iceberg-v3-create/scenario.yaml

@@ -0,0 +1,10 @@
+name: "Iceberg v3 Table Creation"
+description: "Tests creating a format-version 3 table via insert --create-table --format-version=3 and verifies v3 metadata (row lineage)"
+
+catalogConfig:
+  warehouse: "s3://test-bucket/warehouse"
+
+env:
+  NAMESPACE_NAME: "test_v3"
+  TABLE_NAME: "test_v3.t1"
+  INPUT_FILE: "input.parquet"

ice/src/main/java/com/altinity/ice/cli/Main.java

@@ -360,7 +360,12 @@ void createTable(
               names = {"--sort"},
               description =
                   "Sort order, e.g. [{\"column\":\"name\", \"desc\":false, \"nullFirst\":false}]")
-          String sortOrderJson)
+          String sortOrderJson,
+      @CommandLine.Option(
+              names = {"--format-version"},
+              description = "Iceberg table format version (2 or 3). Default: 2",
+              defaultValue = "2")
+          int formatVersion)
       throws IOException {
     setAWSRegion(s3Region);
     try (RESTCatalog catalog = loadCatalog()) {
@@ -387,6 +392,7 @@ void createTable(
           createTableIfNotExists,
           useVendedCredentials,
           s3NoSignRequest,
+          formatVersion,
           partitions,
           sortOrders);
     }
@@ -517,6 +523,12 @@ void insert(
               description =
                   "Sort order, e.g. [{\"column\":\"name\", \"desc\":false, \"nullFirst\":false}]")
           String sortOrderJson,
+      @CommandLine.Option(
+              names = {"--format-version"},
+              description =
+                  "Iceberg table format version (2 or 3) when creating the table with -p/--create-table. Default: 2",
+              defaultValue = "2")
+          int formatVersion,
       @CommandLine.Option(
               names = {"--assume-sorted"},
               description = "Skip data sorting. Assume it's already sorted.")
@@ -639,6 +651,7 @@ void insert(
             createTableIfNotExists,
             useVendedCredentials,
             s3NoSignRequest,
+            formatVersion,
             partitions,
             sortOrders);
       } // delayed in watch mode
@@ -660,6 +673,7 @@ void insert(
               .retryListFile(retryList)
               .partitionList(partitions)
               .sortOrderList(sortOrders)
+              .formatVersion(formatVersion)
               .threadCount(
                   threadCount < 1 ? Runtime.getRuntime().availableProcessors() : threadCount)
               .commitRetries(commitRetries)

ice/src/main/java/com/altinity/ice/cli/internal/cmd/CreateTable.java

@@ -64,9 +64,14 @@ public static void run(
       boolean ignoreAlreadyExists,
       boolean useVendedCredentials,
       boolean s3NoSignRequest,
+      int formatVersion,
       @Nullable List<Main.IcePartition> partitionList,
       @Nullable List<Main.IceSortOrder> sortOrderList)
       throws IOException {
+    if (formatVersion != 2 && formatVersion != 3) {
+      throw new IllegalArgumentException(
+          "--format-version must be 2 or 3 (got " + formatVersion + ")");
+    }
     if (ignoreAlreadyExists && catalog.tableExists(nsTable)) {
       return;
     }
@@ -105,7 +110,12 @@ public static void run(
         // force name-based resolution instead of position-based resolution
         NameMapping mapping = MappingUtil.create(initialSchema);
         String mappingJson = NameMappingParser.toJson(mapping);
-        var props = Map.of(TableProperties.DEFAULT_NAME_MAPPING, mappingJson);
+        var props =
+            Map.of(
+                TableProperties.DEFAULT_NAME_MAPPING,
+                mappingJson,
+                TableProperties.FORMAT_VERSION,
+                String.valueOf(formatVersion));
 
         PartitionSpec partitionSpec =
             partitionList == null

ice/src/main/java/com/altinity/ice/cli/internal/cmd/Describe.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
+import org.apache.iceberg.BaseTable;
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.FileScanTask;
 import org.apache.iceberg.Snapshot;
@@ -142,10 +143,13 @@ private static Table.Data gatherTableData(
     }
 
     boolean includeSchema = optionsSet.contains(Option.INCLUDE_SCHEMA);
+    Integer formatVersion =
+        table instanceof BaseTable bt ? bt.operations().current().formatVersion() : null;
     return new Table.Data(
         includeSchema ? table.schema().toString() : null,
         includeSchema ? table.spec().toString() : null,
         includeSchema ? table.sortOrder().toString() : null,
+        formatVersion,
         optionsSet.contains(Option.INCLUDE_PROPERTIES) ? table.properties() : null,
         table.location(),
         snapshotInfo,
@@ -253,6 +257,7 @@ record Data(
         String schemaRaw,
         String partitionSpecRaw,
         String sortOrderRaw,
+        Integer formatVersion,
         Map<String, String> properties,
         String location,
         Table.Snapshot currentSnapshot,

ice/src/main/java/com/altinity/ice/cli/internal/cmd/DescribeMetadata.java

@@ -11,6 +11,7 @@
 
 import com.altinity.ice.internal.iceberg.io.SchemeFileIO;
 import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import java.io.IOException;
@@ -141,7 +142,8 @@ public static List<SnapshotInfo> extractSnapshots(
               snapshot.operation(),
               currentSnapshotId != null && snapshot.snapshotId() == currentSnapshotId,
               snapshot.summary(),
-              snapshot.manifestListLocation()));
+              snapshot.manifestListLocation(),
+              snapshot.firstRowId()));
     }
     return result;
   }
@@ -213,6 +215,7 @@ private static List<ManifestInfo> extractManifests(
               manifest.existingFilesCount(),
               manifest.deletedFilesCount(),
               manifest.partitionSpecId(),
+              manifest.firstRowId(),
               dataFiles.isEmpty() ? null : dataFiles));
     }
 
@@ -272,7 +275,8 @@ public record SnapshotInfo(
       String operation,
       boolean current,
       Map<String, String> summary,
-      String manifestListLocation) {}
+      String manifestListLocation,
+      @JsonProperty("first-row-id") Long firstRowId) {}
 
   @JsonInclude(JsonInclude.Include.NON_NULL)
   public record HistoryInfo(
@@ -291,6 +295,7 @@ public record ManifestInfo(
       Integer existingFilesCount,
       Integer deletedFilesCount,
       int partitionSpecId,
+      @JsonProperty("first_row_id") Long firstRowId,
       List<DataFileInfo> dataFiles) {}
 
   @JsonInclude(JsonInclude.Include.NON_NULL)

ice/src/main/java/com/altinity/ice/cli/internal/cmd/Insert.java

@@ -1041,6 +1041,7 @@ public record Options(
       @Nullable String retryListFile,
       @Nullable List<Main.IcePartition> partitionList,
       @Nullable List<Main.IceSortOrder> sortOrderList,
+      int formatVersion,
       int threadCount,
       @Nullable String compression,
       int commitRetries,
@@ -1066,6 +1067,7 @@ public static final class Builder {
       private String retryListFile;
       private List<Main.IcePartition> partitionList = List.of();
       private List<Main.IceSortOrder> sortOrderList = List.of();
+      private int formatVersion = 2;
       private int threadCount = Runtime.getRuntime().availableProcessors();
       private String compression;
       private int commitRetries = 10;
@@ -1148,6 +1150,11 @@ public Builder sortOrderList(List<Main.IceSortOrder> sortOrderList) {
         return this;
       }
 
+      public Builder formatVersion(int formatVersion) {
+        this.formatVersion = formatVersion;
+        return this;
+      }
+
       public Builder threadCount(int threadCount) {
         this.threadCount = threadCount;
         return this;
@@ -1185,6 +1192,7 @@ public Options build() {
             retryListFile,
             partitionList,
             sortOrderList,
+            formatVersion,
             threadCount,
             compression,
             commitRetries,

ice/src/main/java/com/altinity/ice/cli/internal/cmd/InsertWatch.java

@@ -363,6 +363,7 @@ private static void insert(
             true,
             options.useVendedCredentials(),
             options.s3NoSignRequest(),
+            options.formatVersion(),
             null,
             null);
       } catch (NotFoundException nfe) {

ice/src/main/resources/META-INF/native-image/com.altinity/ice/reflect-config.json

@@ -244,7 +244,7 @@
   "allDeclaredFields":true,
   "queryAllDeclaredMethods":true,
   "queryAllDeclaredConstructors":true,
-  "methods":[{"name":"currentSnapshot","parameterTypes":[] }, {"name":"location","parameterTypes":[] }, {"name":"metrics","parameterTypes":[] }, {"name":"partitionSpecRaw","parameterTypes":[] }, {"name":"properties","parameterTypes":[] }, {"name":"schemaRaw","parameterTypes":[] }, {"name":"sortOrderRaw","parameterTypes":[] }]
+  "methods":[{"name":"currentSnapshot","parameterTypes":[] }, {"name":"formatVersion","parameterTypes":[] }, {"name":"location","parameterTypes":[] }, {"name":"metrics","parameterTypes":[] }, {"name":"partitionSpecRaw","parameterTypes":[] }, {"name":"properties","parameterTypes":[] }, {"name":"schemaRaw","parameterTypes":[] }, {"name":"sortOrderRaw","parameterTypes":[] }]
 },
 {
   "name":"com.altinity.ice.cli.internal.cmd.Describe$Table$Error",
@@ -300,7 +300,7 @@
   "allDeclaredFields":true,
   "queryAllDeclaredMethods":true,
   "queryAllDeclaredConstructors":true,
-  "methods":[{"name":"addedFilesCount","parameterTypes":[] }, {"name":"dataFiles","parameterTypes":[] }, {"name":"deletedFilesCount","parameterTypes":[] }, {"name":"existingFilesCount","parameterTypes":[] }, {"name":"partitionSpecId","parameterTypes":[] }, {"name":"path","parameterTypes":[] }]
+  "methods":[{"name":"addedFilesCount","parameterTypes":[] }, {"name":"dataFiles","parameterTypes":[] }, {"name":"deletedFilesCount","parameterTypes":[] }, {"name":"existingFilesCount","parameterTypes":[] }, {"name":"firstRowId","parameterTypes":[] }, {"name":"partitionSpecId","parameterTypes":[] }, {"name":"path","parameterTypes":[] }]
 },
 {
   "name":"com.altinity.ice.cli.internal.cmd.DescribeMetadata$MetadataInfo",
@@ -328,7 +328,7 @@
   "allDeclaredFields":true,
   "queryAllDeclaredMethods":true,
   "queryAllDeclaredConstructors":true,
-  "methods":[{"name":"current","parameterTypes":[] }, {"name":"manifestListLocation","parameterTypes":[] }, {"name":"operation","parameterTypes":[] }, {"name":"parentId","parameterTypes":[] }, {"name":"sequenceNumber","parameterTypes":[] }, {"name":"snapshotId","parameterTypes":[] }, {"name":"summary","parameterTypes":[] }, {"name":"timestamp","parameterTypes":[] }, {"name":"timestampMillis","parameterTypes":[] }]
+  "methods":[{"name":"current","parameterTypes":[] }, {"name":"firstRowId","parameterTypes":[] }, {"name":"manifestListLocation","parameterTypes":[] }, {"name":"operation","parameterTypes":[] }, {"name":"parentId","parameterTypes":[] }, {"name":"sequenceNumber","parameterTypes":[] }, {"name":"snapshotId","parameterTypes":[] }, {"name":"summary","parameterTypes":[] }, {"name":"timestamp","parameterTypes":[] }, {"name":"timestampMillis","parameterTypes":[] }]
 },
 {
   "name":"com.altinity.ice.cli.internal.cmd.DescribeMetadata$SnapshotLogEntry",