From 803ba2ed9c0f4640949151eca071f5227741e357 Mon Sep 17 00:00:00 2001 From: Arnav Balyan Date: Sat, 29 Aug 2026 14:50:13 +0530 Subject: [PATCH 1/2] update --- .../paimon/format/orc/OrcReaderFactory.java | 4 + .../orc/OrcPositionalSchemaEvolutionTest.java | 86 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java index 39abe585417d..d5fa7dda3d14 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java @@ -444,6 +444,10 @@ private static OrcRecordReader createRecordReader( .useZeroCopy(OrcConf.USE_ZEROCOPY.getBoolean(conf)) .skipCorruptRecords(OrcConf.SKIP_CORRUPT_DATA.getBoolean(conf)) .tolerateMissingSchema(OrcConf.TOLERATE_MISSING_SCHEMA.getBoolean(conf)) + .forcePositionalEvolution( + OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf)) + .positionalEvolutionLevel( + OrcConf.FORCE_POSITIONAL_EVOLUTION_LEVEL.getInt(conf)) .isSchemaEvolutionCaseAware( OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.getBoolean(conf)); if (!conjunctPredicates.isEmpty() && !deletionVectorsEnabled && selection == null) { diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java new file mode 100644 index 000000000000..9b50ee63eb4c --- /dev/null +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.orc; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.format.FormatReaderContext; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.orc.OrcConf; +import org.apache.orc.OrcFile; +import org.apache.orc.TypeDescription; +import org.apache.orc.Writer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Test positional schema evolution in {@link OrcReaderFactory}. */ +class OrcPositionalSchemaEvolutionTest { + + @TempDir File folder; + + @Test + void testPositionalSchemaEvolution() throws Exception { + Path path = writeOrcFile(); + RowType readType = RowType.builder().field("id", DataTypes.INT()).build(); + Configuration conf = new Configuration(false); + OrcConf.FORCE_POSITIONAL_EVOLUTION.setBoolean(conf, true); + + OrcReaderFactory factory = + new OrcReaderFactory(conf, readType, Collections.emptyList(), 1024, false, false); + LocalFileIO fileIO = new LocalFileIO(); + List values = new ArrayList<>(); + try (RecordReader reader = + factory.createReader( + new FormatReaderContext( + fileIO, path, fileIO.getFileSize(path), null, null))) { + reader.forEachRemaining(row -> values.add(row.isNullAt(0) ? null : row.getInt(0))); + } + + assertThat(values).containsExactly(42); + } + + private Path writeOrcFile() throws Exception { + Path path = new Path(folder.getPath(), "legacy.orc"); + TypeDescription schema = TypeDescription.fromString("struct<_col0:int>"); + try (Writer writer = + OrcFile.createWriter( + new org.apache.hadoop.fs.Path(path.toString()), + OrcFile.writerOptions(new Configuration(false)).setSchema(schema))) { + VectorizedRowBatch batch = schema.createRowBatch(); + ((LongColumnVector) batch.cols[0]).vector[0] = 42; + batch.size = 1; + writer.addRowBatch(batch); + } + return path; + } +} From 9f5a16ff7d2b9bae54a84b03bdb415e2979cc10e Mon Sep 17 00:00:00 2001 From: Arnav Balyan Date: Sun, 30 Aug 2026 16:49:53 +0530 Subject: [PATCH 2/2] update --- .../paimon/format/orc/OrcReaderFactory.java | 10 ++++++---- .../orc/OrcPositionalSchemaEvolutionTest.java | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java index d5fa7dda3d14..32daef9c706e 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java @@ -437,6 +437,7 @@ private static OrcRecordReader createRecordReader( getOffsetAndLengthForSplit(splitStart, splitLength, orcReader.getStripes()); // create ORC row reader configuration + boolean forcePositionalEvolution = OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf); org.apache.orc.Reader.Options options = new org.apache.orc.Reader.Options() .schema(schema) @@ -444,12 +445,13 @@ private static OrcRecordReader createRecordReader( .useZeroCopy(OrcConf.USE_ZEROCOPY.getBoolean(conf)) .skipCorruptRecords(OrcConf.SKIP_CORRUPT_DATA.getBoolean(conf)) .tolerateMissingSchema(OrcConf.TOLERATE_MISSING_SCHEMA.getBoolean(conf)) - .forcePositionalEvolution( - OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf)) - .positionalEvolutionLevel( - OrcConf.FORCE_POSITIONAL_EVOLUTION_LEVEL.getInt(conf)) + .forcePositionalEvolution(forcePositionalEvolution) .isSchemaEvolutionCaseAware( OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.getBoolean(conf)); + if (forcePositionalEvolution) { + options.positionalEvolutionLevel( + OrcConf.FORCE_POSITIONAL_EVOLUTION_LEVEL.getInt(conf)); + } if (!conjunctPredicates.isEmpty() && !deletionVectorsEnabled && selection == null) { // row group filter push down will make row number change incorrect // so deletion vectors mode and bitmap index cannot work with row group push down diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java index 9b50ee63eb4c..2efb610ef197 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcPositionalSchemaEvolutionTest.java @@ -50,11 +50,20 @@ class OrcPositionalSchemaEvolutionTest { @Test void testPositionalSchemaEvolution() throws Exception { - Path path = writeOrcFile(); - RowType readType = RowType.builder().field("id", DataTypes.INT()).build(); Configuration conf = new Configuration(false); OrcConf.FORCE_POSITIONAL_EVOLUTION.setBoolean(conf, true); + assertThat(readOrcFile(conf)).containsExactly(42); + } + + @Test + void testPositionalSchemaEvolutionDisabledByDefault() throws Exception { + assertThat(readOrcFile(new Configuration(false))).containsExactly((Integer) null); + } + + private List readOrcFile(Configuration conf) throws Exception { + Path path = writeOrcFile(); + RowType readType = RowType.builder().field("id", DataTypes.INT()).build(); OrcReaderFactory factory = new OrcReaderFactory(conf, readType, Collections.emptyList(), 1024, false, false); LocalFileIO fileIO = new LocalFileIO(); @@ -66,7 +75,7 @@ void testPositionalSchemaEvolution() throws Exception { reader.forEachRemaining(row -> values.add(row.isNullAt(0) ? null : row.getInt(0))); } - assertThat(values).containsExactly(42); + return values; } private Path writeOrcFile() throws Exception {