From 141d6fc129f5a7eb2e66aa2fffee6d4f2f2a82a4 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 19:11:11 +0000 Subject: [PATCH 01/10] Port the FSST symbol table trainer and compressor to Java FSST is proposed for Parquet in parquet-format#531, with a C++ implementation on an Arrow branch and nothing in Java. A format change needs a second implementation, so this is the codec core it needs: a symbol table, a trainer, and a compressor and decompressor for the 8-bit code stream. The trainer is a port of FSST's reference implementation rather than a reimplementation, because a writer's ratio has to match what other writers produce for the same input, and the symbol table a reader rebuilds has to be the table the writer chose. Seven constructs did not survive a literal port and are commented where they appear: the symbol descriptor needs 64 bits because the length field alone overflows an int at length 8; the reference hash needs an unsigned shift, which matters because the sampler chains it over full 64-bit values; every byte read needs masking; the hash table becomes parallel long arrays rather than 1024 objects; the pair counter reuses one array across pages with the reference's lazy reset; and the compressor's unconditional eight-byte read needs a padded buffer, since Java cannot read past an array. The file's code space is a permutation of the trainer's. The trainer numbers symbols in an order its own shortcuts depend on, while the file orders them by length so a reader can rebuild the table from a length histogram alone. Rather than renumber and break the shortcuts, the trainer's numbering is kept and translated at the emit site, folded into the compressor's inner loop so there is no scratch buffer and no second pass. Everything here sits behind interfaces that name the concept rather than the codec, because two more codecs are planned against them: FSST_16, which the proposal's own conformance files already contain, and OnPair. Each is a new symbol table, trainer and code stream implementation, not a change to these. Two test classes. One round-trips through serialization on adversarial input: every byte value, values that straddle the compressor's chunk boundary, incompressible bytes, empty values, and a corpus large enough to exercise the sampler. The other compares against the reference implementation directly: for six corpora it asserts that the serialized symbol table and the code stream are byte-identical to what the reference produces. The corpora are generated from a spelled-out generator rather than stored, and each one's digest is asserted before the comparison, so a drifting generator reports itself as a generator problem instead of as a codec regression. Not here yet: the writer and reader that put this on a page, and the symbol table's file-level home, which is thrift-gated. --- .gitattributes | 4 + LICENSE | 36 ++ .../values/symboltable/CodeStreamDecoder.java | 47 ++ .../values/symboltable/CodeStreamEncoder.java | 48 ++ .../values/symboltable/SymbolTable.java | 54 +++ .../symboltable/SymbolTableTrainer.java | 36 ++ .../values/symboltable/SymbolTableType.java | 66 +++ .../symboltable/TrainedSymbolTable.java | 44 ++ .../values/symboltable/ValueBuffer.java | 130 ++++++ .../fsst/Fsst8CodeStreamDecoder.java | 93 ++++ .../fsst/Fsst8CodeStreamEncoder.java | 120 +++++ .../symboltable/fsst/Fsst8SymbolTable.java | 213 +++++++++ .../values/symboltable/fsst/FsstCodes.java | 161 +++++++ .../values/symboltable/fsst/FsstCounters.java | 162 +++++++ .../symboltable/fsst/FsstSymbolTable.java | 267 ++++++++++++ .../values/symboltable/fsst/FsstTrainer.java | 410 ++++++++++++++++++ .../fsst/FsstCodecRoundTripTest.java | 230 ++++++++++ .../fsst/FsstReferenceComparisonTest.java | 252 +++++++++++ .../src/test/resources/fsst/allbytes.codes | Bin 0 -> 3648 bytes .../src/test/resources/fsst/allbytes.table | Bin 0 -> 307 bytes .../src/test/resources/fsst/binary.codes | Bin 0 -> 12527 bytes .../src/test/resources/fsst/binary.table | Bin 0 -> 264 bytes .../src/test/resources/fsst/chunks.codes | 5 + .../src/test/resources/fsst/chunks.table | Bin 0 -> 116 bytes .../src/test/resources/fsst/empties.codes | 1 + .../src/test/resources/fsst/empties.table | Bin 0 -> 13 bytes .../src/test/resources/fsst/sampled.codes | 1 + .../src/test/resources/fsst/sampled.table | Bin 0 -> 254 bytes .../src/test/resources/fsst/urls.codes | 39 ++ .../src/test/resources/fsst/urls.table | Bin 0 -> 187 bytes pom.xml | 3 + 31 files changed, 2422 insertions(+) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamDecoder.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamEncoder.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTable.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableTrainer.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableType.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/TrainedSymbolTable.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamDecoder.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamEncoder.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8SymbolTable.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodes.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCounters.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstSymbolTable.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstTrainer.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java create mode 100644 parquet-column/src/test/resources/fsst/allbytes.codes create mode 100644 parquet-column/src/test/resources/fsst/allbytes.table create mode 100644 parquet-column/src/test/resources/fsst/binary.codes create mode 100644 parquet-column/src/test/resources/fsst/binary.table create mode 100644 parquet-column/src/test/resources/fsst/chunks.codes create mode 100644 parquet-column/src/test/resources/fsst/chunks.table create mode 100644 parquet-column/src/test/resources/fsst/empties.codes create mode 100644 parquet-column/src/test/resources/fsst/empties.table create mode 100644 parquet-column/src/test/resources/fsst/sampled.codes create mode 100644 parquet-column/src/test/resources/fsst/sampled.table create mode 100644 parquet-column/src/test/resources/fsst/urls.codes create mode 100644 parquet-column/src/test/resources/fsst/urls.table diff --git a/.gitattributes b/.gitattributes index a533bb4c5e..759dadcbe5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,3 +19,7 @@ * text eol=lf *.png binary core.autocrlf=false + +# FSST test fixtures are byte streams, not text: line-ending normalization would corrupt them. +parquet-column/src/test/resources/fsst/*.table binary +parquet-column/src/test/resources/fsst/*.codes binary diff --git a/LICENSE b/LICENSE index 2c96440ccc..407b288bcb 100644 --- a/LICENSE +++ b/LICENSE @@ -206,3 +206,39 @@ Copyright: 2012-2014 Twitter Home page: https://github.com/twitter/elephant-bird License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the FSST project. parquet-column's FSST symbol +table trainer and compressor are a Java port of FSST's reference +implementation. + +* parquet-column's org.apache.parquet.column.values.symboltable.fsst package + is derived from FSST's libfsst.hpp and libfsst.cpp, at commit + 89f49c580c6388acf3b6ed2a49e1bfde6c05e616. + +Copyright: 2018-2020 CWI, TU Munich, FSU Jena +Home page: https://github.com/cwida/fsst +License: MIT License + +MIT License + +Copyright (c) 2018-2020, CWI, TU Munich, FSU Jena + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamDecoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamDecoder.java new file mode 100644 index 0000000000..24f9166f85 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamDecoder.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +/** + * Expands a stream of codes back into the values it was made from. + * + *

The decoder works on one value's slice of the code stream, which is what makes a value + * addressable without expanding the ones before it. + */ +public interface CodeStreamDecoder { + + /** + * How many bytes one value's codes expand to, so a caller can size a destination buffer. + * + *

Cheaper than expanding, because it reads the codes without copying symbol bytes. + */ + int expandedLength(byte[] codes, int codesOffset, int codesLength); + + /** + * Expands one value's codes. + * + * @param codes the code stream + * @param codesOffset where this value's codes start + * @param codesLength how many bytes of codes belong to this value + * @param destination where the value bytes are written + * @param destinationOffset where to write the value bytes + * @return the number of bytes written + */ + int expand(byte[] codes, int codesOffset, int codesLength, byte[] destination, int destinationOffset); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamEncoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamEncoder.java new file mode 100644 index 0000000000..e06ca7ed0c --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/CodeStreamEncoder.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +/** + * Compresses values into a stream of codes over a symbol table. + * + *

Kept separate from the table itself because a table can be paired with more than one code + * stream framing, and because the encode side needs lookup structures that never reach a file. + */ +public interface CodeStreamEncoder { + + /** + * Upper bound on the compressed size of a value, so a caller can size a destination buffer. + * + *

Must account for escapes, which make the worst case larger than the input. + */ + int maxCompressedLength(int rawLength); + + /** + * Compresses one value. + * + * @param source the value bytes + * @param sourceOffset where the value starts + * @param sourceLength the value's length in bytes + * @param destination where the codes are written; must hold {@link #maxCompressedLength} more + * bytes at {@code destinationOffset} + * @param destinationOffset where to write the codes + * @return the number of bytes written + */ + int compress(byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTable.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTable.java new file mode 100644 index 0000000000..64f207b272 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTable.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import org.apache.parquet.bytes.BytesInput; + +/** + * A table mapping codes to the byte sequences they stand for, in the form that is written to a file + * and read back from one. + * + *

This is the decode-side and serialization view of a table. It deliberately knows nothing about + * how the table was chosen or how values are compressed against it, so that a variant with wider + * codes or a differently built table is a new implementation here rather than a change to the + * writer and reader above it. + */ +public interface SymbolTable { + + SymbolTableType type(); + + /** Number of symbols, excluding the escape mechanism. */ + int symbolCount(); + + /** Length in bytes of the symbol with this code. */ + int symbolLength(int code); + + /** + * Appends the symbol with this code to {@code destination} at {@code position}. + * + * @return the number of bytes written + */ + int copySymbol(int code, byte[] destination, int position); + + /** The serialized table body, as it appears in a file. */ + BytesInput serialize(); + + /** A decoder for code streams written against this table. */ + CodeStreamDecoder decoder(); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableTrainer.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableTrainer.java new file mode 100644 index 0000000000..85e570c95f --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableTrainer.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +/** + * Chooses a symbol table for a set of values. + * + *

This is the seam that separates how a table is chosen from how it is written and used. A + * different code width, or a different way of mining symbols out of the data, is a new + * implementation of this interface and of {@link SymbolTable}, with nothing above them changed. + */ +public interface SymbolTableTrainer { + + SymbolTableType type(); + + /** + * Trains a table on the given values, which the trainer may read in any order and more than once. + */ + TrainedSymbolTable train(ValueBuffer values); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableType.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableType.java new file mode 100644 index 0000000000..85441ed986 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableType.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +/** + * The representation of a symbol table, which fixes the width of a code in the code stream and the + * framing of an escaped literal. + * + *

A column's encoding does not identify the representation on its own: a single encoding covers + * every variant, and the type is carried with the table. A reader therefore has to read the table + * before it can commit to decoding the column, and may reject a type it does not implement. + * + *

The numeric values are not ratified. Only {@link #FSST_8} appears in the format proposal so + * far; {@link #FSST_16}'s value is this implementation's proposal and must be confirmed against + * parquet-format issue #531 before any file written with it is treated as portable. + */ +public enum SymbolTableType { + /** Single-byte codes, at most 255 symbols, escape marker 255 followed by one literal byte. */ + FSST_8(0, 1), + + /** Two-byte codes, at most 65535 symbols, escape marker 65535 followed by a literal. */ + FSST_16(1, 2); + + private final int typeValue; + private final int codeWidth; + + SymbolTableType(int typeValue, int codeWidth) { + this.typeValue = typeValue; + this.codeWidth = codeWidth; + } + + /** The value written to the symbol table page header. */ + public int typeValue() { + return typeValue; + } + + /** Width in bytes of one code in the code stream. */ + public int codeWidth() { + return codeWidth; + } + + public static SymbolTableType fromTypeValue(int typeValue) { + for (SymbolTableType type : values()) { + if (type.typeValue == typeValue) { + return type; + } + } + throw new IllegalArgumentException("Unsupported symbol table type: " + typeValue); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/TrainedSymbolTable.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/TrainedSymbolTable.java new file mode 100644 index 0000000000..234e6202ac --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/TrainedSymbolTable.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +/** + * What training produces: a table to write to the file, and an encoder that compresses against it. + * + *

The two come out together because the encode-side lookup structures are a byproduct of + * training and rebuilding them from the serialized table would be wasted work. + */ +public final class TrainedSymbolTable { + + private final SymbolTable table; + private final CodeStreamEncoder encoder; + + public TrainedSymbolTable(SymbolTable table, CodeStreamEncoder encoder) { + this.table = table; + this.encoder = encoder; + } + + public SymbolTable table() { + return table; + } + + public CodeStreamEncoder encoder() { + return encoder; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java new file mode 100644 index 0000000000..fad31d5b5a --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import java.util.Arrays; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.io.api.Binary; + +/** + * The values of one page, held as a single byte array plus start offsets. + * + *

Training needs to read values in an order of its own choosing, and more than once, so the + * values have to be somewhere addressable when the page is closed. Holding them as one array rather + * than as a list of separately allocated values keeps that at one copy of the page with no + * per-value object, which matters because a writer is already buffering this data anyway. + * + *

The array carries eight bytes of zero padding past the last value. Symbol matching loads eight + * bytes at a time and deliberately reads past the end of a short value; the padding is what makes + * that read in-bounds, and zeros are what make it harmless, since no symbol can match beyond the + * value's own length once the load is masked. + */ +public class ValueBuffer { + + /** Slack past the last value, for eight-byte loads that overshoot. */ + public static final int TAIL_PADDING = 8; + + private static final int DEFAULT_INITIAL_CAPACITY = 64 * 1024; + private static final int DEFAULT_INITIAL_VALUES = 1024; + + private byte[] data; + private int size; + private int[] offsets; + private int valueCount; + + public ValueBuffer() { + this(DEFAULT_INITIAL_CAPACITY, DEFAULT_INITIAL_VALUES); + } + + public ValueBuffer(int initialCapacity, int initialValues) { + this.data = new byte[initialCapacity + TAIL_PADDING]; + this.offsets = new int[initialValues + 1]; + this.offsets[0] = 0; + } + + public void add(Binary value) { + int length = value.length(); + ensureCapacity(size + length); + ensureValues(valueCount + 1); + // Read through a duplicate so the value's own buffer position is left alone. This is the one + // copy of the value; taking the backing array instead would copy for slice-backed values. + value.toByteBuffer().duplicate().get(data, size, length); + size += length; + offsets[++valueCount] = size; + } + + public void add(byte[] source, int offset, int length) { + ensureCapacity(size + length); + ensureValues(valueCount + 1); + System.arraycopy(source, offset, data, size, length); + size += length; + offsets[++valueCount] = size; + } + + public int valueCount() { + return valueCount; + } + + /** Total length of all values, which is the size of the region {@link #data} holds. */ + public int byteCount() { + return size; + } + + /** + * The backing array. Valid from 0 to {@link #byteCount()}, followed by {@link #TAIL_PADDING} zero + * bytes that a reader may load but must not interpret. + */ + public byte[] data() { + return data; + } + + public int offset(int index) { + return offsets[index]; + } + + public int length(int index) { + return offsets[index + 1] - offsets[index]; + } + + public BytesInput asBytesInput() { + return BytesInput.from(data, 0, size); + } + + public void reset() { + // Clear through the padding so the next round of eight-byte overshoot still reads zeros. + Arrays.fill(data, 0, Math.min(size + TAIL_PADDING, data.length), (byte) 0); + size = 0; + valueCount = 0; + } + + private void ensureCapacity(int required) { + if (required + TAIL_PADDING <= data.length) { + return; + } + int capacity = Math.max(data.length * 2, required + TAIL_PADDING); + data = Arrays.copyOf(data, capacity); + } + + private void ensureValues(int required) { + if (required < offsets.length) { + return; + } + offsets = Arrays.copyOf(offsets, Math.max(offsets.length * 2, required + 1)); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamDecoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamDecoder.java new file mode 100644 index 0000000000..9aeb2766d1 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamDecoder.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import org.apache.parquet.column.values.symboltable.CodeStreamDecoder; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * Expands 8-bit FSST codes back into value bytes. + * + *

Decoding needs nothing but the table: each code either names a symbol to copy or announces that + * the next byte stands for itself. There is no state carried between values, which is what lets a + * reader expand one value without touching the ones before it. + */ +class Fsst8CodeStreamDecoder implements CodeStreamDecoder { + + private final byte[] symbolBytes; + private final int[] symbolOffsets; + private final int symbolCount; + + Fsst8CodeStreamDecoder(byte[] symbolBytes, int[] symbolOffsets, int symbolCount) { + this.symbolBytes = symbolBytes; + this.symbolOffsets = symbolOffsets; + this.symbolCount = symbolCount; + } + + @Override + public int expandedLength(byte[] codes, int codesOffset, int codesLength) { + int length = 0; + int position = codesOffset; + int end = codesOffset + codesLength; + while (position < end) { + int code = codes[position++] & 0xFF; + if (code == Fsst8SymbolTable.ESCAPE) { + requireLiteral(position, end); + position++; + length++; + } else { + length += symbolLength(code); + } + } + return length; + } + + @Override + public int expand(byte[] codes, int codesOffset, int codesLength, byte[] destination, int destinationOffset) { + int written = destinationOffset; + int position = codesOffset; + int end = codesOffset + codesLength; + while (position < end) { + int code = codes[position++] & 0xFF; + if (code == Fsst8SymbolTable.ESCAPE) { + requireLiteral(position, end); + destination[written++] = codes[position++]; + } else { + int length = symbolLength(code); + System.arraycopy(symbolBytes, symbolOffsets[code], destination, written, length); + written += length; + } + } + return written - destinationOffset; + } + + private int symbolLength(int code) { + if (code >= symbolCount) { + throw new ParquetDecodingException( + "FSST value references symbol code " + code + " but the table holds " + symbolCount + " symbols"); + } + return symbolOffsets[code + 1] - symbolOffsets[code]; + } + + private static void requireLiteral(int position, int end) { + if (position >= end) { + throw new ParquetDecodingException("FSST value ends with an escape and no literal byte"); + } + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamEncoder.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamEncoder.java new file mode 100644 index 0000000000..5aba4b4fbd --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8CodeStreamEncoder.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_BASE; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.ICL_FREE; + +import org.apache.parquet.column.values.symboltable.CodeStreamEncoder; + +/** + * Compresses value bytes into 8-bit FSST codes. + * + *

Two things here are not free choices. The first is chunking: a value is compressed 511 bytes at + * a time, with the table's separator byte written just past each chunk. That bound is what the + * reference implementation's vectorized compressor uses, and matching it is what keeps the code + * stream identical whichever compressor produced it. Since no symbol contains the separator, no + * match can reach across a chunk boundary, so the boundary is also what makes the eight-byte read at + * the end of a chunk safe without a shorter tail path. + * + *

The compression loop is a port of FSST's reference implementation by Peter Boncz, Viktor Leis + * and Thomas Neumann (CWI / TU Munich), MIT licensed, at https://github.com/cwida/fsst, commit + * 89f49c580c6388acf3b6ed2a49e1bfde6c05e616. Its three hand-tuned variants differ only in branch + * layout and produce the same bytes, so one is enough here. + * + *

The second is the code translation. The trainer numbers symbols in an order that lets the + * compressor skip work, and a file numbers them by length. So every code leaves this class through + * the permutation between the two; an escape passes through untouched, since it is a fixed marker in + * both orders rather than a symbol. + */ +class Fsst8CodeStreamEncoder implements CodeStreamEncoder { + + /** Bytes compressed at a time, matching the reference implementation's vectorized compressor. */ + private static final int CHUNK_SIZE = 511; + + /** Room past the chunk for the separator byte and for an eight-byte read at the chunk's end. */ + private static final int CHUNK_PADDING = 8; + + private final FsstSymbolTable table; + private final byte[] fileCodeForTrainerCode; + private final byte[] chunk = new byte[CHUNK_SIZE + CHUNK_PADDING]; + + Fsst8CodeStreamEncoder(FsstSymbolTable table, Fsst8SymbolTable fileTable) { + this.table = table; + this.fileCodeForTrainerCode = fileTable.encodeCodeMap(); + } + + @Override + public int maxCompressedLength(int rawLength) { + // Every byte could escape, which costs the marker plus the byte itself. + return 2 * rawLength; + } + + @Override + public int compress(byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset) { + int written = destinationOffset; + int consumed = 0; + while (consumed < sourceLength) { + int chunkLength = Math.min(CHUNK_SIZE, sourceLength - consumed); + System.arraycopy(source, sourceOffset + consumed, chunk, 0, chunkLength); + // Bytes past the separator keep whatever a longer previous chunk left there. That is harmless, + // and is what the reference implementation does: a match reaching past the separator would + // have to contain it, and no symbol does. + chunk[chunkLength] = (byte) table.terminator; + written = compressChunk(chunkLength, destination, written); + consumed += chunkLength; + } + return written - destinationOffset; + } + + /** Compresses one chunk out of {@link #chunk}, returning the new write position. */ + private int compressChunk(int chunkLength, byte[] destination, int writePosition) { + char[] shortCodes = table.shortCodes; + long[] hashValues = table.hashValues; + long[] hashDescriptors = table.hashDescriptors; + byte[] codeMap = fileCodeForTrainerCode; + int byteLimit = table.byteLimit; + int written = writePosition; + int position = 0; + while (position < chunkLength) { + long word = FsstCodes.loadWordUnchecked(chunk, position); + int shortCode = shortCodes[(int) (word & 0xFFFF)]; + int bucket = FsstCodes.hashBucket(word); + long descriptor = hashDescriptors[bucket]; + if (descriptor < ICL_FREE && hashValues[bucket] == FsstCodes.maskWord(word, descriptor)) { + // A symbol of three bytes or more. + destination[written++] = codeMap[FsstCodes.code(descriptor)]; + position += FsstCodes.length(descriptor); + } else if ((shortCode & 0xFF) < byteLimit) { + // A two-byte symbol, and the miss above rules out a longer one starting here. + destination[written++] = codeMap[shortCode & 0xFF]; + position += 2; + } else if ((shortCode & CODE_BASE) != 0) { + // No symbol matches, so the byte stands for itself behind the escape marker. + destination[written++] = (byte) Fsst8SymbolTable.ESCAPE; + destination[written++] = (byte) word; + position += 1; + } else { + destination[written++] = codeMap[shortCode & 0xFF]; + position += 1; + } + } + return written; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8SymbolTable.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8SymbolTable.java new file mode 100644 index 0000000000..0f8a86359e --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/Fsst8SymbolTable.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import java.util.Arrays; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.column.values.symboltable.CodeStreamDecoder; +import org.apache.parquet.column.values.symboltable.SymbolTable; +import org.apache.parquet.column.values.symboltable.SymbolTableType; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * An 8-bit FSST symbol table in the form a file carries it. + * + *

The codes here are not the codes the trainer works with. The trainer numbers symbols in an order + * that suits compression, and the format numbers them in order of length, shortest first, so that a + * reader can rebuild the table from a length histogram without storing a length per symbol. The + * permutation between the two is built once, when a trained table is converted, and is what the + * encoder applies to every code it emits. + * + *

Serialized body: one byte holding the symbol count, then eight bytes counting the symbols of + * each length from one to eight, then the symbol bytes end to end in that same order. At most 255 + * symbols of at most 8 bytes each, so at most 2049 bytes. That layout is single-byte-code only, which + * is why the 16-bit variant needs a table format of its own rather than an extra field here. + */ +public class Fsst8SymbolTable implements SymbolTable { + + /** Code standing for one literal byte, which follows it in the code stream. */ + public static final int ESCAPE = 0xFF; + + static final int MAX_SYMBOL_LENGTH = 8; + + /** Symbol count plus the eight histogram entries. */ + static final int HEADER_SIZE = 9; + + /** Largest serialized body: the header plus 255 symbols of 8 bytes. */ + static final int MAX_SERIALIZED_SIZE = HEADER_SIZE + FsstCodes.MAX_SYMBOLS * MAX_SYMBOL_LENGTH; + + /** Symbol bytes end to end, in code order. */ + private final byte[] symbolBytes; + + /** Where each symbol starts in {@link #symbolBytes}, with one extra entry for the end. */ + private final int[] symbolOffsets; + + private final int symbolCount; + + /** + * Maps a trainer code to the code written to a file, or {@link #ESCAPE} where the trainer has no + * such symbol. Null on a table read back from a file, which never encodes. + */ + private final byte[] encodeCodeMap; + + private Fsst8SymbolTable(byte[] symbolBytes, int[] symbolOffsets, int symbolCount, byte[] encodeCodeMap) { + this.symbolBytes = symbolBytes; + this.symbolOffsets = symbolOffsets; + this.symbolCount = symbolCount; + this.encodeCodeMap = encodeCodeMap; + } + + /** + * Converts a trained table into the file representation, renumbering symbols into length order. + * + *

Symbols of the same length keep their relative order from the trainer, which is what makes the + * renumbering reproducible. + */ + static Fsst8SymbolTable of(FsstSymbolTable trained) { + int symbolCount = trained.symbolCount; + if (symbolCount > FsstCodes.MAX_SYMBOLS) { + throw new IllegalArgumentException( + "FSST8 holds at most " + FsstCodes.MAX_SYMBOLS + " symbols, got " + symbolCount); + } + // A counting sort by length, which is stable and needs no comparator. + int[] countByLength = new int[MAX_SYMBOL_LENGTH + 1]; + for (int code = 0; code < symbolCount; code++) { + countByLength[FsstCodes.length(trained.symbolDescriptors[code])]++; + } + int[] nextCodeForLength = new int[MAX_SYMBOL_LENGTH + 2]; + int[] nextOffsetForLength = new int[MAX_SYMBOL_LENGTH + 2]; + int runningCode = 0; + int runningOffset = 0; + for (int length = 1; length <= MAX_SYMBOL_LENGTH; length++) { + nextCodeForLength[length] = runningCode; + nextOffsetForLength[length] = runningOffset; + runningCode += countByLength[length]; + runningOffset += countByLength[length] * length; + } + + byte[] symbolBytes = new byte[runningOffset]; + int[] symbolOffsets = new int[symbolCount + 1]; + byte[] encodeCodeMap = new byte[FsstCodes.MAX_SYMBOLS]; + Arrays.fill(encodeCodeMap, (byte) ESCAPE); + + for (int trainerCode = 0; trainerCode < symbolCount; trainerCode++) { + int length = FsstCodes.length(trained.symbolDescriptors[trainerCode]); + int fileCode = nextCodeForLength[length]++; + int offset = nextOffsetForLength[length]; + nextOffsetForLength[length] += length; + long value = trained.symbolValues[trainerCode]; + for (int i = 0; i < length; i++) { + symbolBytes[offset + i] = (byte) (value >>> (i * 8)); + } + symbolOffsets[fileCode] = offset; + encodeCodeMap[trainerCode] = (byte) fileCode; + } + symbolOffsets[symbolCount] = runningOffset; + return new Fsst8SymbolTable(symbolBytes, symbolOffsets, symbolCount, encodeCodeMap); + } + + /** Reads a table back from the bytes a file carries. */ + public static Fsst8SymbolTable deserialize(byte[] body, int offset, int length) { + if (length < HEADER_SIZE || length > MAX_SERIALIZED_SIZE) { + throw new ParquetDecodingException("Invalid FSST symbol table body size: " + length); + } + int symbolCount = body[offset] & 0xFF; + int[] histogram = new int[MAX_SYMBOL_LENGTH]; + int histogramSum = 0; + int expectedSymbolBytes = 0; + for (int i = 0; i < MAX_SYMBOL_LENGTH; i++) { + histogram[i] = body[offset + 1 + i] & 0xFF; + histogramSum += histogram[i]; + expectedSymbolBytes += histogram[i] * (i + 1); + } + // Validate before allocating, so a corrupt header cannot ask for a large buffer. + if (histogramSum != symbolCount) { + throw new ParquetDecodingException("FSST length histogram sums to " + histogramSum + + " but the table declares " + symbolCount + " symbols"); + } + if (expectedSymbolBytes != length - HEADER_SIZE) { + throw new ParquetDecodingException("FSST symbol bytes are " + (length - HEADER_SIZE) + + " bytes but the length histogram accounts for " + expectedSymbolBytes); + } + + byte[] symbolBytes = new byte[expectedSymbolBytes]; + System.arraycopy(body, offset + HEADER_SIZE, symbolBytes, 0, expectedSymbolBytes); + int[] symbolOffsets = new int[symbolCount + 1]; + int code = 0; + int position = 0; + for (int length1 = 1; length1 <= MAX_SYMBOL_LENGTH; length1++) { + for (int i = 0; i < histogram[length1 - 1]; i++) { + symbolOffsets[code++] = position; + position += length1; + } + } + symbolOffsets[symbolCount] = position; + return new Fsst8SymbolTable(symbolBytes, symbolOffsets, symbolCount, null); + } + + @Override + public SymbolTableType type() { + return SymbolTableType.FSST_8; + } + + @Override + public int symbolCount() { + return symbolCount; + } + + @Override + public int symbolLength(int code) { + return symbolOffsets[code + 1] - symbolOffsets[code]; + } + + @Override + public int copySymbol(int code, byte[] destination, int position) { + int length = symbolLength(code); + System.arraycopy(symbolBytes, symbolOffsets[code], destination, position, length); + return length; + } + + @Override + public BytesInput serialize() { + int[] histogram = new int[MAX_SYMBOL_LENGTH]; + for (int code = 0; code < symbolCount; code++) { + histogram[symbolLength(code) - 1]++; + } + byte[] body = new byte[HEADER_SIZE + symbolBytes.length]; + body[0] = (byte) symbolCount; + for (int i = 0; i < MAX_SYMBOL_LENGTH; i++) { + body[1 + i] = (byte) histogram[i]; + } + System.arraycopy(symbolBytes, 0, body, HEADER_SIZE, symbolBytes.length); + return BytesInput.from(body); + } + + @Override + public CodeStreamDecoder decoder() { + return new Fsst8CodeStreamDecoder(symbolBytes, symbolOffsets, symbolCount); + } + + /** The trainer-code to file-code map, for the encoder paired with this table. */ + byte[] encodeCodeMap() { + if (encodeCodeMap == null) { + throw new IllegalStateException("a symbol table read from a file cannot encode"); + } + return encodeCodeMap; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodes.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodes.java new file mode 100644 index 0000000000..05b629b993 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodes.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +/** + * Constants and bit-field helpers for the FSST symbol representation. + * + *

This file is a Java port of parts of the FSST reference implementation by Peter Boncz, + * Viktor Leis and Thomas Neumann (CWI / TU Munich), distributed under the MIT license and + * available at https://github.com/cwida/fsst. The port follows commit + * 89f49c580c6388acf3b6ed2a49e1bfde6c05e616. Encoding decisions are reproduced exactly, because + * the symbol table a writer produces has to match what other implementations produce for the + * same input. + * + *

A symbol is a byte sequence of length 1 to 8, held as a little-endian {@code long} together + * with a packed descriptor called {@code icl}: {@code (length << 28) | (code << 16) | + * ignoredBits}, where {@code ignoredBits} is {@code (8 - length) * 8}. The descriptor is kept in + * a {@code long} rather than an {@code int} on purpose: at length 8 the length field alone is + * {@code 8 << 28}, which overflows a signed 32-bit integer. Because every descriptor fits in 32 + * bits, signed {@code long} comparison of two descriptors matches the unsigned comparison the + * reference implementation performs. + */ +final class FsstCodes { + + private FsstCodes() {} + + /** Maximum number of bytes in one symbol. */ + static final int MAX_SYMBOL_LENGTH = 8; + + /** Width of the length field in the packed values held by the code lookup tables. */ + static final int LEN_BITS = 12; + + static final int CODE_BITS = 9; + + /** Codes below this value are pseudo codes standing for an escaped single byte. */ + static final int CODE_BASE = 256; + + /** One past the highest representable code; also marks a symbol with no code assigned yet. */ + static final int CODE_MAX = 1 << CODE_BITS; + + static final int CODE_MASK = CODE_MAX - 1; + + /** Number of buckets in the symbol hash table, which holds symbols of three bytes or more. */ + static final int HASH_TAB_SIZE = 1 << 10; + + /** Descriptor value marking a free hash bucket: length 15 and an unassigned code. */ + static final long ICL_FREE = (15L << 28) | ((long) CODE_MASK << 16); + + /** Highest number of symbols a table may hold, so that every code fits in one byte. */ + static final int MAX_SYMBOLS = 255; + + private static final long HASH_PRIME = 2971215073L; + private static final int HASH_SHIFT = 15; + + /** + * The reference implementation's hash, used both for symbol lookup and to drive the sampler. + * + *

The shift must be unsigned. For symbol lookup the input is the next three bytes, so the + * product cannot reach the sign bit and the distinction does not arise; the sampler chains this + * function over full 64-bit values, where a signed shift would send the whole sequence down a + * different path and produce a different symbol table. + */ + static long hash(long w) { + long product = w * HASH_PRIME; + return product ^ (product >>> HASH_SHIFT); + } + + /** Packs a symbol descriptor from a code and a length. */ + static long icl(int code, int length) { + return ((long) length << 28) | ((long) code << 16) | ((long) (MAX_SYMBOL_LENGTH - length) * 8); + } + + static int length(long icl) { + return (int) (icl >>> 28); + } + + static int code(long icl) { + return (int) ((icl >>> 16) & CODE_MASK); + } + + /** Number of high bits to clear in an input word before comparing it against a symbol. */ + static int ignoredBits(long icl) { + return (int) (icl & 0xFF); + } + + /** The symbol's first byte, as an unsigned value. */ + static int first(long val) { + return (int) (val & 0xFF); + } + + /** The symbol's first two bytes, as an unsigned value. */ + static int first2(long val) { + return (int) (val & 0xFFFF); + } + + /** Hash bucket for a symbol, keyed on its first three bytes. */ + static int hashBucket(long val) { + return (int) (hash(val & 0xFFFFFF) & (HASH_TAB_SIZE - 1)); + } + + /** Builds the little-endian word for the first {@code length} bytes at {@code offset}. */ + static long loadSymbolBytes(byte[] src, int offset, int length) { + long val = 0; + for (int i = 0; i < length; i++) { + val |= (long) (src[offset + i] & 0xFF) << (i * 8); + } + return val; + } + + /** + * Reads eight bytes little-endian, substituting zeros past {@code limit}. + * + *

The reference implementation reads eight bytes unconditionally and relies on the input + * buffer being padded. Java cannot read out of bounds, so the tail is zero-filled here; callers + * that must reproduce the reference behaviour byte for byte pad their input instead of relying + * on this. + */ + static long loadWord(byte[] src, int offset, int limit) { + long val = 0; + int n = Math.min(MAX_SYMBOL_LENGTH, limit - offset); + for (int i = 0; i < n; i++) { + val |= (long) (src[offset + i] & 0xFF) << (i * 8); + } + return val; + } + + /** + * Reads eight bytes little-endian with no bounds reasoning beyond the array itself. + * + *

For the compressor's inner loop, which works out of a buffer padded past its logical end so + * that the read is always in bounds. A byte-array view is used rather than eight shifts because + * this is the single hottest read in the codec. + */ + static long loadWordUnchecked(byte[] src, int offset) { + return (long) LITTLE_ENDIAN_LONG.get(src, offset); + } + + private static final java.lang.invoke.VarHandle LITTLE_ENDIAN_LONG = + java.lang.invoke.MethodHandles.byteArrayViewVarHandle(long[].class, java.nio.ByteOrder.LITTLE_ENDIAN); + + /** Clears the high bits of {@code word} that a symbol with this descriptor ignores. */ + static long maskWord(long word, long icl) { + return word & (-1L >>> ignoredBits(icl)); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCounters.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCounters.java new file mode 100644 index 0000000000..fca687996b --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstCounters.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_MAX; + +import java.util.Arrays; + +/** + * Occurrence counters used while training a symbol table: how often each symbol occurs, and how + * often each ordered pair of symbols occurs back to back. + * + *

Java port of the 64-bit {@code Counters} from the FSST reference implementation by Peter Boncz, + * Viktor Leis and Thomas Neumann (CWI / TU Munich), MIT licensed, at https://github.com/cwida/fsst, + * commit 89f49c580c6388acf3b6ed2a49e1bfde6c05e616. + * + *

Each counter is split into a low byte and a high part, and the pair counter's high part is only + * four bits wide, two counters to a byte. That is not only a space optimization to be simplified + * away: four bits saturate, so the split decides which candidate symbols survive a training round + * and therefore which table comes out. The reference implementation has a wider variant for 32-bit + * platforms that produces different tables; this follows the 64-bit variant, which is what other + * implementations run. + * + *

The high part is incremented when the low byte wraps from zero rather than when it saturates, + * which is what makes a nonzero high part equivalent to a nonzero count and lets the scans below + * skip runs of empty counters eight bytes at a time. + * + *

Those scans deliberately read past the counter they were asked about. The reference + * implementation lets the read spill into whichever array is laid out next; here each array carries + * eight bytes of zero padding instead. The two agree: a spilled read only changes the result if + * every valid counter it covers is zero, and in that case the scan has already advanced past the end + * of the range and returns zero either way. + */ +final class FsstCounters { + + /** Slack for scans that read eight bytes from the last counter. */ + private static final int PADDING = 8; + + private final byte[] count1High = new byte[CODE_MAX + PADDING]; + private final byte[] count1Low = new byte[CODE_MAX + PADDING]; + private final byte[] count2High = new byte[CODE_MAX * (CODE_MAX / 2) + PADDING]; + private final byte[] count2Low = new byte[CODE_MAX * CODE_MAX + PADDING]; + + /** Position reached by the most recent scan, which the caller must adopt. */ + private int scanPosition; + + void reset() { + Arrays.fill(count1High, (byte) 0); + Arrays.fill(count1Low, (byte) 0); + Arrays.fill(count2High, (byte) 0); + Arrays.fill(count2Low, (byte) 0); + } + + void count1Set(int position, int value) { + count1Low[position] = (byte) (value & 255); + count1High[position] = (byte) (value >>> 8); + } + + void count1Increment(int position) { + // Post-increment on a byte array wraps, and yields the value before the increment, so this + // raises the high part exactly when the low byte wraps back to zero. + if (count1Low[position]++ == 0) { + count1High[position]++; + } + } + + void count2Increment(int first, int second) { + if (count2Low[first * CODE_MAX + second]++ == 0) { + // Add one to the four-bit counter, in the low or the high nibble according to parity. + count2High[first * (CODE_MAX / 2) + (second >> 1)] += (byte) (1 << ((second & 1) << 2)); + } + } + + /** + * Reads the counter for a single symbol, skipping forward over empty counters. + * + * @return the count, or zero once the scan leaves the range. Either way the caller must adopt + * {@link #scanPosition()} as its new position. + */ + int count1Next(int position) { + long high = loadLittleEndianLong(count1High, position); + int zeroBytes = (high != 0) ? (Long.numberOfTrailingZeros(high) >>> 3) : 7; + high = (high >>> (zeroBytes << 3)) & 255; + position += zeroBytes; + scanPosition = position; + if (position >= CODE_MAX || high == 0) { + return 0; + } + int low = count1Low[position] & 0xFF; + if (low != 0) { + high--; // the high part was raised early + } + return (int) ((high << 8) + low); + } + + /** + * Reads the counter for a pair of symbols, skipping forward over empty counters. + * + * @return the count, or zero once the scan leaves the range. Either way the caller must adopt + * {@link #scanPosition()} as its new second position. + */ + int count2Next(int first, int second) { + long high = loadLittleEndianLong(count2High, first * (CODE_MAX / 2) + (second >> 1)); + high >>>= ((second & 1) << 2); // an odd position starts halfway into its byte + int zeroNibbles = (high != 0) ? (Long.numberOfTrailingZeros(high) >>> 2) : (15 - (second & 1)); + high = (high >>> (zeroNibbles << 2)) & 15; + second += zeroNibbles; + scanPosition = second; + if (second >= CODE_MAX || high == 0) { + return 0; + } + int low = count2Low[first * CODE_MAX + second] & 0xFF; + if (low != 0) { + high--; // the high part was raised early + } + return (int) ((high << 8) + low); + } + + int scanPosition() { + return scanPosition; + } + + /** Size of the buffer {@link #backupSingleCounts} needs. */ + static int backupSize() { + return 2 * CODE_MAX; + } + + /** Saves the single-symbol counters, so the best round's counts can be brought back. */ + void backupSingleCounts(byte[] buffer) { + System.arraycopy(count1High, 0, buffer, 0, CODE_MAX); + System.arraycopy(count1Low, 0, buffer, CODE_MAX, CODE_MAX); + } + + void restoreSingleCounts(byte[] buffer) { + System.arraycopy(buffer, 0, count1High, 0, CODE_MAX); + System.arraycopy(buffer, CODE_MAX, count1Low, 0, CODE_MAX); + } + + private static long loadLittleEndianLong(byte[] source, int offset) { + long value = 0; + for (int i = 0; i < 8; i++) { + value |= (long) (source[offset + i] & 0xFF) << (i * 8); + } + return value; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstSymbolTable.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstSymbolTable.java new file mode 100644 index 0000000000..1ea80548f3 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstSymbolTable.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_BASE; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_MASK; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_MAX; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.HASH_TAB_SIZE; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.ICL_FREE; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.LEN_BITS; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.MAX_SYMBOLS; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.MAX_SYMBOL_LENGTH; + +/** + * The mutable symbol table the FSST trainer builds and the compressor reads. + * + *

Java port of {@code SymbolTable} from the FSST reference implementation by Peter Boncz, + * Viktor Leis and Thomas Neumann (CWI / TU Munich), MIT licensed, at + * https://github.com/cwida/fsst, commit 89f49c580c6388acf3b6ed2a49e1bfde6c05e616. + * + *

Symbols are held in parallel {@code long} arrays rather than as objects. The reference + * implementation copies symbols by value throughout; parallel arrays reproduce that without the + * aliasing that an array of mutable objects would introduce, and keep the inner loops free of + * pointer chasing. The two code lookup tables use {@code char}, which is Java's only unsigned + * 16-bit type and so matches the reference implementation's {@code u16} without masking. + * + *

Two representations of a code appear here and must not be confused. The lookup tables hold + * {@code (length << 12) | code}; a symbol descriptor holds {@code (length << 28) | (code << 16) | + * ignoredBits}. + */ +final class FsstSymbolTable { + + /** Code for a 2-byte pattern, else the pseudo code for its escaped first byte. */ + final char[] shortCodes = new char[65536]; + + /** Code for a 1-byte symbol, else its escaped pseudo code. Not needed after {@link #finish}. */ + final char[] byteCodes = new char[256]; + + /** Symbol bytes, little-endian. Indices below {@link FsstCodes#CODE_BASE} are pseudo symbols. */ + final long[] symbolValues = new long[CODE_MAX]; + + /** Symbol descriptors, parallel to {@link #symbolValues}. */ + final long[] symbolDescriptors = new long[CODE_MAX]; + + /** Symbols of three bytes or more, replicated here to avoid an indirection. */ + final long[] hashValues = new long[HASH_TAB_SIZE]; + + final long[] hashDescriptors = new long[HASH_TAB_SIZE]; + + /** Count of symbols of each byte length, indexed by length minus one. */ + final int[] lengthHistogram = new int[FsstCodes.CODE_BITS]; + + int symbolCount; + + /** Codes at or above this value may have a longer suffix; only meaningful after {@link #finish}. */ + int suffixLimit = CODE_MAX; + + /** Codes at or above this value are one byte long; only meaningful after {@link #finish}. */ + int byteLimit; + + /** A 1-byte symbol usable as a separator during compression. */ + int terminator; + + FsstSymbolTable() { + for (int i = 0; i < 256; i++) { + // Pseudo symbols: one per byte value, standing for that byte escaped. + symbolValues[i] = i; + symbolDescriptors[i] = FsstCodes.icl(i | (1 << LEN_BITS), 1); + } + long unused = FsstCodes.icl(CODE_MASK, 1); + for (int i = 256; i < CODE_MAX; i++) { + symbolValues[i] = 0; + symbolDescriptors[i] = unused; + } + for (int i = 0; i < HASH_TAB_SIZE; i++) { + hashDescriptors[i] = ICL_FREE; + } + for (int i = 0; i < 256; i++) { + byteCodes[i] = (char) ((1 << LEN_BITS) | i); + } + for (int i = 0; i < 65536; i++) { + shortCodes[i] = (char) ((1 << LEN_BITS) | (i & 255)); + } + } + + /** Empties the table, touching only the positions that were used. */ + void clear() { + java.util.Arrays.fill(lengthHistogram, 0); + for (int i = CODE_BASE; i < CODE_BASE + symbolCount; i++) { + long val = symbolValues[i]; + int length = FsstCodes.length(symbolDescriptors[i]); + if (length == 1) { + int b = FsstCodes.first(val); + byteCodes[b] = (char) ((1 << LEN_BITS) | b); + } else if (length == 2) { + int b2 = FsstCodes.first2(val); + shortCodes[b2] = (char) ((1 << LEN_BITS) | (b2 & 255)); + } else { + int idx = FsstCodes.hashBucket(val); + hashValues[idx] = 0; + hashDescriptors[idx] = ICL_FREE; + } + } + symbolCount = 0; + } + + private boolean hashInsert(long val, long icl) { + int idx = FsstCodes.hashBucket(val); + if (hashDescriptors[idx] < ICL_FREE) { + return false; // bucket taken + } + hashDescriptors[idx] = icl; + hashValues[idx] = FsstCodes.maskWord(val, icl); + return true; + } + + /** Copies another table over this one, so the best round's table can be kept aside. */ + void copyFrom(FsstSymbolTable other) { + System.arraycopy(other.shortCodes, 0, shortCodes, 0, shortCodes.length); + System.arraycopy(other.byteCodes, 0, byteCodes, 0, byteCodes.length); + System.arraycopy(other.symbolValues, 0, symbolValues, 0, symbolValues.length); + System.arraycopy(other.symbolDescriptors, 0, symbolDescriptors, 0, symbolDescriptors.length); + System.arraycopy(other.hashValues, 0, hashValues, 0, hashValues.length); + System.arraycopy(other.hashDescriptors, 0, hashDescriptors, 0, hashDescriptors.length); + System.arraycopy(other.lengthHistogram, 0, lengthHistogram, 0, lengthHistogram.length); + symbolCount = other.symbolCount; + suffixLimit = other.suffixLimit; + byteLimit = other.byteLimit; + terminator = other.terminator; + } + + /** Adds a symbol, returning false when its hash bucket is already taken. */ + boolean add(long val, long icl) { + int length = FsstCodes.length(icl); + int code = CODE_BASE + symbolCount; + long descriptor = FsstCodes.icl(code, length); + if (length == 1) { + byteCodes[FsstCodes.first(val)] = (char) (code + (1 << LEN_BITS)); + } else if (length == 2) { + shortCodes[FsstCodes.first2(val)] = (char) (code + (2 << LEN_BITS)); + } else if (!hashInsert(val, descriptor)) { + return false; + } + symbolValues[code] = val; + symbolDescriptors[code] = descriptor; + symbolCount++; + lengthHistogram[length - 1]++; + return true; + } + + /** Returns the code of the longest symbol matching the input at {@code position}. */ + int findLongestSymbol(byte[] input, int position, int end) { + int length = Math.min(MAX_SYMBOL_LENGTH, end - position); + long val = FsstCodes.loadSymbolBytes(input, position, length); + long icl = FsstCodes.icl(CODE_MAX, length); + int idx = FsstCodes.hashBucket(val); + if (hashDescriptors[idx] <= icl && hashValues[idx] == FsstCodes.maskWord(val, hashDescriptors[idx])) { + return FsstCodes.code(hashDescriptors[idx]); + } + if (length >= 2) { + int code = shortCodes[FsstCodes.first2(val)] & CODE_MASK; + if (code >= CODE_BASE) { + return code; + } + } + return byteCodes[FsstCodes.first(val)] & CODE_MASK; + } + + /** + * Renumbers codes into a single byte each and groups symbols by length. + * + *

Named {@code finalize} in the reference implementation; renamed here because that name is + * reserved on {@link Object}. Afterwards real codes occupy {@code [0, symbolCount)} grouped by + * length as 2,3,4,5,6,7,8 then 1, two-byte symbols with no longer suffix come first so the + * compressor can take a shortcut, escapes in {@link #shortCodes} carry the eighth bit, and + * {@link #byteCodes} is folded into {@link #shortCodes} so the compressor never consults it. + * + *

The reference implementation also supports zero-terminated input, which Parquet never + * produces because values carry an explicit length. That mode is fixed off here, so terms that + * depend on it fall away. + */ + void finish() { + if (symbolCount > MAX_SYMBOLS) { + throw new IllegalStateException("FSST symbol table holds " + symbolCount + " symbols, at most " + + MAX_SYMBOLS + " can be renumbered into one byte each"); + } + int[] newCode = new int[256]; + int[] runningSum = new int[8]; + byteLimit = symbolCount - lengthHistogram[0]; + + runningSum[0] = byteLimit; // 1-byte codes sort highest + runningSum[1] = 0; + for (int i = 1; i < 7; i++) { + runningSum[i + 1] = runningSum[i] + lengthHistogram[i]; + } + + suffixLimit = runningSum[1]; + newCode[0] = 0; + symbolValues[0] = symbolValues[CODE_BASE]; + symbolDescriptors[0] = symbolDescriptors[CODE_BASE]; + + for (int i = 0, j = runningSum[2]; i < symbolCount; i++) { + long val = symbolValues[CODE_BASE + i]; + int length = FsstCodes.length(symbolDescriptors[CODE_BASE + i]); + // For 2-byte symbols, scan for a longer symbol sharing the same first two bytes. The scan + // bound doubles as the answer: clearing it both records the find and ends the loop, exactly + // as the reference implementation does. + int scan = (length == 2) ? symbolCount : 0; + if (scan != 0) { + int first2 = FsstCodes.first2(val); + for (int k = 0; k < scan; k++) { + long otherIcl = symbolDescriptors[CODE_BASE + k]; + if (k != i + && FsstCodes.length(otherIcl) > 1 + && first2 == FsstCodes.first2(symbolValues[CODE_BASE + k])) { + scan = 0; + } + } + newCode[i] = (scan != 0) ? suffixLimit++ : --j; + } else { + newCode[i] = runningSum[length - 1]++; + } + symbolValues[newCode[i]] = val; + symbolDescriptors[newCode[i]] = FsstCodes.icl(newCode[i], length); + } + + for (int i = 0; i < 256; i++) { + if ((byteCodes[i] & CODE_MASK) >= CODE_BASE) { + byteCodes[i] = (char) (newCode[byteCodes[i] & 0xFF] + (1 << LEN_BITS)); + } else { + byteCodes[i] = (char) (511 + (1 << LEN_BITS)); + } + } + + for (int i = 0; i < 65536; i++) { + if ((shortCodes[i] & CODE_MASK) >= CODE_BASE) { + shortCodes[i] = (char) (newCode[shortCodes[i] & 0xFF] + (shortCodes[i] & (15 << LEN_BITS))); + } else { + shortCodes[i] = byteCodes[i & 0xFF]; + } + } + + for (int i = 0; i < HASH_TAB_SIZE; i++) { + if (hashDescriptors[i] < ICL_FREE) { + int renumbered = newCode[FsstCodes.code(hashDescriptors[i]) & 0xFF]; + hashValues[i] = symbolValues[renumbered]; + hashDescriptors[i] = symbolDescriptors[renumbered]; + } + } + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstTrainer.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstTrainer.java new file mode 100644 index 0000000000..1bde3d3dc5 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/fsst/FsstTrainer.java @@ -0,0 +1,410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_BASE; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_MASK; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.CODE_MAX; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.ICL_FREE; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.MAX_SYMBOLS; +import static org.apache.parquet.column.values.symboltable.fsst.FsstCodes.MAX_SYMBOL_LENGTH; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.column.values.symboltable.SymbolTableTrainer; +import org.apache.parquet.column.values.symboltable.SymbolTableType; +import org.apache.parquet.column.values.symboltable.TrainedSymbolTable; +import org.apache.parquet.column.values.symboltable.ValueBuffer; + +/** + * Chooses an FSST symbol table for a set of values. + * + *

Java port of the training half of the FSST reference implementation by Peter Boncz, Viktor Leis + * and Thomas Neumann (CWI / TU Munich), MIT licensed, at https://github.com/cwida/fsst, commit + * 89f49c580c6388acf3b6ed2a49e1bfde6c05e616. The algorithm is reproduced rather than reinterpreted: + * a writer that mines symbols differently still produces readable files, but it produces different + * tables, and then the compression ratio can no longer be compared against another implementation + * to tell a correct port from a subtly broken one. + * + *

Training runs five rounds over a sample of the data. Each round compresses the sample with the + * table built so far, counting how often each symbol occurs and how often each ordered pair of + * symbols occurs back to back; frequent pairs are then concatenated into candidate symbols, scored + * by how many bytes they would save, and the best 255 become the next round's table. Rounds use a + * growing fraction of the sample, and the table kept at the end is the round that scored best rather + * than the last one. + * + *

One departure is deliberate and is documented at {@link #CANDIDATE_ORDER}. + */ +public class FsstTrainer implements SymbolTableTrainer { + + /** Bytes of sample the trainer aims for. */ + static final int SAMPLE_TARGET = 1 << 14; + + /** Worst-case sample size, which also bounds the score of a round. */ + static final int SAMPLE_MAX_SIZE = 2 * SAMPLE_TARGET; + + /** Length of one run of bytes taken from a value into the sample. */ + static final int SAMPLE_LINE = 512; + + /** Seed of the sampler's hash chain, from the reference implementation. */ + private static final long SAMPLE_SEED = 4637947; + + /** + * Order candidates are offered to the table in: best score first, and among equal scores the + * numerically smaller symbol, then the shorter one. + * + *

The reference implementation collects candidates in a hash set and drains them through a + * heap, so when two candidates tie on both score and numeric value the winner depends on the hash + * set's iteration order, which its own standard library does not fix. That last tie is possible, + * because a shorter symbol whose trailing bytes are zero has the same numeric value as a longer + * one. Breaking it on length makes this port's output a function of the input alone, which is what + * lets a table be compared against another implementation's at all. Any table this produces is + * readable by any reader; only the tie-breaking rule would have to be agreed to make two writers + * agree byte for byte. + */ + private static final Comparator CANDIDATE_ORDER = Comparator.comparingLong( + (Candidate candidate) -> candidate.gain) + .reversed() + .thenComparing((left, right) -> Long.compareUnsigned(left.value, right.value)) + .thenComparingInt(candidate -> candidate.length); + + private final FsstCounters counters = new FsstCounters(); + private final byte[] bestCounters = new byte[FsstCounters.backupSize()]; + private final Map candidates = new HashMap<>(); + private final ValueBuffer sample = new ValueBuffer(SAMPLE_MAX_SIZE, SAMPLE_MAX_SIZE / SAMPLE_LINE); + + /** Fraction of the sample, out of 128, that the current round compresses. */ + private int sampleFraction; + + @Override + public SymbolTableType type() { + return SymbolTableType.FSST_8; + } + + @Override + public TrainedSymbolTable train(ValueBuffer values) { + FsstSymbolTable table = buildSymbolTable(makeSample(values)); + Fsst8SymbolTable fileTable = Fsst8SymbolTable.of(table); + return new TrainedSymbolTable(fileTable, new Fsst8CodeStreamEncoder(table, fileTable)); + } + + /** + * Draws a sample of at least {@link #SAMPLE_TARGET} bytes as runs taken from randomly chosen + * values, or returns the values themselves when there are fewer bytes than that. + * + *

The choice of runs is driven by a chain of the same hash the symbol lookup uses, seeded by a + * constant, so it is already reproducible and is ported as it stands. Replacing it with a + * different sampler, however reasonable, would change the table for the same input. + */ + private ValueBuffer makeSample(ValueBuffer values) { + int valueCount = values.valueCount(); + if (valueCount == 0 || values.byteCount() < SAMPLE_TARGET) { + return values; + } + sample.reset(); + long random = FsstCodes.hash(SAMPLE_SEED); + int lineLimit = valueCount + SAMPLE_MAX_SIZE / SAMPLE_LINE; + while (sample.byteCount() < SAMPLE_TARGET && sample.valueCount() < lineLimit) { + // Choose a value, skipping forward over empty ones. + random = FsstCodes.hash(random); + int index = (int) Long.remainderUnsigned(random, valueCount); + while (values.length(index) == 0) { + if (++index == valueCount) { + index = 0; + } + } + // Choose one of its runs. + int runCount = 1 + ((values.length(index) - 1) / SAMPLE_LINE); + random = FsstCodes.hash(random); + int runStart = SAMPLE_LINE * (int) Long.remainderUnsigned(random, runCount); + int runLength = Math.min(values.length(index) - runStart, SAMPLE_LINE); + sample.add(values.data(), values.offset(index) + runStart, runLength); + } + return sample; + } + + private FsstSymbolTable buildSymbolTable(ValueBuffer sampleValues) { + FsstSymbolTable table = new FsstSymbolTable(); + FsstSymbolTable best = new FsstSymbolTable(); + int bestGain = -SAMPLE_MAX_SIZE; // the score if every byte had to be escaped + table.terminator = chooseTerminator(sampleValues); + + for (sampleFraction = 8; ; sampleFraction += 30) { + counters.reset(); + int gain = compressCount(table, sampleValues); + if (gain >= bestGain) { + counters.backupSingleCounts(bestCounters); + best.copyFrom(table); + bestGain = gain; + } + if (sampleFraction >= 128) { + break; // five rounds, at fractions 8, 38, 68, 98 and 128 + } + makeTable(table); + } + + // Rebuild the best round's table from the counts that round produced. The fraction is 128 here, + // so this pass only re-ranks the symbols it already has and creates no new ones. + counters.restoreSingleCounts(bestCounters); + makeTable(best); + best.finish(); + return best; + } + + /** + * Picks the least frequent byte as the terminator, preferring the lowest such byte. + * + *

The terminator is appended to each run the compressor works on, which is what lets the + * compressor read eight bytes at a time without a bounds check: a symbol containing the + * terminator is never in the table, so a match cannot run past the end of the value. + */ + private static int chooseTerminator(ValueBuffer values) { + int[] byteHistogram = new int[256]; + byte[] data = values.data(); + for (int i = 0; i < values.valueCount(); i++) { + int end = values.offset(i) + values.length(i); + for (int position = values.offset(i); position < end; position++) { + byteHistogram[data[position] & 0xFF]++; + } + } + int terminator = 256; + int minimum = SAMPLE_MAX_SIZE; + for (int i = 255; i >= 0; i--) { + if (byteHistogram[i] > minimum) { + continue; + } + terminator = i; + minimum = byteHistogram[i]; + } + return terminator; + } + + /** + * Compresses the sample with the table as it stands, counting symbols and symbol pairs, and + * returns the number of bytes the table saves over escaping everything. + */ + private int compressCount(FsstSymbolTable table, ValueBuffer values) { + int gain = 0; + for (int index = 0; index < values.valueCount(); index++) { + int position = values.offset(index); + int end = position + values.length(index); + if (sampleFraction < 128 && randomFraction(index) > sampleFraction) { + continue; // earlier rounds skip most of the sample, which roughly halves the work + } + if (position >= end) { + continue; + } + byte[] data = values.data(); + int start = position; + int code1 = table.findLongestSymbol(data, position, end); + position += FsstCodes.length(table.symbolDescriptors[code1]); + gain += FsstCodes.length(table.symbolDescriptors[code1]) - (1 + escapeCost(code1)); + while (true) { + // Count the symbol as it stands, that is, the option of not extending it. + counters.count1Increment(code1); + // As an alternative, count just its first byte, unless that is the same thing. + if (FsstCodes.length(table.symbolDescriptors[code1]) != 1) { + counters.count1Increment(data[start] & 0xFF); + } + if (position == end) { + break; + } + + start = position; + int code2; + if (position < end - 7) { + // Eight bytes are available, so the three lookups can be done without bounds checks. + long word = FsstCodes.loadSymbolBytes(data, position, MAX_SYMBOL_LENGTH); + int bucket = FsstCodes.hashBucket(word); + long bucketDescriptor = table.hashDescriptors[bucket]; + code2 = table.shortCodes[FsstCodes.first2(word)] & CODE_MASK; + word = FsstCodes.maskWord(word, bucketDescriptor); + if (bucketDescriptor < ICL_FREE && table.hashValues[bucket] == word) { + code2 = FsstCodes.code(bucketDescriptor); + position += FsstCodes.length(bucketDescriptor); + } else if (code2 >= CODE_BASE) { + position += 2; + } else { + code2 = table.byteCodes[FsstCodes.first(word)] & CODE_MASK; + position += 1; + } + } else { + code2 = table.findLongestSymbol(data, position, end); + position += FsstCodes.length(table.symbolDescriptors[code2]); + } + + gain += (position - start) - (1 + escapeCost(code2)); + + if (sampleFraction < 128) { // the last round does not need pair counts + // Count the pair, that is, the option of concatenating the two symbols. + counters.count2Increment(code1, code2); + // As an alternative, count extending by just the next byte, unless that is the same thing. + if ((position - start) > 1) { + counters.count2Increment(code1, data[start] & 0xFF); + } + } + code1 = code2; + } + } + return gain; + } + + /** A value between 1 and 128, fixed for a given value index and round. */ + private int randomFraction(int index) { + return 1 + (int) (FsstCodes.hash((long) (index + 1) * sampleFraction) & 127); + } + + /** Cost in bytes a code adds beyond the one byte it always costs: one more if it escapes. */ + private static int escapeCost(int code) { + return code < CODE_BASE ? 1 : 0; + } + + /** + * Replaces the table with the best-scoring candidates from the counts just gathered. + * + *

Candidates are the symbols already in the table, the concatenation of each counted pair, and + * each symbol extended by one byte. Single-byte symbols are scored eight times higher than their + * frequency warrants, which the reference implementation notes both lowers the escape rate and + * speeds up compression and decompression. + */ + private void makeTable(FsstSymbolTable table) { + candidates.clear(); + + // Force the terminator into the table by making it look like the most frequent symbol. + int terminatorPosition = table.symbolCount != 0 ? CODE_BASE : table.terminator; + counters.count1Set(terminatorPosition, 65535); + + int positionLimit = CODE_BASE + table.symbolCount; + for (int pos1 = 0; pos1 < positionLimit; pos1++) { + int count1 = counters.count1Next(pos1); + pos1 = counters.scanPosition(); // the scan skips empty counters + if (count1 == 0) { + continue; + } + long value1 = table.symbolValues[pos1]; + int length1 = FsstCodes.length(table.symbolDescriptors[pos1]); + addOrIncrement(value1, length1, (length1 == 1 ? 8L : 1L) * count1); + + if (sampleFraction >= 128 // the last round does not create new symbols + || length1 == MAX_SYMBOL_LENGTH // this symbol cannot be extended + || FsstCodes.first(value1) == table.terminator) { // and none may contain the terminator + continue; + } + for (int pos2 = 0; pos2 < positionLimit; pos2++) { + int count2 = counters.count2Next(pos1, pos2); + pos2 = counters.scanPosition(); + if (count2 == 0) { + continue; + } + long value2 = table.symbolValues[pos2]; + if (FsstCodes.first(value2) != table.terminator) { + int length2 = FsstCodes.length(table.symbolDescriptors[pos2]); + addOrIncrement(concatenate(value1, length1, value2), concatenatedLength(length1, length2), count2); + } + } + } + + List ranked = new ArrayList<>(candidates.values()); + ranked.sort(CANDIDATE_ORDER); + table.clear(); + for (Candidate candidate : ranked) { + if (table.symbolCount >= MAX_SYMBOLS) { + break; + } + // A candidate whose hash bucket is taken is dropped rather than retried, as upstream does. + table.add(candidate.value, FsstCodes.icl(CODE_MASK, candidate.length)); + } + } + + /** + * Records a candidate symbol, or adds to the score of one already recorded. + * + *

Rare candidates are dropped, on a threshold that grows with the round. Upstream notes this + * improves the compression ratio as well as the speed of training. + */ + private void addOrIncrement(long value, int length, long count) { + if (count < (5L * sampleFraction) / 128) { + return; + } + Candidate candidate = new Candidate(value, length); + Candidate existing = candidates.get(candidate); + if (existing != null) { + existing.gain += count * length; + } else { + candidate.gain = count * length; + candidates.put(candidate, candidate); + } + } + + /** + * Joins two symbols into one, truncated to the maximum symbol length. + * + *

The shift is safe because a symbol already at the maximum length is never extended, so + * {@code firstLength} is at most seven here. That matters more in Java than in C: a shift count of + * 64 would be taken modulo 64 and quietly leave the value unshifted. + */ + private static long concatenate(long first, int firstLength, long second) { + return (second << (8 * firstLength)) | first; + } + + private static int concatenatedLength(int firstLength, int secondLength) { + return Math.min(firstLength + secondLength, MAX_SYMBOL_LENGTH); + } + + /** + * A symbol under consideration and the bytes it would save. + * + *

Identity is the symbol, not the score, so that repeated proposals of the same symbol + * accumulate. Hashing on the value alone matches upstream and stays consistent with equality. + */ + private static final class Candidate { + private final long value; + private final int length; + private long gain; + + Candidate(long value, int length) { + this.value = value; + this.length = length; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Candidate)) { + return false; + } + Candidate that = (Candidate) other; + return value == that.value && length == that.length; + } + + @Override + public int hashCode() { + return Long.hashCode(value); + } + } + + static { + // Guards the assumption the position arithmetic in makeTable relies on. + if (CODE_BASE + MAX_SYMBOLS >= CODE_MAX) { + throw new AssertionError("symbol positions must stay below " + CODE_MAX); + } + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java new file mode 100644 index 0000000000..47ad302aa9 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.parquet.column.values.symboltable.CodeStreamDecoder; +import org.apache.parquet.column.values.symboltable.CodeStreamEncoder; +import org.apache.parquet.column.values.symboltable.SymbolTable; +import org.apache.parquet.column.values.symboltable.SymbolTableType; +import org.apache.parquet.column.values.symboltable.TrainedSymbolTable; +import org.apache.parquet.column.values.symboltable.ValueBuffer; +import org.junit.Test; + +/** + * Round trips the FSST codec over inputs chosen to reach the places the port could be wrong: the + * chunk boundary, the escape path, the byte values a signed Java byte gets wrong, and a table read + * back from its serialized form rather than the one training produced. + */ +public class FsstCodecRoundTripTest { + + /** Trains on the values, compresses each one, expands it again and requires the bytes back. */ + private static Result roundTrip(List values) throws IOException { + ValueBuffer buffer = new ValueBuffer(); + for (byte[] value : values) { + buffer.add(value, 0, value.length); + } + TrainedSymbolTable trained = new FsstTrainer().train(buffer); + SymbolTable table = trained.table(); + CodeStreamEncoder encoder = trained.encoder(); + + // Decode through a table read back from bytes, not the one training produced, so the + // serialization and the renumbering are both under test. + byte[] serialized = table.serialize().toByteArray(); + SymbolTable reread = Fsst8SymbolTable.deserialize(serialized, 0, serialized.length); + assertEquals(SymbolTableType.FSST_8, reread.type()); + assertEquals(table.symbolCount(), reread.symbolCount()); + assertArrayEquals(serialized, reread.serialize().toByteArray()); + CodeStreamDecoder decoder = reread.decoder(); + + int codeBytes = 0; + int rawBytes = 0; + for (int i = 0; i < values.size(); i++) { + byte[] value = values.get(i); + byte[] codes = new byte[encoder.maxCompressedLength(value.length)]; + int codeLength = encoder.compress(buffer.data(), buffer.offset(i), buffer.length(i), codes, 0); + assertTrue("compressed past the declared bound", codeLength <= codes.length); + + assertEquals( + "expandedLength disagrees with expand on value " + i, + value.length, + decoder.expandedLength(codes, 0, codeLength)); + byte[] expanded = new byte[value.length]; + int written = decoder.expand(codes, 0, codeLength, expanded, 0); + assertEquals("wrong expanded length for value " + i, value.length, written); + assertArrayEquals("value " + i + " did not survive the round trip", value, expanded); + + codeBytes += codeLength; + rawBytes += value.length; + } + return new Result(table, rawBytes, codeBytes); + } + + private static final class Result { + final SymbolTable table; + final int rawBytes; + final int codeBytes; + + Result(SymbolTable table, int rawBytes, int codeBytes) { + this.table = table; + this.rawBytes = rawBytes; + this.codeBytes = codeBytes; + } + } + + private static List strings(String... values) { + List result = new ArrayList<>(); + for (String value : values) { + result.add(value.getBytes(StandardCharsets.UTF_8)); + } + return result; + } + + @Test + public void compressesRepetitiveTextAndGetsItBack() throws IOException { + List values = new ArrayList<>(); + for (int i = 0; i < 2000; i++) { + values.add(("https://www.example.com/products/widget-" + i + "/reviews?page=" + (i % 7)) + .getBytes(StandardCharsets.UTF_8)); + } + Result result = roundTrip(values); + assertTrue("the table should hold symbols", result.table.symbolCount() > 0); + // A corpus this repetitive is the case the encoding exists for; if it does not shrink here the + // trainer is not finding the shared substrings. + assertTrue( + "expected well under half the bytes, got " + result.codeBytes + " of " + result.rawBytes, + result.codeBytes * 2 < result.rawBytes); + } + + @Test + public void handlesEveryByteValue() throws IOException { + List values = new ArrayList<>(); + for (int i = 0; i < 256; i++) { + byte[] value = new byte[16]; + for (int j = 0; j < value.length; j++) { + value[j] = (byte) ((i + j) & 0xFF); + } + values.add(value); + } + roundTrip(values); + } + + @Test + public void handlesValuesLongerThanOneChunk() throws IOException { + // 511 bytes is the chunk the compressor works in, so a value has to be driven across it. + List values = new ArrayList<>(); + for (int length : new int[] {509, 510, 511, 512, 513, 1021, 1022, 1023, 4096}) { + StringBuilder builder = new StringBuilder(); + while (builder.length() < length) { + builder.append("the quick brown fox jumps over the lazy dog "); + } + values.add(builder.substring(0, length).getBytes(StandardCharsets.UTF_8)); + } + roundTrip(values); + } + + @Test + public void handlesIncompressibleBytes() throws IOException { + // Random bytes give the trainer nothing to work with, so nearly every byte escapes. + Random random = new Random(20260908); + List values = new ArrayList<>(); + for (int i = 0; i < 500; i++) { + byte[] value = new byte[1 + random.nextInt(64)]; + random.nextBytes(value); + values.add(value); + } + Result result = roundTrip(values); + assertTrue("escaping should cost bytes, not save them", result.codeBytes >= result.rawBytes); + } + + @Test + public void handlesEmptyAndTinyInputs() throws IOException { + roundTrip(new ArrayList<>()); + roundTrip(strings("")); + roundTrip(strings("a")); + roundTrip(strings("", "", "")); + roundTrip(strings("a", "", "bb", "", "ccc")); + } + + @Test + public void handlesAValueMadeOnlyOfOneRepeatedByte() throws IOException { + List values = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + byte[] value = new byte[300]; + java.util.Arrays.fill(value, (byte) 0); + values.add(value); + } + roundTrip(values); + } + + @Test + public void trainsOnACorpusLargerThanTheSample() throws IOException { + // Past 16 KiB the trainer samples rather than reading everything, which is a different path. + Random random = new Random(4637947); + String[] words = {"alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"}; + List values = new ArrayList<>(); + for (int i = 0; i < 5000; i++) { + StringBuilder builder = new StringBuilder(); + for (int j = 0; j < 8; j++) { + builder.append(words[random.nextInt(words.length)]).append(' '); + } + values.add(builder.toString().getBytes(StandardCharsets.UTF_8)); + } + Result result = roundTrip(values); + assertTrue( + "expected well under half the bytes, got " + result.codeBytes + " of " + result.rawBytes, + result.codeBytes * 2 < result.rawBytes); + } + + @Test + public void symbolTableStaysWithinItsFormatLimits() throws IOException { + Random random = new Random(1); + List values = new ArrayList<>(); + for (int i = 0; i < 3000; i++) { + byte[] value = new byte[32]; + for (int j = 0; j < value.length; j++) { + value[j] = (byte) ('a' + random.nextInt(26)); + } + values.add(value); + } + Result result = roundTrip(values); + assertTrue(result.table.symbolCount() <= FsstCodes.MAX_SYMBOLS); + for (int code = 0; code < result.table.symbolCount(); code++) { + int length = result.table.symbolLength(code); + assertTrue("symbol " + code + " has length " + length, length >= 1 && length <= 8); + } + // The serialized table must be in length order, which is what lets a reader rebuild it from the + // length histogram alone. + for (int code = 1; code < result.table.symbolCount(); code++) { + assertTrue( + "symbols are not in length order at code " + code, + result.table.symbolLength(code - 1) <= result.table.symbolLength(code)); + } + int size = (int) result.table.serialize().size(); + assertTrue("serialized table is " + size + " bytes", size >= 9 && size <= 2049); + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java new file mode 100644 index 0000000000..1c3d632d03 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.fsst; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import org.apache.parquet.column.values.symboltable.CodeStreamEncoder; +import org.apache.parquet.column.values.symboltable.TrainedSymbolTable; +import org.apache.parquet.column.values.symboltable.ValueBuffer; +import org.junit.Test; + +/** + * Requires this implementation to produce the same symbol table and the same code bytes as the + * reference implementation, for corpora chosen to reach the places a port can drift. + * + *

This is the test that makes the port trustworthy. A round trip only proves the decoder undoes + * whatever the encoder did, so a trainer that picks worse symbols, or numbers them differently, + * passes it while writing files no other implementation would have written. Comparing against bytes + * the reference produced catches both. + * + *

The fixtures under {@code src/test/resources/fsst} hold, per corpus, the serialized symbol table + * and the concatenated code bytes, produced by the reference implementation. The corpora themselves + * are not stored: they are generated here from the arithmetic below, which is simple enough to + * restate in any language, and each one's digest is asserted so a drifting generator fails as a + * generator rather than as a codec. + */ +public class FsstReferenceComparisonTest { + + /** + * The generator both sides use, spelled out rather than taken from a library so that regenerating + * the fixtures from another language gives the same bytes. + */ + private static final class Lcg { + private int state; + + Lcg(int seed) { + this.state = seed & 0x7FFFFFFF; + } + + int next() { + state = (state * 1103515245 + 12345) & 0x7FFFFFFF; + return state; + } + + int below(int bound) { + return next() % bound; + } + + int nextByte() { + return (next() >> 16) & 0xFF; + } + } + + private static final String[] WORDS = { + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", + "hotel", "india", "juliet", "kilo", "lima", "mike", "november" + }; + + /** Ordinary text with heavy shared structure: what the encoding is for. */ + private static List urls() { + List values = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + values.add(ascii("https://www.example.com/products/widget-" + i + "/reviews?page=" + (i % 7))); + } + return values; + } + + /** Past the sample target, so the trainer samples instead of reading everything. */ + private static List sampled() { + Lcg random = new Lcg(1); + List values = new ArrayList<>(); + for (int i = 0; i < 1200; i++) { + StringBuilder builder = new StringBuilder(); + for (int j = 0; j < 9; j++) { + if (j > 0) { + builder.append(' '); + } + builder.append(WORDS[random.below(WORDS.length)]); + } + values.add(ascii(builder + " id=" + i)); + } + return values; + } + + /** Nothing to compress, so nearly every byte escapes and the table saturates. */ + private static List binary() { + Lcg random = new Lcg(2); + List values = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + // The length is drawn before any of the bytes, which is the order the fixtures were made in. + int length = 1 + random.below(64); + byte[] value = new byte[length]; + for (int j = 0; j < length; j++) { + value[j] = (byte) random.nextByte(); + } + values.add(value); + } + return values; + } + + /** Every byte value, including the ones a signed Java byte gets wrong. */ + private static List allBytes() { + List values = new ArrayList<>(); + for (int i = 0; i < 256; i++) { + byte[] value = new byte[16]; + for (int j = 0; j < value.length; j++) { + value[j] = (byte) ((i + j) & 0xFF); + } + values.add(value); + } + return values; + } + + /** Values straddling the 511-byte chunk the compressor works in. */ + private static List chunks() { + String base = "the quick brown fox jumps over the lazy dog "; + List values = new ArrayList<>(); + for (int length : new int[] {509, 510, 511, 512, 513, 1021, 1022, 1023, 4096}) { + StringBuilder builder = new StringBuilder(); + while (builder.length() < length) { + builder.append(base); + } + values.add(ascii(builder.substring(0, length))); + } + return values; + } + + private static List empties() { + List values = new ArrayList<>(); + for (String value : new String[] {"", "a", "", "bb", "", "ccc", ""}) { + values.add(ascii(value)); + } + return values; + } + + private static byte[] ascii(String value) { + return value.getBytes(StandardCharsets.ISO_8859_1); + } + + @Test + public void matchesTheReferenceOnSharedStructure() throws IOException { + check("urls", urls(), "f3f5706428f9100812a29c36d865faf7298bd84b0bdec2d4090f1a5aee0f2f7d"); + } + + @Test + public void matchesTheReferenceWhenTheTrainerSamples() throws IOException { + check("sampled", sampled(), "f1517b97140990c898ce71a9acf144c4fa670afb4e391d4682972710965e6f64"); + } + + @Test + public void matchesTheReferenceOnIncompressibleBytes() throws IOException { + check("binary", binary(), "3b1a118902ff50b0359663084ac76190d0e865c6ce3774c5085187ebebdd38a6"); + } + + @Test + public void matchesTheReferenceOnEveryByteValue() throws IOException { + check("allbytes", allBytes(), "db4288e84084c52f1dbb79b88715529fa50c4ddd4159307c562c1f93e8521d0d"); + } + + @Test + public void matchesTheReferenceAcrossTheChunkBoundary() throws IOException { + check("chunks", chunks(), "9163babc5f91c5bd2564a64ee6f40f0968595fc93e66e89db8001e8636f9e083"); + } + + @Test + public void matchesTheReferenceOnEmptyAndTinyValues() throws IOException { + check("empties", empties(), "807d87de83260feea2276cabc85fe028f9c35d439a05d6eb50689e4873c8945b"); + } + + private void check(String corpus, List values, String expectedDigest) throws IOException { + ValueBuffer buffer = new ValueBuffer(); + for (byte[] value : values) { + buffer.add(value, 0, value.length); + } + assertEquals( + corpus + ": the generated corpus does not match the one the fixtures were made from", + expectedDigest, + sha256(buffer.data(), buffer.byteCount())); + + TrainedSymbolTable trained = new FsstTrainer().train(buffer); + assertArrayEquals( + corpus + ": symbol table differs from the reference implementation", + resource(corpus + ".table"), + trained.table().serialize().toByteArray()); + + CodeStreamEncoder encoder = trained.encoder(); + ByteArrayOutputStream codes = new ByteArrayOutputStream(); + for (int i = 0; i < values.size(); i++) { + byte[] output = new byte[encoder.maxCompressedLength(buffer.length(i))]; + int length = encoder.compress(buffer.data(), buffer.offset(i), buffer.length(i), output, 0); + codes.write(output, 0, length); + } + assertArrayEquals( + corpus + ": code bytes differ from the reference implementation", + resource(corpus + ".codes"), + codes.toByteArray()); + } + + private static byte[] resource(String name) throws IOException { + try (InputStream in = FsstReferenceComparisonTest.class.getResourceAsStream("/fsst/" + name)) { + if (in == null) { + throw new IOException("missing test resource /fsst/" + name); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + int read; + while ((read = in.read(chunk)) > 0) { + out.write(chunk, 0, read); + } + return out.toByteArray(); + } + } + + private static String sha256(byte[] data, int length) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + digest.update(data, 0, length); + StringBuilder hex = new StringBuilder(); + for (byte b : digest.digest()) { + hex.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } +} diff --git a/parquet-column/src/test/resources/fsst/allbytes.codes b/parquet-column/src/test/resources/fsst/allbytes.codes new file mode 100644 index 0000000000000000000000000000000000000000..58c5ac43f4828e32c7637c5226ebc9ed516afb29 GIT binary patch literal 3648 zcmZXXd0@|F9LI-7ZBtEQs%gqK6@IH6Ta-!0)G!sc$_@%sIVNEg#VQqP2va$x!c>ka z$5uHywA6lSu9-}xC|AkGzCEAk^BkYg(?7rO`}KUT*T27j%l->r|Gf7X^(o!A|G>dR zUOjv)Jh{f70x zvcnFR^J@7K+gxSFs_NsC*{?)2aq-)9! zmmm2RRs{tEUh7~euZA{Y&7lphqq739C#wLD+szWsm_vx3PahWiuX2F+4eH3)P?}+E z1d=K?CNa#uhBYw_VNGt(y&~QSdmA=t4Dlus!|iK$)8l76pBH7?!kgawzlR97Bcd6f zh-h{TT_EsQeOBxPY_2Rcwm>O^Ev2KyR-jCwsk5MO(MeMmX=tb`_*%M`lb6DZUW%7Wn}NMFJAF}I?9%n( zcX(T*f20J2`@lwqZYIR)m)P!JvMR=Qzu%@FI3V77JODeo$<))PIJEH6O)ah$Xt9Uh zF30n7d>=C1JHF3@-Y|VBt{3n_4kjoMRZZxNS_b=pZ%>Q;9Zcla!~v{1alpfLR^TIK z72u<2mC(0ee~AO7q{TttOmMJ+NxYhr%$k#uAEUDZA2){v4F-Hd-&Ht7X=qGAD}$-v zJkVmAgXz4QK9n`54;@Bl1wKhu0X~IRLsC-HBo3F77Ds?H!I2JT@M^{=)|@fwX*w(L z8FOgFNWf?HU4^5ShQ={yWpFGw54SkZ!AxGw9M76F$3I7B1x_HV0G~&z(PPGrlQ>aI zTAT#V1SdO~#j9CUSaa5tsdQH03+B+I$$&5Fy9%c%4UN;$%HT`jJkjD&2P=8C@&s$H zJn_4otu|Y#ccaU+>D9YmmR2&&06$DKZOS%p*{|qo_Bqv-eeQeMF=M6(ubS}MQE~VW z{d}H5F|8EOB5lR9Ubiy>-Y^RU{H<=Bhx{hq!#rEwR(TG1==>JV6rfCJ)f!8>wPriHKs(;~S0 zm&L^n=JIOp64so%|o2qq71(C#wKA+szWUpw;>f8xgmH^E!*$94zG3!XnmO zSo8&*75F7t1^AUYv`yl6eOKWQrJ->rIB(f%aF>Gtyc$r8H3!t%M`s1@C#wJt*v%3T qqScO_yATh7^LD_?CiwM>zqTs*YkNW1Lo_tFp7SwYIss zy}rM}!o$SN%+1cv(9zP<)YaD4*xB0K+}+;a;NjxqdFX5ina^ zwzs;wzrbZ?2?`4?FEBAOU}0ioWEvYB9UdR6tgWuEu*Aj2$H>V60s{mE1_ulc4i69! F5)*X)k5~Wz literal 0 HcmV?d00001 diff --git a/parquet-column/src/test/resources/fsst/binary.codes b/parquet-column/src/test/resources/fsst/binary.codes new file mode 100644 index 0000000000000000000000000000000000000000..c26c88f5485f3ba354101901a2f7eb7785aa362f GIT binary patch literal 12527 zcmV-&%6w<{x7-*T> zC)h0t7Xsl8|k%-g=XXf zbQij2tml!0Kl`cD+JJmq>yrXlr(52pclI#Jwv;cndo(}tN}mdG3l~M#3PXWX+%AnE1&g=yA|D=I}1k` zWjh8$)2!}wm)K|^RI^YsdmrdnOX&!lVlL(TCD1ifp=MQR&)O2j_nuN$?;o9WE^8bV z`Sh_6uYD+^N;Ea1I{$6G&T5{rUp+@>^4wi{E#plvI}H37_&Bs`|xQeqold%Jd7=l4ki~)lEwGv zKnV~<+1U%s995Ei*{sHk;oodliHK2@B)*eD5F+l9KHhkwC|4!akY!*X4(AfQXi?`f z#knq?%Ae8WWu0u~hO0&CLqrgQGe9FB*W{jH9W2#R7kor%!7u2DBOOm|i2a#-3_N;; z5^Dx|JjM0;c$wflJT-AP$-{qa!jl32{oe5PPB8=Tw~d%aKAvs%3Z@=%WV&k*BoTs3 zO?V7lbT`os!PnO$4Sgox?QRXGz zIr1(#h_Xrv{BEcegF0({5eyf%vbnNl1{q7O1kRzT)W z8Y)jhi%xc0+VYW|8&f&5yW(4*pRy7Ke|M^H#98|}X^L7gIIwio;(eB$Fv<8v2Xz?( zh9^G)FgX*X^i;53`waqYaakHIZ%)m?Ad%ih)9DA)BVvJc7;B}{r2qZ2g(|HEMf`w( z4)9oSpOARBF+Ru%YKycDGAjsCfA%@PUnY?*!UC*8bNY6md{!nJHR@Gth}O+{CoO%xXaDBqDI@JtFL*aCxy0&L$pjz&A>8O04G?ta(YNCTD_CyL>Z6)zAB zt3*xV948mOWFBZ7kuCEI2WAx;3u@hvchR~-Z>6v%jU9}E7&blHJqS4C=^hG73@tV{ zGh8blgkD{ll(-JQis4}bl;WU8|NU2KoH2O_1jQ)N zH}NDSX;$ySNLXF&X4*CW_q~<1$cie7(H8v~lLYEvNcSqj73F=sBvx3CkTJ*|cLlD> zLE+?!@k%)*3A|K(;q}yf57`eNbqzFB?-n&a@u-e#4^U``G}J*y^N~Ve-z^{vp@brG zNm)4uVj-<7FDZVmLyoI$zbZ<4YS@>iVnzuRo7V?`Mb)tguf{6F#;H~YIDO7cO^VuF zIpOthI5rDla4sS3))`AhH>Z>_qktY^N0~hKceV-a@_T~loh48!lMM2CxJ=c2vf#=< z3Z6YZ_q8r;vX_i1{*%BSg^%!0m)~Co99b*Iu{7)nZ!r1}e}&IoKO4#k=uMu)*h--C z>T_x0-eqoSUp4_K>BKd&Gr$lKpm5#5o6Dlj_J{iUSw~-ohPL zh1`VI7oGkUe<2>2ECZisc3deI6BD-3K8i-zGd4LRA@W}eI2WY~_&3_A2E?@v=eYD8~ZZFZD z@$<1p8RGNE@H*Dww)RteN}}|_SR1rW2Kli7X=iL7q=MJXbDPC{Dh+-t{>+fI{l)ye znyF0^X8xInk**K5w|_l)RArNdEiUGx)7a}$v|S%#ZN#~5+pr?E4s68U6Q zj)}`zA{;VY8B(8x&aTkpqR6RJ;ExJyTw61Cm0_bj_)g1X5fBxXk7`yVwI=k_l})bi zbCm%Wy%$uI&@z6aT8PbLfrBkk&ct3+MBGEdo}B4LbmBcv*z5@si%&fc6PvyI*9DL2 zTVxocebY*|>QeIH933gaBAt3?b{7Xgd$|g3X*ZwL-L~BjcO+J>1sQ4OuW+rcC5zx7 z=HCoQMgv`@2o?|XY=64+L5|es)c83Lj)6e8^XcJbh^;Ba&u<#Jy=$2=;S+|GCu0k` z55Zq7B!rLIs-KhsM-%={zds5quo*q0o`TXeza1Gz+unalG%rjSK)>`4OAC+16_o>> zZlIzM-FiIOM}Ey8&OLlo#wk^78nTR>uGe>~eH^IYY8xfSAW6vx@hqOi*a-!jIqg7d z3Z<+bUA7184me^WS5XgNHaZRDFw|*qe?Ko?j2(>K9*qM-pQI1o(}yWXE*YC4DSl); z!IEcey&pmpnRck%%-g*AFAV#fL75C89ox=ckqvd!JPYUEVpLjqFV4(6N^fLT*7#v| zOGN?UqX+=+-ix-DT@z(O-jbnq>z?y7%@-9!VV3$4Zj zF_o#+1($pSJJ4lC& z1VZgTis`JSIfAvvRet~u#9`oBv2-RGgG4EVu!@H}ut(UWb*IG0<-lK;xg>~$moec@ zF)C+#C+kSDC>n|lodZZ|cEjUmK-1@h_zn@PNL|*OI`fZ$eXaRQnr4vQuP3xPytkl> zPN_s6GUgZ}HH0zd9{qzI;4tS$ZdDxpUz5Y&Ap#+25*TO`A5#FW{{Q{{D>%6koo2C- zA!5-o)y+Lviy=Df4OTPt1I99s+8gW5%eVf0l@=(b#y523UURbFu#`71kAQZq$cz?K zMy~EZvCdJ;2PH;5v?^g8`h**3j(`v`9OoA=d&Fk(ww*RRq#(k-n3W>UaB!FstN13@ ztP%7voeG4!p;|Kk{g}u{1`d%3!LXneL>NIM$2SimCTMqEevCgDlRJrLLqsoLT6d2I z>OkxYdIis=cL2jt;HlLbY1g&MGFhxW27PsrHVP+C&<+UPBm-1HL)N_#UUhQ#F=kao zP?Iu?#xCT!c72e0VsHqKOCt>|Q%}u4VVD2?OZu{OD{~URg0-fL!~p>ZCWqZ4a^!Uj zY~n2cvZ9jn?KCnjOMuM2ljlLvNA zdG$z07=Sa+EV1mYurqSAzZn4fTfFtOPba%iS1~&fP(}B52|}1^7h$$Wrxb@Qo9Z_r zOP3yAHZcbrloAJ{To}}|*|4(YFsnhi1U72L1j%#!EROJ{mWLmj7xSTcZEz@QK=&o_ zfLvK8$M8P){;xGUHD>j8fTxCk)#vQ(34>;8Ej$E!T@?KB`p!gX z&aDzAd#mNwQ#@y0{$LS^lYGtMvZ_BMfYoF{y`^bp)gmy@@sHSgl|$cdWP$(*C#RH; z<%S)AZ3_wX@yg-lJ%WKvLhWcJy68-OB$gEDiJgd?D*)Q6aE-NxCa~*Kf!m9UPBB`A z1@qKI^ZenFsUHr{n-&*letfwy^`iw@(0d0u3PpCb6;>jyX(Z{GyS`>_5#Dih5KV{ojUt9#WF`IIGiZ5&TDG@`wf z)V0a*nUU$+DMU~WrECXBds{>)^aj3p|NV_zpv%tT3p2qkDhYMU`xoCCgffe^5koav z8v;u?i%HEKy&%fF!WIoNMt^D+qGCwoy+I0drB5Kc2lo#`5U8(-Bd-K}gy z6=6^GcYv<;mvt6OM*YTxjb?w{K5)Ls)~W8NdU~=d3MhJS!nkash`*TFmbx& zJ8=b7Pk-!W@zPv)96a@{aHNRtGAIDU0bDxs+%Rffki)S>Vzt|9>&AYnVh`2v7=Rw8 z4c+|yHcR`e4Y$i0yxeo5guXqY^G23QqGt$0UPPZE1V@=pL9MSkX_Du&fY~#{gG!E; zScw-wxlfap%fS3}l;l*0PfyDbMTe^rIarM|hf7VPK5{b}8@fiC2OXg17Bl%qPr}1n zoEp1Vm`q=&o~3brVOd55JHE;l*&GI|lbDHDT^$daTLv`&dKeaS4yT3&oUc0;$763C zY_=BBvr74KUgA@fqDSNqzZ6_c7F3&S3igg5do0xLY%3M^wX}&k(YtYLV^?XYs&QOBUnDG!0z3r= zl;0u?>%L*SY<>j{oY*uEQS_vkQ0?kwkEgs^*PqIFQcwr-gs~K(;>N^=Z zOoe9eYkgP;xs>x5>a%-&_Hp<4Dt?Sw%*T$w6C2J@MEFt({;_!9J9^Udjb(5%egf|p zKA6t3a2f}h0;CEOTyD64VgeuGZzZ)L=i(yZr>(I%k*G!LU7dNlEZE?0=y~5?l)r z<8E52|NSZv8HzC6cKsVF1#Sf>B0~`UX-6vAUyYsW9vVCwU<(ffvuOEJsIgEJLUuqd zSPicO$cTf|z7b&I2WNMU^f#nq2h^W^8tyK5jZnQx5PYctO=hERC8pp8LxYzg(;3i) zlKivMSaBxDHzgZvg4SPF1SZO3sG0)Ai9Ls@<9$cja1<`3*YV!-?I6Oo7y*w}!GwVY zH?}Z(X6)@$&@&B?N){h|3h~R9ranf(nr7x+u2x+P&j^mAfZ#brC;CrJQR5Sah7N!* zI*~ggoKHt2k^lWD_CI8EN;2rNUV2fO+ZBA~lQ`Sm?fJP~1XKav<-D!aaL=irb)}}z z{I)LTL4zg3#A4E!3{E&c)gmSCy@zgG$KF89_1ec%N(%V#5rDGmdgC$AoMV7dAip4I z3MnawM-o(ZU6e<7bHjUN3uH=pD;;t%uOPnIg34QA+?}?(=R~t*7NtklFb}hJYgr=I zFt~w)zbEZIWwmt#8&3!~_(4$!BVt(8N)Ov#T z_F};-VZ(*~wjvwNM8RPM!f5{GwC20EFhln%32#!YeAomiBZCME7H@1P>+gZ0drY*s zSS_OFxjDC=X^f*hF?j)ZWMu=*5IUCNIHRs{2`%OeWSNsho7Ep>Fq6W8bh=_wJ*2h6 z9r;au2m^@M@ymeAQ)<5Cvt88Wb3dxfL!JrZO&utIA6#q=gEWKK8N7hdLpR&UDmN%Q z>vIsSbt?I$HOPR=bm@-XH0n1WX0WFq2BU#{HrsT|TnTY~cJjzg?Nu>t@grC!Pe`lk z7tB-2vfb%2E%K!xgmF!#6bsaTtUQNGv-xcuLQLe7WOs~d%CkCJ?lR9+3WM?#LH)n+ z3Q8#PIq!-yBX@tuo-9Ny$6ZR@MRXbnV@-~4-9%V(#nJaAs7uc(t!8vD#kmL z&|+=R;lo2S9c4No5z%;y3VIZDZj3q?uS9I0HSo>QO1iluOyU^zTVYzZgS9aT$~zDN zeMfrQ>%vsnzh(f0?Ue8!NNFMb2MEzW(TG5C%}p}#KZ>;Vf&B^hGg~{o|NRdh`Avz` zo!EWy_{K$YJ|-aJZZSMjc^bc+;h5+;MKh`fUL0~s_PpT*rc=EqA9+BEiJNvs0z`T# zK6eBzV7IZ}*PyIT@zw?@T>9+TM7-#8rQLoDKrFaowVFT65q7r6krW0o&t0iyg2dtQR0b2T~;+77CJe-SUpT}VzbRnvx-hGz9}4IwwON`+t%3Bv)jK8nvpt`bSZ zT@`#TTTrYcq)@X43K3O;B#)|enyR0f7c9$=0W-iGOP*lYD)o|zTkK#AC-i-Hjy>ug zGnV{|>Uz*|O|H9f9@(3Ydug4@79Q*3fu#)xJ|=JnJpA)G;$lDRY&@seKI5 znR!x1iHkz&*_P2>XXWhsAO?dUNvYO}&3Wvkwtc&j`kcRe>_N2}z7^z@no~9Y*kkM<3BIBW_o?uWP`)!b)c?4jU1q@5Gy`GC3*Efm9(PyXHml9G9$(rki6o z3uhf66F1E=K3@2_Uz3%A6Ep8Nh%R@Iw&hbBcy&e6Q{Tt0e7d%C)6K1L*Voz1(K9V| zQpR;}(47#Ny>y)NRP4k0gYee*0wT1lR!b_4?vg>GbXWmbjJ1&UQtJGM!@H4AE7=P2iTH(?pVjTCrLb-`Y`|f zuAsLNT7m_L1khbC2sQpDM&N)_D^PhNYr0|@8KsmFQ`npUV2-lE70MZeZH17z2MkB9 zfZ0t$loT#uOZD_1Lo|Q~4#75>qx}!a_<<89O}P$#LT?2n*LyHs*#`V(YUTq~zDNUT zR{6^!pQg^cqoR|v=1pL0jjGf!1pKdOGLCjg#XXK8I!A{pn`vvw^l>kdzimFF&;W^b zSi?Ef;{GXNVlqAYM&o~2#+4Z`PgbyDdxbGmp!cQ1N~;tNd9m2ot{zW;MC0q;?7z|UtkU$&XzP&*_TPK^UDk-hfep6IwxL$b} z(x#b|dWV(hEje-62`-F75Vkhk7D$O^mIBI^(|(NXIObadqYGF!g%K(uQ0Ozn24+4o zKy!xtHbzs-!PQ{BZLqu=fa|?htBofv_Rnq1WB)YMCK;6l!Dl$jq@*F3pUg$GcZt~`< zG3QSs`dlju&WSTj*6y*~>*t)y`EOS6%+yoalO4DDQ}pEp+V(?5E(DidS#ON%a=j#I zQ>xMzpT2JKvJl8vcFMv5QkcUa+^{+oAh}vn%Euwq&-gf0e7eq!3HAA`Y;52Zw+6>g z!ad%9bpot#F@!1{TOfTa8ey&Lvrokf9tYOr9Cxy8C~EmiJ6Z zO35vXqBq0^hB!(cB)sJc4xn8j9uX1YC>fOn4qVWK z2LZiu>&Qu`Q{5xUU`=$SL_8V!=oK<9?B!}wh1A|kSzXsb$jh;ocr!Ios6XsLkj3$psx%k6?5;ci@E3TPW0pZcWQA4Zyq>pb_~M6 z6*AV#TleFT_of`BB_j-W_q3PWgYG!wzkq3|hV;jt5cf|t9U&U_6-LcxHOFwRK#~;W zx$zz;*$Z3?<|gu=VdCwY( zbJ8I6R#?|C0Y3=R`-es-e53JWD(batt2)A$$f8jXSCBY=#W)i#0F@I!PFvMT2+BRv zGzg8ak5&wik6UVoz&^DBS1w5+!va{lmF`3aJHc!DqX_#fhc^- zRE!~Uhh3R)aC+u>a2l1B4pQFl>Q>Sm6?aA;-G{~?_#0yrIq=oCa3a~<`rhM1xy@9X zcqfY!ra(szElkaDmYA<_jm4Od!s^uFvVM(=x6?*V?c6Yk9A{q0!Z=V4aO21cCiN+W ze_#9kKkNmLFtq&BHEUOnY>j{m(1)Z&*;yz9n1sKb%F)|s4hdpym{u*x(;;1ZHSmRZ zUZUe}*dltnI@?} zgy@j!O;QTHVR#}%4Yhe&u=$67U8o*q3W>wvrF@9B=XdP;L9@!fmrWL&f;4|5AwZKg zM@na|ofb(n@R8koV@MB?(1*$=`|h9@35H_ zL7Ff6ufi!!q;Tg~E9-Qjm6L9+cX_;g@F~X!!5E;sFg#gb*kX!i`)4tg3--$D0bp~2 zaKaO^%NnRPqi{*9d>L)|8xR;cVH1*@AntH(R56v6gk0p~mb`nkdwc_(2?Lb?J8$Y+ zLy!-UlVC)9#sJrCA{K`&mx8RGSDl5d=!yGKN_Z!j+|q97J^D377a@t2MR8 zd}~=?7gN?Xqf5i*u^!BUtZmD3?mq!nyo=Sq@-i$SNGc^&{942^1zrglHn0eHyel+i zmRzp%P;cxO1`qhZY~+xvZ)d{OTFGb)Bs`7Z6v-wB3xyw4yPEWo9mN;vIzkjOg+-JBk{oR*r8u#MGIP|VLN(Khu|I(3pKkl1!PQtq z8CUXx*m3#fEX;!USS4!P+8w9>7v)Lu96*$EKQ$X2%EN zlq|W|<jket7q97x-7#hD`;<4QrBm!hKa%F&ugb1|x5yt4gx?nqivPOlpB zs27Z8*c4h0zHQ0_Gcr!TpwNPBSi`xeLFPC&7X;vF$A!P?J3Y8SgHFrYHFPtpC>y1> zta~tm8BMUa9gFlJyaVZoCt0KgrJt_6T=CdW+>6h(Y0Ejr9d0JLd$-jv>}Ci1=Dg3n zCYIBMLnZb*AJHTDl~T>|_2RYpbV9(B6&b6-s4_supy((o2D9decYHY5pa5l?i>#@K)@xyBn9{1$`+`Ozt(<4f z1?8NFe`Bi;Km4UXUjS(L7L8lBI!xL&+m>Y@f?5_GDJFaYfu1itsfz(|AuAU%2T_B! zrXfeVb6{cYX=Wqz%`i9c8DCbtSLP>91{MU8rX&!I6({VfZ@n6O1x^e>WseC;jDgH2 zWURE|U8&tX+h30ZJQ+jXI`TeVQ+rW(7P0jSBBf3ICi`x^JHX%n{U2+mq(++6{CMJu zF3AfvMJ|ySS4N-en%?fh5|>3Da@$f;JK1PI4C_qJ#ipp3p)qWrVY3SRElk+|{WZWH zbmOYheBJwsiPbmA!Ckd@)96F2BHLOb9zn zlXxzqig4{%vqrU(=K}%dbDeF}!OP1M_mx+#?MVizNZm^4Ai_a#@lYx~xpQk|U^Lz# z%&60co#lgZgIC7HQ4bZk7_{0EYD=7CF3pJWNhB#Z205?^lm>5aHkf5PSGQFX?8B@o zp{$1xnBOLgF-ycuh0nXNhI|GZ4ZC#ylYSE1tx57D88aKt9#T8}IydZU-$+B>_D~Jt zCb0>L>b3SjWIhg*A5AY$o+f2A3TP7pt}o^jFhM!HwN^-PStr9v%E@?qPu?Me^JRNu za`aD8`6UY;AH6Tq4+kj7GiRe&HA@doJc%F2zF;$9;cpBaPRo}>;+%Un6bv*U4ofdA zB34^W9xFsy?i4LPT6`a30*Q|&J)Y?#g|gCa0QaF|@)Et3B{#U^!fC~*ny3O$D7r8l zjGb*@FA|OlkeJO&`C5wN=%Ad_ViSD9JqUW2QwvuYU>vvY3f=aE37lyQM3Z8V z2q|8CZ^xzrQS|F?B%4}H9u%7xJ$Bh(eE~?{Cq5=yDijdATQbVN83uE75}2dJ$V(9p zkk#hrYdOYo>#lT`5NRPE)>^Oq2Jtfy^|f1AV*rVbIT4VI*i`IH)hpYkP1u0*oh$yY z9M!Yy@cEH%7wKem^#iM2Fk=PcQpV)G+H=^JyS9eOj)z;KQ4B(H zeGYzdf!y_XIDjFjT~8xO90-|dv?V5DFlYyV+XF>Cz6=EW#-F@NmEgfCsD%Le=y*UiF{C^K3&XB?6;-{vK7l-O9HPL?(ZR(|I`%ZNGU57rbc zY)9%NIL&_5GPEN?L6~_cDhazPJc2$ysq`0h*~x-D(`B)L^Ul`c1X1P9JpC~YrXxk5k! zTdfU|O-B99gDYu+l&AxGt%;va>_u(dfvsuf>2>|n$hN$7SnfL=8KS&ffywoA%`*0e zp-LInsJI2LVNt|fsM0{%qS${?N#>^*(jcqwSN3_A;b=IvdEl$Zx7^6uXCz&i=fDU+ zM>z-l+dNQ{136e0H|?%wB_fX&X}pwOq;QROseSMl zo8EyuI3g6M4sZ*67fj15myHy%brSGF=cINmuN}&elBeYz_MS9KPzV9rF<@d?J<(OS z%ztIYLryb1(y~HsIc_Ofq7cYdw(r$%L$LvaGgOld1$b$1Ix~Uh$Y;`;i3Xr_awvvC z?wQ1WKl8k8KOvG+Y+Rv)+7+F*GDAc;&krw;^uYDI;n4a?myqGmPeQ6-B(2$n8P$kGKP*3! zE)O)YR?JQ#s4I6*`F7`Qc}y>WQ4)i}Q5PGGZ5Ww~7a59^Q5GYoeacbt{?>SRw22_w zSJqGQEmJMzu}`4|GoKp{`G)2{2GY!DC*Pv#%=+LCPeDD?9@{~T z_}$e0&ttf@n@cyhO71|oPbFC3F-EK=d%+TBN6dfx+L0VyRhlrG# zs&|z5*r>i zI8IboXL5RjkG9_X1|uXhH&0k*XmE6dk)o@+zQN=000RmwFf>C*Q)O;>n5Dea-|PGh zCn`cxm7$}jti8$7L`qK?f4uYC{=NNiKf)_`V23P@$>Zo OI%2QZ88S&$nXN~6fq;Sl literal 0 HcmV?d00001 diff --git a/parquet-column/src/test/resources/fsst/chunks.codes b/parquet-column/src/test/resources/fsst/chunks.codes new file mode 100644 index 0000000000..851fa256b7 --- /dev/null +++ b/parquet-column/src/test/resources/fsst/chunks.codes @@ -0,0 +1,5 @@ +       ÿv  ÿv +  ÿv   ÿv    ÿv    ÿv  ÿr + ÿl ÿzÿy ÿx ÿg  +  ÿv +ÿr  ÿrÿwÿn  ÿzÿy  \ No newline at end of file diff --git a/parquet-column/src/test/resources/fsst/chunks.table b/parquet-column/src/test/resources/fsst/chunks.table new file mode 100644 index 0000000000000000000000000000000000000000..e7f2f3cbf4e2d922ae20000145274a6312bd19a9 GIT binary patch literal 116 zcmXwxw+?_X5JR&wvZMR8R8k0+O8a`aNO-dRY%en#npmjH*p|_2-a)#Ugu)e?)et}w lJa_o)#-Dp+xjSj_0^eh*fQK&7s6!bs+Wd?*^hF)|cmZ%TC!+uW literal 0 HcmV?d00001 diff --git a/parquet-column/src/test/resources/fsst/empties.codes b/parquet-column/src/test/resources/fsst/empties.codes new file mode 100644 index 0000000000..2c59794a1f --- /dev/null +++ b/parquet-column/src/test/resources/fsst/empties.codes @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/parquet-column/src/test/resources/fsst/empties.table b/parquet-column/src/test/resources/fsst/empties.table new file mode 100644 index 0000000000000000000000000000000000000000..3385247f7cf9b82a8bcfd7a83be20af2b8ed11cd GIT binary patch literal 13 OcmZQ!VSs|:?:LG3=M8A?@LD?3M><=A?<6E3>A89LB=<839=:?;?<6C3=A><=96HL7:8<8ALDLG32?M?<=<67@LCLDLE893=:KA=;?@6NLE67:?A>9LD?A3=:=M8@89K3$@6B8:6D6D3=M6NI@=@>3;89>@?3:>9>@?M?93!=;K;LH><38<><6B=9?3<6H8A6B6H3+K;LB?;K;>3&:>:=9?<6C3>:LELD=;>3A>;K98@36BK3M>A8<><6H38ALG6H=A>3@>@>;36NK:=A=AK3:6NF@K;=M3LNLELD6HL7:>M3=9?;LC=;67A>M=;=A6C3>M8ALNLNJ39LNLNLB=@3-8;8@LB8AL7/@6NOM6NIA3LNJ96B?;K3;89?AK;>M3*?9=:LG>;?3M=:=MKMLD3>A6HLE8;?3@8;><=M?A36HK;8A3=9?MLC8@83M?;?9>96C3)=AK;6DLCK3;83%A?;6B3LB=:?:?A>:=93?:LC=M8AL7:?:>A>:KM36NF;K@6HL7@=:LNO9=@3K:K<=:?<83MK;8A6ELN78:?;>9>;K3A8M3898:?@>@K30;LD89>@?96H8:=:K<=93=:=9=:?A67:8:>@6E=3'@LE=M6G6H3?9><=:LH=3@?@6B>:KA32K9>32:=MK@6B=@32=;LEK@LCK32;>:6G6C6B32LH6GLE?9?32@>MKA>9LD32LNI;?<6B672MK<><6B89316DLG6D?;=319?@8M?<>31;=AKA>A671;LE>:?M=<31LDLD6B>;671@=M>;LH?@316BLG=ALEK319>:LD><=936DKM>;6E67$M?<8A8:67;>M?@8;=<3 LB?;8<=9=39?:=:=3;8:K:?@?938<=98A8;K398:>M6G6G3=@KMLE6NL7@=:=;3=@LDK;>M=3<89LB8@?;3>@?;?:=M=3:K@KM3LHLCLE89>3#;>;8;LB=M3?@?M?:=9L7A?A6D?3.96G6C>M3K:KM8@?AL79>:LEK@>:3?:=:=9?;L7M=936NFMK@KMK3@=;6G6D>M3-LH=;3*8A><6C8A67;8;>;>A6B3K:>@K:LHL7;LE83,9>:K@8@=:3LD6D?:>M=39LE6NLGK;3)LB>9=A89679LCLG=M?983";><=;=:?93"8;6E6C>;>3"9=:=9?M6H3"8M?@=;KA?3"MLHLN?@6G3"LH>;6E6GK3"<8M=:KALN7K<8@>A=967AKA8;?;?<3?9K;=<=M>3A8;K98A893?<=;=396NOA?A?:36D=<>:=:830ALGLCLE?<3KM?;8@6E>3M?96G>;8A38:8A6C?;?3@K;6H6B6C3KMLH>98:K9?A3=M>@KM6D>3;=@=@>@8<3=<=96D6G679>96GLNLD3?9>@>M6DK3:6GKMKMK938MK@LCLEK3<>M>;=M?<38ALB?@LG839>96E=:6E38@?@LD6CL7M?M>:?9=93=:=:>:K;K39KMK;K@K;3$LH>;=A>M?3$M>@6G8A?A3$K:6D=:K:=3$:6G=;=:=<>3$;=A>;6CK93$?9=AL7$<8A896B?;3$6D=A6NK9?3$9LNFM?9KA38;>A>;>9LE6N7>:=@K9>;>3:8:89>:=<3!LB?9?96NI39?9?:KM8:3KA?:=:6BL7@898M>AKM3=98:K96H>3M8:?@LG>M3+?9?:8A?;67&;8:>9K98<3LHK;8<=AK3:6B?:>MLD3LC89=@=9K3<8:8@8:K@3K@?;=M8M67#9?:8A?;?3 <8@>9=:LH3 >A><6GLB?3 @K9LGLDLN7 K@LG=@8<67 <=9KMK<6N7LNIA>9K;?3;KA3-K9LCLG=@?M>;3KMK@=93!=<8;LD=9>3!<>;LE8<>A3!K<6B?AKA?3!9KMK@LCK<3!>A6G?;=9>3!:8ALD6B?@3!8<=:><6H83!;=:6HLD>A3!>@=98<8M83!MK@=MKALG8M?3M6DLH?@8;3=@>9?A>@67@?@=:=M6H3>M8:8AKML7;6B?AK@>93KM8ALDK9K3;K:>3;6B?:6NJA3LD=96C8A=3@LH=989LCLB=3MK@?MLG?<3LN?M=:=<=30AK<=A?96G3=96C=3:LB=;8M8<38@6GLG>@>3M?A6B=;6C3=A?M?M?@83M?<6H>:K@3>M8:?:KM=3A?9K@LNJ93?3;>@6C?;?<3LC6DLC6EK3;6GLDKM8<36N?;>AKA>3A?:8@>9K@38M?;K3@KA6G6G6D3=;6DLC=:67@8M6G=@6G3KALBLD>MK3;8@8MLB=M326E=:?M?M?31:89LE6D6H36ELN?A?A>3M6D8MLBK:3=M?<6C>@>3A=A8ALHLB?3"<=:3LEK<=<8967ALNO:>9><3+8<>;>A6C>3+A=A=9>:8@3+898<6CLC>3+M>;K@8:LC3+K;=A=A>:67+A8MKA>M67+M?AK9K@?3+<8@?;LNJ:3&8@>M=A=M=3&<=@=@K98A3&=A8:?<6D67&9KM?@8@6N7&=;8A6B6G?3&;K9LDLNLB3&K@LE8@?A83&:89><8MLB3&6GLH>@=AL7&@6NKAKM=:36H?M><=:83@?@6C>;>;3(K:=;=A?AL7.;K;8A?A6D36H=A?A6H83@=98:KMKA38<8MK<8M=398M?9L79=@?@=M=93LB=:LNOA83@?<=@>A?@3-6H6DKA>:>3/@8:=AK;?93K<6BLN?A83MK@6B=M?A3*K@=M8:=:8936D>A896NF;3)6D?@8;8:L798@6H>A=@3LD8AK:8A83%<8@6DK<=938ALE?;6NF3:?M=;89KM3LB=;?@8MK3A=ALG?@LH38;LD896NF3A>@>M8;>936DLH?@8;67<89K@K:LH3>9>;8:?AK3;?;8;>39LC89>A893>AKM8M=9670MKA?:>MLB?3M=A6G=;3#8@LB6HLB>3#@>989LH=M3#?MLNF:6H=3#96D>A?<8;3#>@K;K@LC83#AKA?ALE8@3#6B=M>;LGL7#<6G?9>@8:3=;=A>;?A67:8MKM8;=A3?@LC?@=:K3@KM>:K;?M3?9LE?9=MK3:LH6HKAK93K;LD=:8@L7ALHLCK@8932LBK<8:6H83198:>MLG=<38<=;8;LNK3ALELG=:6D3LH6D>;=@>3M=;89K<6G38A8A?;=AK3"A8:KA=;LD3>ALNJM>;?3;=ALNO9>:3=A=MLEKM83$@?<8<6D?;3K@?:8<>:=3@6G6E6CK<3 =M6GLE>;K3@=A=ALD6N7!8M?:6H=:LNFAK398;=:?:K93+K9KM?ALC83&M>;3KMLE6C?9>3<6HLDLN?M36E>M6C6G=3A=:8A6NO:36CLG6H6NL7#M=MK98MLG3>:KM>9KAK3;6CK936EKMK9?:>3<=:?9=9>:3>;?;>;=@83A8<>;?:=M3(=;>;LD36NO;K<8@?39><>:8;=;3K@=<>A=9L7<6B6C8:>:3>MK@LB=:?3M6D3LE8A?9=:>3@LE6GK<6E3(?3(M=;?;=<6G3(=A8@LE>:>3(;6CLEK:>@3(6D?9=:K9>3(<><8:LDLN7(?M=;LH6E>3(;6HLHLNFA3(LNI;8:=@83(A=@?MK9?@3.KM>A=<8:>3.MK;LE>:>@3.=;6DKM6C=3.;=:6HLH?A3.8A=MK3.:=:6D6CK93.?;?A>:LD>3.@LE?A8<=<3LH>:=96NF3<>@LD>9>:36G>9LELG67A89LELNK<3K;><>@KA83<898;?M8:3?@6C8:6BL70:6HK96E6B389>;=;>;L7MKM8<8A?:3LH>;=:LCL7M6D=MK98A3=:KA?9=@67A?@3K@=:KA?A6E3LH6E>M6C>39=MK9LC67A8A6C3KAK@?@?@>3:6GLNOM6E3LD=A=A?ML79>ALBK:8A3=A6NKA=:>A=<6E=:67@?MK:=M6D3?96E8:K;>39=M32?AK:><=;K31:K9K<8:LD3?:6DKAKML7@>;>:3=;6H8:89=3:=9KA?@8;3><=98;6HL7"MKM8@=:>93=<8@89>M3=989?AKA>3<8<6H=MLG3 LG>9?;>@>3<>9=@6D>@3!=A=;=9K;>3<=A3=;LG?M?<67@6D>;K@KA3LH6B=9LG?3M6G>3&<=A=:8@6D3K<6H?@=M>39LE=A>:LD3=:=9?@K:?3A>@LE>M6D3LBLC8@=:83#A?M><6CKA3=@>:><8;>3AK9LE8:=MK9?;3(K;LD=9LE=3.;6NF@?98@3>M>9=:6C=3MLN7K9>9?;8:67;K;8@K3/:6G=@3-6B?9?M8M>3-9LB=9>:?;3-=M8;LE=M67-9=;?M>:8<3->M89LH6DL7-AK<8:K3-;KA6E8A>:3/8@K@KA=A67/;?MLE=A=@3/>@6B=:KA?3/M8;?;KMLG3/6H>;83/M=@>;=@K<3/LCK<=9K9?3/@8M=9?@8:38A6GK:KA=3@LBLG6H=3@6N?<6HLN7=;=<8;>:?3<>:6E=<>A36NIM>M6B830:K@?@=:?A38<6NK<8@=39K@K:=3M>9=9LC6C3>9LNI98:K3A8@?9>A8@3=M8;=@K:8@LC?3A8@8@=@8:36E>:6E>@67@=:?9=:?:3*6CKM6BK9L7*AK:6NLGK@3*K;?A?AKM>3*M8A>M>@LD3*>96E>ALCK3*9=;8:8:6G3*6N?A?98;67*ALG><6GK3*M>M8A8@=@32?9>;>;=@?31M8:6BK:?3:6DLD6C><38;6NJ98AL7@>:3KA?@?;LC83";LH=;=;=A36CK9K:LD>3:6DLBK9?M3?:6B6GK;83$<8<8@LG>M36NI@LC=:>39=A6C=@LH3 8M?:KA3!LDK<6HLHK3@K96G8;?M3K9K:=@8<839?M?<8@>3M6B6B=MK;3+8:=@?A?A83&M?M3KMLEK:?A?3<>A6ELD>:38@8@KA><6793=@?<>A?@?3#<>@K;=9LE83@=@6ELD=93?<=ALB=:L7@8;8M>@KA36B>;LG8:L7;?A8:=9><3(8;KA=:=<67.A6N76N?A89?M67MK9?:89>A3,>;LH=AK@K3,:>@LD8A=A=A>93,K;>M6BK983,;=MLH?9><3,?<>;=9=A>3,9K@>@>:=A3,8A6BLG?9L7,96E89>A6H3*?@=@=M?A67;?;?:K:=A36E>:?@LG67@>@89LNK@3K@6NLC><>@8<67A89LC?3@>A?M=;LG36D6CLC8ML7%:KALD=M=;3KA6HLH>ML7:8:=9=;K@3?:?M8<6HK3<=:KM6D6E3=M?98M>ALB6B3=@>;8@89>3M?A6C?<6D3)K@?@8M?:=3)@=9KA8M=@3)>@?;8M6D=3)M6GLE=@LC3)?9>M89>;>3)MKMLGKAKA3)8M=A=3)M=9?;6NI@3)=98;8@K:>3)M8:K:6DK;3LE8:KM?9>3M=;?<6H8M3?@LE8:>A=3A?;LN?9?@3K@>98:K<67MLD6G?@?:38M?9>MLE3=A?;?9=;=3@=:3>9?@LNF<67@KM?MKA6C38;?;8:8;?3;KA6E><6H3LNKM?3M6H>M8M=:>9><=3%:67%@><8<=@8:3%89>@6DLC?3%9LCKA3%K;>;LB=3$A>9K@><8:3K;LC=:8@67@>9LE6B6E3 LC6NLH?;>3@>@8;?ALC3!>A8MLC8;>3:K9>M=9LB38:=A=<6NK3M?@?@898;3KM?MKAK:?3@?;6NF@8<3+?M6G=:?ML7&;KM6D3LDKALC=<679>;?<6E8;38@K9>:8:=3@6C=A8;6D3=<8A679?:KM8ALD3K;=:?;>M=3:>M?;89>:3(K989?;8:K3.A?9LG89=M383M=:LEK:?@3=<8M>@6G?3@8;K396CKM>@>M3=M6NO;8;?3<>@6E6C>@3-6H=:LB6G67/@6C89>9><3?9=:8<>:8396BLC8;?93*=9LBLH>967;=@8M?;LD3>A?39K:8;6B=;3K96CLD=3<=A3)K9?<=@>M67M=A?@?;6H3=M?;>:=@>3%@?MK98;LD3LB6B6G>A83A89LG>A893>;KA>A=@>3;>9LB><6G3LBLG?MK;>3<>;?MK<>;=:>:>3;8;=;?9?@3=M?M>@?<67MLC>9?ALC3>AK9K@=M83MK@6C6B>;3?@6E>;K:83A=:8;?9>M38M8@>M>M>30A=<6HLE6G3=M?A?@6E83936NJ;=989>3A6E=;=<8M83A>@8;K9LH3?A=96H>;L7'A6C67:>M6D8@6N7=;?@K@8:?3:=A?MLH8<38@8@=:>;?3MLC8@KA=A3>@67@LE=9LB6D326BK:LNI:831:6GK;=M>;3=AK96C6D83M=MLH>M?;3>:>;?9LB3>@8;LG?9=3"M6HLG6G>A3?<6E=9=MK;LE6H3LBLB=;6G>3$@K9>A8398AK9K;><3 ?A8;?9>:K3A6BLD?<=@3!8A?A89?ML7<8:?<=A8<3>:>M=AK;>3A?:?96H=<3?ALD>ALB>39LG6GLHKA3+=@>@LB=@389K<6D>9L7:?96E?A>93LEKM?9LG67:>@?M6G8A3>;K@8:6G=3#;6B=3:K;>96H8A3>MK@LNO@?3:LD3(=:8@=<6B67.MLNOM89=@3LD=<=@8A3-8@K9=A6NK3/:>9LCLDK:3K3:?@=:8AK93K:?989?<67@>A8MKM6D83@6D6B898@3)LC?M6B6NO3ALG>@8<=:K30<6B?:>@LH30=M>M?;LB=3098MLB?;><30=<>;LE>;=30:8@LDK@>;308@>9?:>9830;>9K:LBK<30LH?M=<6C830;>@KA>A=A=;LC3KMK;36G>:?A>:>:3=;?9?ALBK3A6E6B=;K938A?:893LELE=M6E=3AK9=9LEKM3?98A8A6DK3A=@=:LHLE83;KA?A8AK936B898@?:>3;>:KMK@LH3>;LD6NLNK3:6D>A36GLB>@?;K3@K9?MLG><32LG>;=MLH8319=:LNO<8<3LH=MLNI983@K3:8@8@LE>93>;>:=@8MK3"9KA>9K:=<38M8<=96B=3AK@K:6B?;3>:896DLH>3$:6C>;?9K93?@8;K;8:83@K;K:K:LE3 ?@=:LD=:K39K@>M8@?:3!6NJ9?A?<>3:6GK:3+8<=A6HK@83&M6GLE=;LB3?:?:>9K:?39=98M=M6C3?98:6NJA=3#@6C?A8<6D3>A>;3>M=96G3LHK:8<6B83:K;?<89LC3'K96H?<8;?3':?M=M=A=;3'=;>A6E8:K3'A8A=;?<8A3'=M=AK<8@K3'M?9>:6CK;3'KA?MLB?:>3'ALB>A=9>:3'LH=;K96DK3'96E=@=A6N7*>:LNO@=M?39LH6B?@><3>;K@6E>;K3;K9=@LBLN7>;?<8MK;83,<=96ELB6C3K9>96D>9=3;6D=:KAK93)K:8M=A?ML7:?:><6DKA38:>A=M=M?3%:=@6HLC=M3=MLNLB6EL7:=M>9LCKA6B3K@=:>;?:679>96D>A=:3K9LG89=@?3@?:8<8<>:36E>@=@KA>3@6D?96D6E38;K9=:=:=3M8@6G?A>A3=MK;6C?9830<=A6NJ;?<3>:?M>:6CL7:?9?:?@?:3LD>M8AK9=3A?@?3'A>AK:8:>M3LE>@?@?A6D=<3?@?98A8M>3A6DLELG6N72LBKALCK9K32@89?9=:K932K9898:LD=32@KA6CK9KM328@=A>M=98;=@32>@8;><8;K32<8:LH6NK;322=98M6G>A6721;6ELD8;=;32K:KAK:LH832MK;8M?<>:32LBLH=9?M832MLGLNKMK932K;=@LH6NL72"96DLGKM=A32>@=;6NJ9672;LNI;6E?935LE6D6DK;835MLB?<6G?M35K9=;>@?ML75MLD8:6GK@35K9K9?:K9>35A6N?@8M6C35K98M>ALG=M356B>MKALGK35<6G8;32?A?M=:=M?32A6H?MK:>@32+LN?A?A>A832&;>:>;>MLE32=M?:LH=@83296B?@>:8:32=<89K9=@32LNO:8ALH?32#9=9LNJA6B328@KMK:K:K32@?M?@=@8M32KMK9>;6B?32M>;K:8@32?;K@?;KA67296D8@K;8@326DLH8A=ML72;>;>:>A?;32=9LD>:>:?32@>98@>9?A32LH8;=98A>32@LNF;6EK932-K;LE6G8A>32/;?M=MKA?<32LD8:LD?A>329LH=:KA8932*?@8A>9=M=32M=9>@KAK;672,@LG>@8M=<32>;LCLNO:=32A32)6B>;8:K@=32;K<6B>:=<32LGK<=ALCLG32LB?@=;8A>32;>ALG>ALD32K@8A896B?32:?@K;LH6G32=ALNF;K9672M>ALH8<>A326CK9832A?:6NLEK;32K98A6NKA=<6D328A8:8;8;672;LCLC>A=M326NI:8M8983209K9LBLEKM32LBK96B6D832;?@=@=@>@32K:>:=98M?329?A=A8M8<32=@6DLH=9>32:>989=@?<328:8@8:><672'@?@=MLELG32KALCK@>9=32@=96C?A>M32?M6E=9K<672<6D>9>AK<31=98A?@=9>@316D6NJ<>@?31@>31A=<>@=AK@316BLC=9?M>3131@>M8@LG8<31K:LELELNL719LCKMK96C31>M8A?;=@831"@6GLH>:?@31LD=98;>9671@?98@>;LE31KA=31$;LH6H8@8<318<6NI;8;>31MLD89?9=:31 6GLE=A=@L719=A8<6G=A31!LD=;8:=@L71:6NK<=;6D31=;6C>:=@K319>ALD>MLE31?96HLN?A>31M?:6H=M831&ALDKM>;K9318@8@KAK9831M8M>9?@6G31K@8;LD6DK31;K:=M6C8A31LEK<89KA>31#;><6D8M8M34K@674M8A>@8;6C34>A6G>@8;>34;?9K@LG6C34K@=:6NO<>34@?;K:6CKA34LNI;LGLNF34:6HK:LC>:34?@?M=@6E?34MK@6C>;8A31?@K;6D>@>31AK<8A>9?;316H8;>M8M671A6N?9K@6C318A?;?@LEK3198:?;?;KA31-6E?MK9?A=31/M>M?:?;8@31?;LH?;>@K31M=98:LNIA31*6G=;?@LHL71M>9K;=;?@31=@6G=A6C=@8A31><>;=@8@831,:=@6E31)K9>@8@=9>31:=@>9LNF@316G?;>9?@L71%9>@>;KM893189=:?31M6G=;K:K<31KM8ALN?:>31MLB>A=:6G316NLB6B?AL71M6HLNI<=@31LCK;6B=@?3196E>@K;89316D>;?M89>31M=96NJ<>M31=9LNO@=9>96BK:318@6BLD=:=31:8M6B=MKA31K@8@8<8MLE31K:?A?@6C=31:6HK;?98M31=;=AK@KA831AK<8M31>:?ALN?:6E><318;8A>M6B=31':6N?A>@=931>@8;LH><>319?M?;LB=M3189>M6NK9671 \ No newline at end of file diff --git a/parquet-column/src/test/resources/fsst/sampled.table b/parquet-column/src/test/resources/fsst/sampled.table new file mode 100644 index 0000000000000000000000000000000000000000..699c4c918ad41569fd00fef1b1297563390e0234 GIT binary patch literal 254 zcmXxeO%8%E5Qbr+vhf5aZsrJ`{=&jLc!dIm778^@As$~sVK46? z-`RR@hiU8^ZIG4~7m^VbhlMImL{$>XsYJypiV7L41edaugi$Gk3N}lxDsDjtL P;VtiY&j&v8J**/dependency-reduced-pom.xml **/*.rej **/src/main/thrift/parquet-format.version + + **/src/test/resources/fsst/*.table + **/src/test/resources/fsst/*.codes From 52b0461bd4c34746043c3414c4c0fd514cb8dea1 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 19:21:26 +0000 Subject: [PATCH 02/10] Add the symbol table data page body and the table handoff seams The page body is the part of this encoding two implementations have to agree on byte for byte, so it is one class with the framing in one place, and it is the same class for every codec built on a symbol table: a code stream plus one end offset per value, behind a fixed nine-byte header. End offsets rather than lengths, because that is what makes a value readable without expanding the ones in front of it, which is the property a codec over a page cannot give and the reason for preferring this over compressing the page whole. The offset section is written through the encoders this module already has rather than by hand, since plain int32 and delta binary packing are both spelled out here already and a second spelling of either would be a second thing to keep compatible. Delta offsets are offered alongside plain ones because on a page of short values the offsets are a large fraction of the payload and almost all of it is the same increment repeated; a test measures that, and it is the argument for making delta the default when the writer lands. The read side validates before it trusts: the header against the page header's own value count, the offset section's length against what the encoding implies, and the offsets against the code section they partition, including the case where they stop short and leave bytes no value can reach. Every later read depends on those bounds, so they are checked once rather than per value. Twelve tests cover the layout byte for byte and each rejection. A symbol table belongs to a column chunk rather than to a page, and a values writer cannot write anything outside its own page, so the writer publishes the trained table through a sink and the reader takes it from a source. Both are one method. That is a test harness today and a page of its own once the format carries one, and neither choice reaches the codec. Alongside them, one class maps a table representation to the code that implements it, so a second representation is a new implementation plus two lines of dispatch rather than a change to anything above. --- .../symboltable/SymbolTablePayload.java | 217 +++++++++++++++ .../symboltable/SymbolTablePayloadWriter.java | 128 +++++++++ .../values/symboltable/SymbolTableSink.java | 45 ++++ .../values/symboltable/SymbolTableSource.java | 37 +++ .../values/symboltable/SymbolTables.java | 60 +++++ .../symboltable/SymbolTablePayloadTest.java | 247 ++++++++++++++++++ 6 files changed, 734 insertions(+) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayload.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadWriter.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayload.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayload.java new file mode 100644 index 0000000000..be99940954 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayload.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * A data page body holding a code stream and the boundaries between its values. + * + *

The layout is the same for every symbol table codec: + * + * 

+ *   [1B] offset encoding: 0 = PLAIN, 1 = DELTA_BINARY_PACKED
+ *   [4B] number of values, little-endian
+ *   [4B] byte length of the offset section, little-endian
+ *   [..] one end offset into the code stream per value
+ *   [..] the code stream
+ *
+ * + *

Offsets are ends rather than starts, so a value's codes run from the previous end to its own + * and the first value starts at zero. That is what makes a value readable without expanding the + * ones in front of it, and it is why a reader that only wants the tenth value pays for the offsets + * alone. + * + *

The number of values restates a field the page header already carries. Both copies are read + * and the page header's is treated as the bound: a payload may describe fewer values than the page + * holds, never more. + * + *

This class is the read view. {@link SymbolTablePayloadWriter} produces the same layout. + */ +public final class SymbolTablePayload { + + /** + * How the offset section is encoded. + * + *

Delta encoding is worth having here rather than being a tuning knob: offsets grow with the + * page, so on a page whose values compress to a handful of bytes each, four bytes of offset per + * value can outweigh the codes they point at. + */ + public enum OffsetEncoding { + PLAIN(0), + DELTA_BINARY_PACKED(1); + + private final int value; + + OffsetEncoding(int value) { + this.value = value; + } + + /** The value written as the payload's first byte. */ + public int value() { + return value; + } + + static OffsetEncoding fromValue(int value) { + for (OffsetEncoding encoding : values()) { + if (encoding.value == value) { + return encoding; + } + } + throw new ParquetDecodingException("Unsupported symbol table offset encoding: " + value); + } + } + + /** Size of the fixed part: the offset encoding byte, the value count, the offset section length. */ + public static final int HEADER_SIZE = 9; + + private final int valueCount; + private final byte[] codes; + private final int codesOffset; + private final int[] endOffsets; + + private SymbolTablePayload(int valueCount, byte[] codes, int codesOffset, int[] endOffsets) { + this.valueCount = valueCount; + this.codes = codes; + this.codesOffset = codesOffset; + this.endOffsets = endOffsets; + } + + /** + * Reads a payload, consuming the rest of the stream. + * + * @param in positioned at the payload's first byte + * @param pageValueCount the value count from the page header, an upper bound on the payload's own + */ + public static SymbolTablePayload parse(ByteBufferInputStream in, int pageValueCount) throws IOException { + int length = in.available(); + if (length < HEADER_SIZE) { + throw new ParquetDecodingException( + "Symbol table page body is shorter than its " + HEADER_SIZE + "-byte header: " + length); + } + ByteBuffer header = in.slice(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + OffsetEncoding offsetEncoding = OffsetEncoding.fromValue(header.get(header.position()) & 0xFF); + int valueCount = header.getInt(header.position() + 1); + int offsetSectionSize = header.getInt(header.position() + 5); + + if (valueCount < 0 || valueCount > pageValueCount) { + throw new ParquetDecodingException("Symbol table page describes " + valueCount + + " values, but the page header says the page holds " + pageValueCount); + } + if (offsetSectionSize < 0 || offsetSectionSize > length - HEADER_SIZE) { + throw new ParquetDecodingException("Invalid symbol table offset section length: " + offsetSectionSize); + } + + int[] endOffsets = readOffsets(in, offsetEncoding, valueCount, offsetSectionSize); + + int codeSectionSize = length - HEADER_SIZE - offsetSectionSize; + ByteBuffer codeSection = in.slice(codeSectionSize); + byte[] codes; + int codesOffset; + if (codeSection.hasArray()) { + codes = codeSection.array(); + codesOffset = codeSection.arrayOffset() + codeSection.position(); + } else { + codes = new byte[codeSectionSize]; + codeSection.get(codes); + codesOffset = 0; + } + + checkOffsets(endOffsets, codeSectionSize); + return new SymbolTablePayload(valueCount, codes, codesOffset, endOffsets); + } + + private static int[] readOffsets( + ByteBufferInputStream in, OffsetEncoding offsetEncoding, int valueCount, int offsetSectionSize) + throws IOException { + if (valueCount == 0) { + if (offsetSectionSize != 0) { + throw new ParquetDecodingException( + "Symbol table page holds no values but has a " + offsetSectionSize + "-byte offset section"); + } + return new int[0]; + } + int[] endOffsets = new int[valueCount]; + if (offsetEncoding == OffsetEncoding.PLAIN) { + long expected = (long) valueCount * Integer.BYTES; + if (offsetSectionSize != expected) { + throw new ParquetDecodingException("Symbol table PLAIN offset section is " + offsetSectionSize + + " bytes; expected " + expected + " for " + valueCount + " values"); + } + ByteBuffer offsets = in.slice(offsetSectionSize).order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < valueCount; i++) { + endOffsets[i] = offsets.getInt(offsets.position() + i * Integer.BYTES); + } + } else { + DeltaBinaryPackingValuesReader offsets = new DeltaBinaryPackingValuesReader(); + offsets.initFromPage(valueCount, in.sliceStream(offsetSectionSize)); + for (int i = 0; i < valueCount; i++) { + endOffsets[i] = offsets.readInteger(); + } + } + return endOffsets; + } + + /** + * Rejects offsets that do not describe a partition of the code section. + * + *

Every later read trusts these bounds, so they are checked once here rather than per value. + * A payload whose offsets stop short of the code section is rejected too: the leftover bytes + * would be unreachable, which means the payload is not the one the writer produced. + */ + private static void checkOffsets(int[] endOffsets, int codeSectionSize) { + int previous = 0; + for (int endOffset : endOffsets) { + if (endOffset < previous || endOffset > codeSectionSize) { + throw new ParquetDecodingException("Symbol table offsets are not monotonic within a " + codeSectionSize + + "-byte code section: " + previous + " then " + endOffset); + } + previous = endOffset; + } + if (previous != codeSectionSize) { + throw new ParquetDecodingException("Symbol table offsets end at " + previous + " but the code section is " + + codeSectionSize + " bytes"); + } + } + + /** How many values this payload holds, which may be fewer than the page holds. */ + public int valueCount() { + return valueCount; + } + + /** The array holding the code stream; {@link #codeStart} indexes into it. */ + public byte[] codes() { + return codes; + } + + /** Where this value's codes begin in {@link #codes}. */ + public int codeStart(int index) { + return codesOffset + (index == 0 ? 0 : endOffsets[index - 1]); + } + + /** How many bytes of codes belong to this value. */ + public int codeLength(int index) { + return index == 0 ? endOffsets[0] : endOffsets[index] - endOffsets[index - 1]; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadWriter.java new file mode 100644 index 0000000000..f9efc7a617 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadWriter.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; +import org.apache.parquet.column.values.plain.PlainValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; + +/** + * Builds the data page body {@link SymbolTablePayload} reads. + * + *

Holds the code stream and the offsets, so a codec's writer only has to compress values and + * hand the codes over. The layout is documented on the read side. + */ +public final class SymbolTablePayloadWriter implements AutoCloseable { + + private final OffsetEncoding offsetEncoding; + private final CapacityByteArrayOutputStream codes; + + /** + * The offset section, written through an existing encoder rather than by hand. + * + *

Plain int32 and delta binary packing are both already spelled out elsewhere in this module, + * and a second spelling of either would be a second thing to keep byte-compatible. + */ + private final ValuesWriter offsetWriter; + + private int valueCount; + + public SymbolTablePayloadWriter( + OffsetEncoding offsetEncoding, int initialSlabSize, int pageSize, ByteBufferAllocator allocator) { + this.offsetEncoding = offsetEncoding; + this.codes = new CapacityByteArrayOutputStream(initialSlabSize, pageSize, allocator); + this.offsetWriter = offsetEncoding == OffsetEncoding.PLAIN + ? new PlainValuesWriter(initialSlabSize, pageSize, allocator) + : new DeltaBinaryPackingValuesWriterForInteger( + DeltaBinaryPackingValuesWriter.DEFAULT_NUM_BLOCK_VALUES, + DeltaBinaryPackingValuesWriter.DEFAULT_NUM_MINIBLOCKS, + initialSlabSize, + pageSize, + allocator); + } + + /** Appends one value's codes and records where it ends. */ + public void addValue(byte[] valueCodes, int offset, int length) { + codes.write(valueCodes, offset, length); + long end = codes.size(); + if (end > Integer.MAX_VALUE) { + // The offsets are int32 on the wire, so this is a format limit rather than a Java one. + throw new IllegalStateException("Symbol table code stream exceeds 2 GB: " + end); + } + offsetWriter.writeInteger((int) end); + valueCount++; + } + + public int valueCount() { + return valueCount; + } + + /** Bytes buffered so far, excluding the fixed header, for a caller watching the page size. */ + public long bufferedSize() { + return codes.size() + offsetWriter.getBufferedSize(); + } + + public long allocatedSize() { + return codes.getCapacity() + offsetWriter.getAllocatedSize(); + } + + /** + * The finished page body. + * + *

A page holding no values gets an empty offset section rather than an encoder's empty-input + * output, because the offset encoding's own framing would otherwise be the only thing in the + * section and a reader has no values to spend it on. + */ + public BytesInput getBytes() { + BytesInput offsets = valueCount == 0 ? BytesInput.empty() : offsetWriter.getBytes(); + long offsetSectionSize = offsets.size(); + if (offsetSectionSize > Integer.MAX_VALUE) { + throw new IllegalStateException("Symbol table offset section exceeds 2 GB: " + offsetSectionSize); + } + byte[] header = new byte[SymbolTablePayload.HEADER_SIZE]; + header[0] = (byte) offsetEncoding.value(); + writeIntLittleEndian(header, 1, valueCount); + writeIntLittleEndian(header, 5, (int) offsetSectionSize); + return BytesInput.concat(BytesInput.from(header), offsets, BytesInput.from(codes)); + } + + private static void writeIntLittleEndian(byte[] destination, int position, int value) { + destination[position] = (byte) value; + destination[position + 1] = (byte) (value >>> 8); + destination[position + 2] = (byte) (value >>> 16); + destination[position + 3] = (byte) (value >>> 24); + } + + public void reset() { + codes.reset(); + offsetWriter.reset(); + valueCount = 0; + } + + @Override + public void close() { + codes.close(); + offsetWriter.close(); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java new file mode 100644 index 0000000000..0190aa4d16 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import org.apache.parquet.bytes.BytesInput; + +/** + * Where a writer hands off the symbol table it trained. + * + *

A symbol table belongs to a column chunk rather than to a page: every page of the chunk is + * compressed against it, and it has to be readable before any of them. A values writer cannot write + * anything outside its own page, so it publishes the table here instead and something above it + * decides where the bytes go. + * + *

That indirection is the point. It is a test harness today, and a page written next to the + * dictionary page once the format carries one, and neither choice reaches the writer. + */ +public interface SymbolTableSink { + + /** + * Publishes the table a column chunk's pages are compressed against. + * + *

Called once per chunk, before the first page that uses the table. + * + * @param type the representation, which a reader needs in order to interpret the body + * @param body the serialized table + */ + void putSymbolTable(SymbolTableType type, BytesInput body); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java new file mode 100644 index 0000000000..6335e5594d --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +/** + * Where a reader gets the symbol table a column chunk's pages were compressed against. + * + *

The counterpart of {@link SymbolTableSink}. The table arrives already deserialized, because + * which implementation to build from the bytes depends on the representation and that decision + * belongs in one place: {@link SymbolTables}. + */ +public interface SymbolTableSource { + + /** + * The table for the chunk being read. + * + *

Called before the first page. Implementations are expected to deserialize once and hand back + * the same table for every page of the chunk. + */ + SymbolTable getSymbolTable(); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java new file mode 100644 index 0000000000..3ea11380b9 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import org.apache.parquet.column.values.symboltable.fsst.Fsst8SymbolTable; +import org.apache.parquet.column.values.symboltable.fsst.FsstTrainer; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * The one place that maps a symbol table representation to the code that implements it. + * + *

Both directions live here so that a new representation is a new implementation plus two lines + * of dispatch, and so that a reader's rejection of a representation it does not implement happens in + * one place with one message. + */ +public final class SymbolTables { + + private SymbolTables() {} + + /** A trainer producing tables of this representation. */ + public static SymbolTableTrainer trainer(SymbolTableType type) { + switch (type) { + case FSST_8: + return new FsstTrainer(); + default: + throw new IllegalArgumentException("No symbol table trainer for " + type); + } + } + + /** + * Rebuilds a table from a serialized body. + * + * @throws ParquetDecodingException if this implementation cannot read the representation, which a + * reader is allowed to do and is not the same as the file being corrupt + */ + public static SymbolTable deserialize(SymbolTableType type, byte[] body, int offset, int length) { + switch (type) { + case FSST_8: + return Fsst8SymbolTable.deserialize(body, offset, length); + default: + throw new ParquetDecodingException("Unsupported symbol table type: " + type); + } + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java new file mode 100644 index 0000000000..2a31eb1aad --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.HeapByteBufferAllocator; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.io.ParquetDecodingException; +import org.junit.Test; + +/** + * The data page body's framing: what a writer produces byte for byte, and what a reader refuses. + * + *

The framing is the part of the encoding that two implementations have to agree on exactly, so + * the write-side assertions here are on bytes rather than on a round trip. + */ +public class SymbolTablePayloadTest { + + private static final int SLAB_SIZE = 64; + private static final int PAGE_SIZE = 1 << 20; + + private static SymbolTablePayloadWriter writer(OffsetEncoding offsetEncoding) { + return new SymbolTablePayloadWriter( + offsetEncoding, SLAB_SIZE, PAGE_SIZE, HeapByteBufferAllocator.getInstance()); + } + + private static byte[] write(OffsetEncoding offsetEncoding, byte[]... values) throws IOException { + try (SymbolTablePayloadWriter writer = writer(offsetEncoding)) { + for (byte[] value : values) { + writer.addValue(value, 0, value.length); + } + assertEquals(values.length, writer.valueCount()); + return writer.getBytes().toByteArray(); + } + } + + private static SymbolTablePayload parse(byte[] body, int pageValueCount) throws IOException { + return SymbolTablePayload.parse(ByteBufferInputStream.wrap(ByteBuffer.wrap(body)), pageValueCount); + } + + /** Reads back every value, so a bound that is off by one shows up as wrong bytes. */ + private static List valuesOf(SymbolTablePayload payload) { + List values = new ArrayList<>(); + for (int i = 0; i < payload.valueCount(); i++) { + byte[] value = new byte[payload.codeLength(i)]; + System.arraycopy(payload.codes(), payload.codeStart(i), value, 0, value.length); + values.add(value); + } + return values; + } + + @Test + public void writesThePlainLayoutByteForByte() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2, 3), bytes(4), bytes(5, 6)); + + ByteBuffer expected = ByteBuffer.allocate(9 + 12 + 6).order(ByteOrder.LITTLE_ENDIAN); + expected.put((byte) 0); // PLAIN offsets + expected.putInt(3); // number of values + expected.putInt(12); // offset section length + expected.putInt(3); // end of the first value + expected.putInt(4); + expected.putInt(6); + expected.put(bytes(1, 2, 3, 4, 5, 6)); + assertArrayEquals(expected.array(), body); + } + + @Test + public void writesTheDeltaLayoutWithTheSameHeaderShape() throws IOException { + byte[] body = write(OffsetEncoding.DELTA_BINARY_PACKED, bytes(1, 2, 3), bytes(4), bytes(5, 6)); + + ByteBuffer header = ByteBuffer.wrap(body, 0, 9).order(ByteOrder.LITTLE_ENDIAN); + assertEquals(1, header.get() & 0xFF); + assertEquals(3, header.getInt()); + int offsetSectionSize = header.getInt(); + assertEquals(body.length - 9 - 6, offsetSectionSize); + assertArrayEquals(bytes(1, 2, 3, 4, 5, 6), java.util.Arrays.copyOfRange(body, body.length - 6, body.length)); + } + + @Test + public void roundTripsBothOffsetEncodings() throws IOException { + for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { + byte[] body = write(offsetEncoding, bytes(1, 2, 3), bytes(4), bytes(5, 6)); + SymbolTablePayload payload = parse(body, 3); + + assertEquals(3, payload.valueCount()); + List values = valuesOf(payload); + assertArrayEquals(bytes(1, 2, 3), values.get(0)); + assertArrayEquals(bytes(4), values.get(1)); + assertArrayEquals(bytes(5, 6), values.get(2)); + } + } + + /** + * Values of zero length are ordinary: two offsets that are equal, not a special case. + */ + @Test + public void roundTripsValuesWithNoCodes() throws IOException { + for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { + byte[] body = write(offsetEncoding, bytes(), bytes(7), bytes(), bytes()); + SymbolTablePayload payload = parse(body, 4); + + assertEquals(4, payload.valueCount()); + List values = valuesOf(payload); + assertEquals(0, values.get(0).length); + assertArrayEquals(bytes(7), values.get(1)); + assertEquals(0, values.get(2).length); + assertEquals(0, values.get(3).length); + } + } + + @Test + public void aPageWithNoValuesIsJustAHeader() throws IOException { + for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { + byte[] body = write(offsetEncoding); + assertEquals(SymbolTablePayload.HEADER_SIZE, body.length); + assertEquals(0, parse(body, 0).valueCount()); + } + } + + /** + * Why delta offsets are the default rather than a knob: on a page of short values the plain + * section is a large fraction of the payload, and almost all of it is the same increment repeated. + */ + @Test + public void deltaOffsetsCostFarLessThanPlainOnesOnAPageOfShortValues() throws IOException { + byte[][] values = new byte[2000][]; + for (int i = 0; i < values.length; i++) { + values[i] = bytes(i & 0xFF, (i >>> 8) & 0xFF, 0x2C, 0x2E); + } + int plain = write(OffsetEncoding.PLAIN, values).length; + int delta = write(OffsetEncoding.DELTA_BINARY_PACKED, values).length; + + int codeBytes = values.length * 4; + assertEquals(9 + values.length * 4 + codeBytes, plain); + assertTrue( + "delta offsets should cost a small fraction of plain ones, but the payloads were " + delta + " and " + + plain, + delta - codeBytes < (plain - codeBytes) / 8); + } + + @Test + public void rejectsABodyShorterThanTheHeader() { + ParquetDecodingException thrown = assertThrows(ParquetDecodingException.class, () -> parse(new byte[8], 1)); + assertTrue(thrown.getMessage().contains("shorter than its 9-byte header")); + } + + @Test + public void rejectsAnUnknownOffsetEncoding() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1)); + body[0] = 2; + assertThrows(ParquetDecodingException.class, () -> parse(body, 1)); + } + + @Test + public void rejectsMoreValuesThanThePageHeaderDeclares() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1), bytes(2)); + assertThrows(ParquetDecodingException.class, () -> parse(body, 1)); + } + + @Test + public void rejectsAnOffsetSectionLongerThanTheBody() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1)); + putInt(body, 5, 1000); + assertThrows(ParquetDecodingException.class, () -> parse(body, 1)); + } + + @Test + public void rejectsAPlainOffsetSectionOfTheWrongLength() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); + // Claim one value's worth of offsets while the section holds two. + putInt(body, 1, 1); + assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + } + + @Test + public void rejectsOffsetsThatGoBackwards() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); + putInt(body, 9, 3); // the first value ends past where the second one does + putInt(body, 13, 1); + assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + } + + @Test + public void rejectsOffsetsThatLeaveCodeBytesUnreachable() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); + putInt(body, 13, 2); // the last value's codes are dropped + assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + } + + @Test + public void rejectsOffsetsThatRunPastTheCodeSection() throws IOException { + byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); + putInt(body, 13, 4); + assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + } + + @Test + public void rejectsAnEmptyPageCarryingAnOffsetSection() throws IOException { + ByteArrayOutputStream body = new ByteArrayOutputStream(); + byte[] header = new byte[9]; + header[0] = (byte) OffsetEncoding.DELTA_BINARY_PACKED.value(); + putInt(header, 1, 0); + putInt(header, 5, 4); + body.write(header); + body.write(new byte[4]); + assertThrows(ParquetDecodingException.class, () -> parse(body.toByteArray(), 0)); + } + + private static void putInt(byte[] destination, int position, int value) { + ByteBuffer.wrap(destination).order(ByteOrder.LITTLE_ENDIAN).putInt(position, value); + } + + private static byte[] bytes(int... values) { + byte[] result = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = (byte) values[i]; + } + return result; + } +} From d0607bcbbf4129653b0acbc0b4e820be0e60cd22 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 19:36:45 +0000 Subject: [PATCH 03/10] Encode and decode pages against a symbol table Adds the values writer and reader that turn buffered binary values into a code stream over a trained symbol table, and back. One writer and one reader serve every symbol table representation. The representation decides how a table is trained, serialized and framed, and all three already sit behind seams, so nothing about them reaches these two classes beyond the type they are asked for. The awkward part of the encoding is that a table belongs to a column chunk while a writer only ever sees a page. The writer trains at the first page, publishes the table through the sink, and keeps it for the rest of the chunk; a later page must not train its own or the reader would decode it wrongly. Values are buffered raw because a trainer reads them in an order of its own choosing and more than once, and because a fallback to another encoding has to replay them. Compressing a page can also make it bigger, which is what the fallback contract is for: whether the codes came out smaller than the values is the only question worth asking, and it is asked once. A page of single bytes with plain offsets is the case that answers no -- and the same page with packed offsets answers yes, which is the argument against writing offsets plain. Encoding.FSST hands out the reader for BINARY and rejects every other type. The reader it hands out has nowhere to get its table from, because the format has nowhere to put one yet; it says so rather than failing later. --- .../org/apache/parquet/column/Encoding.java | 21 + .../values/symboltable/SymbolTableSink.java | 5 +- .../symboltable/SymbolTableValuesReader.java | 118 +++++ .../symboltable/SymbolTableValuesWriter.java | 208 ++++++++ .../values/symboltable/ValueBuffer.java | 5 + .../SymbolTableValuesRoundTripTest.java | 487 ++++++++++++++++++ 6 files changed, 843 insertions(+), 1 deletion(-) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesReader.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java b/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java index 874c99fded..64a172df26 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java @@ -54,6 +54,7 @@ import org.apache.parquet.column.values.plain.PlainValuesReader.LongPlainValuesReader; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesReader; import org.apache.parquet.column.values.rle.ZeroIntegerValuesReader; +import org.apache.parquet.column.values.symboltable.SymbolTableValuesReader; import org.apache.parquet.io.ParquetDecodingException; /** @@ -253,6 +254,26 @@ public ValuesReader getDictionaryBasedValuesReader( public boolean usesDictionary() { return true; } + }, + + /** + * Values are replaced by codes over a table of the byte sequences that recur in the column, with + * one table per column chunk. The table's own representation decides the width of a code and how a + * byte that no symbol covers is escaped, so this one encoding covers every such representation and + * a reader has to read the table before it can commit to decoding the column. + *

+ * Not ratified: parquet-format issue #531. A writer will not produce this encoding unless it is + * turned on explicitly, and no file written by this library carries it yet, because the format has + * nowhere to put the table. + */ + FSST { + @Override + public ValuesReader getValuesReader(ColumnDescriptor descriptor, ValuesType valuesType) { + if (descriptor.getType() != BINARY) { + throw new ParquetDecodingException("Encoding FSST is only supported for type BINARY"); + } + return new SymbolTableValuesReader(); + } }; int getMaxLevel(ColumnDescriptor descriptor, ValuesType valuesType) { diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java index 0190aa4d16..cf9a3e1242 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java @@ -36,7 +36,10 @@ public interface SymbolTableSink { /** * Publishes the table a column chunk's pages are compressed against. * - *

Called once per chunk, before the first page that uses the table. + *

Called once per chunk, when the first page is compressed. A chunk that then abandons the + * encoding — because the codes did not come out smaller than the values — leaves a table behind + * that no page refers to, so whoever stores it should write it only if some page of the chunk was + * actually written with the encoding. * * @param type the representation, which a reader needs in order to interpret the body * @param body the serialized table diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesReader.java new file mode 100644 index 0000000000..e53193b045 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesReader.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import java.io.IOException; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.api.Binary; + +/** + * Reads the pages {@link SymbolTableValuesWriter} writes. + * + *

One reader serves every symbol table representation, for the same reason one writer does: the + * page body is the same shape whatever the table holds, and expanding a value's codes is the table's + * own business. Which representation a chunk used is carried with the table rather than by the + * column's encoding, so this reader learns it from the {@link SymbolTableSource} and never has to + * guess. + * + *

A value is addressable without expanding the ones before it, because the page records where + * each value's codes end. That is what makes {@link #skip} cost nothing and is the property the + * encoding exists for. + */ +public class SymbolTableValuesReader extends ValuesReader { + + private final SymbolTableSource source; + + private SymbolTable table; + private CodeStreamDecoder decoder; + private SymbolTablePayload payload; + private int index; + + /** + * A reader with nowhere to get its table from, which cannot decode a page. + * + *

The format does not carry a symbol table yet, so there is no place for a page reader to find + * one and nothing to hand this constructor. It exists so that the encoding is complete on the read + * side up to that one missing piece; supply a source and the reader works. See parquet-format issue + * #531. + */ + public SymbolTableValuesReader() { + this(null); + } + + public SymbolTableValuesReader(SymbolTableSource source) { + this.source = source; + } + + @Override + public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException { + if (table == null) { + if (source == null) { + throw new ParquetDecodingException( + "Cannot decode a symbol table encoded page without the chunk's symbol table, which the " + + "format does not carry yet: see parquet-format issue #531"); + } + // Once per chunk. The table outlives the page: every page of the chunk shares it. + table = source.getSymbolTable(); + decoder = table.decoder(); + } + payload = SymbolTablePayload.parse(in, valueCount); + index = 0; + } + + /** The representation of the table this reader is decoding against. */ + public SymbolTableType symbolTableType() { + return table == null ? null : table.type(); + } + + @Override + public Binary readBytes() { + checkHasValue(1); + byte[] codes = payload.codes(); + int offset = payload.codeStart(index); + int length = payload.codeLength(index); + index++; + byte[] value = new byte[decoder.expandedLength(codes, offset, length)]; + decoder.expand(codes, offset, length, value, 0); + return Binary.fromConstantByteArray(value); + } + + @Override + public void skip() { + skip(1); + } + + @Override + public void skip(int n) { + checkHasValue(n); + index += n; + } + + private void checkHasValue(int n) { + if (payload == null) { + throw new ParquetDecodingException("Symbol table reader used before a page was read"); + } + if (index + n > payload.valueCount()) { + throw new ParquetDecodingException("Read past the end of a symbol table page: asked for value " + + (index + n) + " of " + payload.valueCount()); + } + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java new file mode 100644 index 0000000000..70b37440d9 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.RequiresFallback; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.io.api.Binary; + +/** + * Writes binary values as codes over a symbol table trained on the data. + * + *

One writer serves every symbol table representation. The representation decides how a table is + * trained, how it is serialized and how a code stream is framed, and all three sit behind + * {@link SymbolTableTrainer}, {@link SymbolTable} and {@link CodeStreamEncoder}; nothing about them + * reaches this class beyond the {@link SymbolTableType} it is asked for. + * + *

What happens when

+ * + *

Values are buffered raw as they arrive. The table is trained at the first {@link #getBytes()}, + * on that page's values, and then published through the {@link SymbolTableSink} and kept for the + * rest of the chunk: a table belongs to a column chunk, so a later page must not train its own. + * Training on the first page rather than on the whole chunk is deliberate — a page is already far + * more text than a trainer samples, and buffering the chunk to feed it would cost a second copy of + * the column. + * + *

Buffering is the reason values are held at all: a trainer reads them in an order of its own + * choosing and more than once, and a fallback to another encoding has to replay them. The cost is + * the page held twice, once raw and once as codes, which is what {@link #getAllocatedSize()} + * reports. + * + *

Falling back

+ * + *

Compressing a page can make it bigger — short values, or values that share nothing with the + * rest of the column. Wrapping this writer in a + * {@link org.apache.parquet.column.values.fallback.FallbackValuesWriter} over a plain writer is what + * makes the encoding safe to turn on: {@link #isCompressionSatisfying} answers the only question + * that matters, which is whether the codes came out smaller than the values. + */ +public class SymbolTableValuesWriter extends ValuesWriter implements RequiresFallback { + + private final SymbolTableType type; + private final SymbolTableSink sink; + private final ValueBuffer values; + private final SymbolTablePayloadWriter payload; + + /** The chunk's table, trained at the first {@link #getBytes()} and reused after that. */ + private TrainedSymbolTable trained; + + /** Scratch for one value's codes, grown as needed and reused across values. */ + private byte[] codes = new byte[0]; + + /** + * How many of the buffered values have been compressed into {@link #payload}. + * + *

Compression happens at {@link #getBytes()}, because it cannot start before the table exists. + * Counting what is already done rather than flagging it keeps a second call correct whether or not + * more values arrived in between. + */ + private int compressedCount; + + public SymbolTableValuesWriter( + SymbolTableType type, + SymbolTableSink sink, + OffsetEncoding offsetEncoding, + int initialSlabSize, + int pageSize, + ByteBufferAllocator allocator) { + this.type = type; + this.sink = sink; + this.values = new ValueBuffer(); + this.payload = new SymbolTablePayloadWriter(offsetEncoding, initialSlabSize, pageSize, allocator); + } + + public SymbolTableType symbolTableType() { + return type; + } + + @Override + public void writeBytes(Binary v) { + values.add(v); + } + + /** + * The size of the values buffered for this page, counted as a plain page would count them. + * + *

Not the compressed size, which is not known until the table has been trained. A page boundary + * has to be decided while values are still arriving, so it is decided on what the page would cost + * unencoded — the same choice {@link org.apache.parquet.column.values.fallback.FallbackValuesWriter} + * makes, and for the same reason: a page sized by its compressed length is a page that becomes too + * big the moment the encoding is abandoned. + */ + @Override + public long getBufferedSize() { + return values.byteCount() + 4L * values.valueCount(); + } + + @Override + public BytesInput getBytes() { + if (trained == null) { + trained = SymbolTables.trainer(type).train(values); + sink.putSymbolTable(type, trained.table().serialize()); + } + compressBufferedValues(); + return payload.getBytes(); + } + + @Override + public Encoding getEncoding() { + return Encoding.FSST; + } + + @Override + public void reset() { + values.reset(); + payload.reset(); + compressedCount = 0; + } + + /** + * Drops the table, which is what a new column chunk needs. + * + *

Named for the dictionary because that is the only chunk-scoped state a values writer had + * before this one. In the file writer a values writer does not outlive its chunk, so this is + * belt-and-braces rather than the path that runs. + */ + @Override + public void resetDictionary() { + trained = null; + } + + @Override + public void close() { + payload.close(); + } + + @Override + public long getAllocatedSize() { + return values.allocatedSize() + payload.allocatedSize() + codes.length; + } + + @Override + public String memUsageString(String prefix) { + return String.format( + "%s %s{raw %d bytes, codes %d bytes}", prefix, type, values.byteCount(), payload.bufferedSize()); + } + + // RequiresFallback + + /** + * Never, because there is no state here that can run away. + * + *

This is the check a dictionary needs, where the encoding stops paying once the dictionary + * outgrows the data. A symbol table is bounded by its representation whatever the data does, so + * the only question worth asking is the one {@link #isCompressionSatisfying} asks, once, when + * there is an answer to it. + */ + @Override + public boolean shouldFallBack() { + return false; + } + + @Override + public boolean isCompressionSatisfying(long rawSize, long encodedSize) { + return encodedSize < rawSize; + } + + @Override + public void fallBackAllValuesTo(ValuesWriter writer) { + byte[] data = values.data(); + for (int i = 0; i < values.valueCount(); i++) { + writer.writeBytes(Binary.fromReusedByteArray(data, values.offset(i), values.length(i))); + } + } + + private void compressBufferedValues() { + CodeStreamEncoder encoder = trained.encoder(); + byte[] data = values.data(); + for (int i = compressedCount; i < values.valueCount(); i++) { + int length = values.length(i); + int bound = encoder.maxCompressedLength(length); + if (codes.length < bound) { + codes = new byte[Math.max(bound, codes.length * 2)]; + } + payload.addValue(codes, 0, encoder.compress(data, values.offset(i), length, codes, 0)); + } + compressedCount = values.valueCount(); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java index fad31d5b5a..b6209c7f47 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/ValueBuffer.java @@ -86,6 +86,11 @@ public int byteCount() { return size; } + /** Bytes held by the two arrays, which is what this buffer costs a writer's memory budget. */ + public long allocatedSize() { + return data.length + 4L * offsets.length; + } + /** * The backing array. Valid from 0 to {@link #byteCount()}, followed by {@link #TAIL_PADDING} zero * bytes that a reader may load but must not interpret. diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java new file mode 100644 index 0000000000..aea8edabd5 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.HeapByteBufferAllocator; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.ValuesType; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.fallback.FallbackValuesWriter; +import org.apache.parquet.column.values.plain.BinaryPlainValuesReader; +import org.apache.parquet.column.values.plain.PlainValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.junit.Test; + +/** + * The writer and reader as a pair: values in, the same values out, over the seams that a symbol table + * has to cross. + * + *

The table lives with the column chunk while a page is what a writer produces, and that mismatch + * is where the interesting cases are — a table trained on the first page and used by the fifth, a + * chunk that abandons the encoding after the table has already been trained, and a reader that has to + * be handed the table from outside the page it is reading. + */ +public class SymbolTableValuesRoundTripTest { + + private static final int SLAB_SIZE = 1024; + private static final int PAGE_SIZE = 1 << 20; + + /** + * Stands in for wherever the serialized table ends up. + * + *

Both halves of the seam, so a test can hand the writer's own table straight back to the + * reader. Deserializing on every call is deliberate: it means the reader is decoding against a table + * rebuilt from bytes rather than against the trainer's own object. + */ + private static final class SymbolTableRelay implements SymbolTableSink, SymbolTableSource { + + private SymbolTableType type; + private byte[] body; + private int publishCount; + + @Override + public void putSymbolTable(SymbolTableType type, BytesInput body) { + this.type = type; + try { + this.body = body.toByteArray(); + } catch (IOException e) { + throw new AssertionError(e); + } + this.publishCount++; + } + + @Override + public SymbolTable getSymbolTable() { + return SymbolTables.deserialize(type, body, 0, body.length); + } + } + + private static SymbolTableValuesWriter writer(SymbolTableSink sink, OffsetEncoding offsetEncoding) { + return new SymbolTableValuesWriter( + SymbolTableType.FSST_8, + sink, + offsetEncoding, + SLAB_SIZE, + PAGE_SIZE, + HeapByteBufferAllocator.getInstance()); + } + + private static List binaries(String... values) { + List result = new ArrayList<>(); + for (String value : values) { + result.add(Binary.fromString(value)); + } + return result; + } + + /** Writes each page, reads each page back, and returns what came out. */ + private static List> roundTrip(List> pages, OffsetEncoding offsetEncoding) + throws IOException { + SymbolTableRelay relay = new SymbolTableRelay(); + List bodies = new ArrayList<>(); + try (SymbolTableValuesWriter writer = writer(relay, offsetEncoding)) { + for (List page : pages) { + for (Binary value : page) { + writer.writeBytes(value); + } + assertEquals(Encoding.FSST, writer.getEncoding()); + bodies.add(writer.getBytes().toByteArray()); + writer.reset(); + } + } + assertEquals("one table for the whole chunk", 1, relay.publishCount); + + SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); + List> read = new ArrayList<>(); + for (int i = 0; i < pages.size(); i++) { + int valueCount = pages.get(i).size(); + reader.initFromPage(valueCount, ByteBufferInputStream.wrap(ByteBuffer.wrap(bodies.get(i)))); + List page = new ArrayList<>(); + for (int v = 0; v < valueCount; v++) { + page.add(reader.readBytes()); + } + read.add(page); + } + return read; + } + + private static void assertRoundTrips(List values) throws IOException { + for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { + assertEquals( + "offsets " + offsetEncoding, + Arrays.asList(values), + roundTrip(Arrays.asList(values), offsetEncoding)); + } + } + + @Test + public void roundTripsTextThatSharesSubstrings() throws IOException { + List values = new ArrayList<>(); + for (int i = 0; i < 500; i++) { + values.add(Binary.fromString("https://example.com/catalogue/item/" + i + "?ref=newsletter")); + } + assertRoundTrips(values); + } + + @Test + public void roundTripsAPageWithNoValues() throws IOException { + assertRoundTrips(binaries()); + } + + @Test + public void roundTripsASingleValue() throws IOException { + assertRoundTrips(binaries("only")); + } + + @Test + public void roundTripsEmptyStrings() throws IOException { + assertRoundTrips(binaries("", "", "", "")); + } + + @Test + public void roundTripsEmptyStringsMixedWithText() throws IOException { + assertRoundTrips(binaries("", "alpha", "", "", "alphabet", "a", "")); + } + + /** Every byte value, including the one the code space reserves for an escape. */ + @Test + public void roundTripsAllTwoHundredAndFiftySixByteValues() throws IOException { + List values = new ArrayList<>(); + for (int i = 0; i < 256; i++) { + values.add(Binary.fromConstantByteArray(new byte[] {(byte) i})); + } + values.add(Binary.fromConstantByteArray(allByteValues())); + assertRoundTrips(values); + } + + /** High-entropy bytes, which is the input that forces the escape path to carry the page. */ + @Test + public void roundTripsIncompressibleBytes() throws IOException { + Random random = new Random(20260908L); + List values = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + byte[] value = new byte[1 + random.nextInt(40)]; + random.nextBytes(value); + values.add(Binary.fromConstantByteArray(value)); + } + assertRoundTrips(values); + } + + /** Values far longer than the longest symbol a table can hold. */ + @Test + public void roundTripsValuesLongerThanAnySymbol() throws IOException { + List values = new ArrayList<>(); + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < 400; i++) { + builder.append("repetitionrepetition"); + values.add(Binary.fromString(builder.toString())); + } + assertRoundTrips(values); + } + + /** + * A table trained on the first page decodes the pages that follow it. + * + *

The pages deliberately drift: page one has none of the text page four is made of, so a table + * retrained per page would give a smaller page four, and a reader given only the first page's table + * would decode it wrongly. The point of the assertion is that the second thing does not happen. + */ + @Test + public void oneTableTrainedOnTheFirstPageServesLaterPages() throws IOException { + List> pages = new ArrayList<>(); + String[] themes = {"warehouse-inventory", "flight-departure", "clinical-observation", "seismic-reading"}; + for (int page = 0; page < themes.length; page++) { + List values = new ArrayList<>(); + for (int i = 0; i < 300; i++) { + values.add(Binary.fromString(themes[page] + "/" + i)); + } + pages.add(values); + } + for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { + assertEquals("offsets " + offsetEncoding, pages, roundTrip(pages, offsetEncoding)); + } + } + + /** A new chunk trains a new table, which is what a row group boundary needs. */ + @Test + public void resetDictionaryTrainsAgain() throws IOException { + SymbolTableRelay relay = new SymbolTableRelay(); + try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + for (Binary value : binaries("alpha", "alphabet", "alpine")) { + writer.writeBytes(value); + } + writer.getBytes(); + writer.reset(); + assertEquals(1, relay.publishCount); + + writer.resetDictionary(); + for (Binary value : binaries("zeta", "zenith", "zephyr")) { + writer.writeBytes(value); + } + writer.getBytes(); + assertEquals(2, relay.publishCount); + } + } + + @Test + public void skipReachesTheSameValuesAsReading() throws IOException { + List values = binaries("alpha", "", "alphabet", "beta", "betamax", "gamma", "gamma-ray", "delta"); + SymbolTableRelay relay = new SymbolTableRelay(); + byte[] body; + try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + for (Binary value : values) { + writer.writeBytes(value); + } + body = writer.getBytes().toByteArray(); + } + + for (int start = 0; start < values.size(); start++) { + SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); + reader.initFromPage(values.size(), ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); + reader.skip(start); + for (int i = start; i < values.size(); i++) { + assertEquals("from " + start + " at " + i, values.get(i), reader.readBytes()); + } + } + + // One value at a time, which is the overload a column reader calls for a null. + SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); + reader.initFromPage(values.size(), ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); + for (int i = 0; i < values.size(); i++) { + if (i % 2 == 0) { + reader.skip(); + } else { + assertEquals(values.get(i), reader.readBytes()); + } + } + } + + @Test + public void readingPastTheEndOfAPageIsRejected() throws IOException { + SymbolTableRelay relay = new SymbolTableRelay(); + byte[] body; + try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + writer.writeBytes(Binary.fromString("one")); + body = writer.getBytes().toByteArray(); + } + SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); + reader.initFromPage(1, ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); + reader.readBytes(); + assertThrows(ParquetDecodingException.class, reader::readBytes); + assertThrows(ParquetDecodingException.class, reader::skip); + } + + @Test + public void aReaderWithNoTableSaysSoRatherThanFailingLater() { + SymbolTableValuesReader reader = new SymbolTableValuesReader(); + assertNull(reader.symbolTableType()); + ParquetDecodingException thrown = assertThrows( + ParquetDecodingException.class, + () -> reader.initFromPage(1, ByteBufferInputStream.wrap(ByteBuffer.wrap(new byte[9])))); + assertTrue(thrown.getMessage().contains("#531")); + } + + @Test + public void theEncodingHandsOutTheReaderForBinaryOnly() { + assertTrue( + Encoding.FSST.getValuesReader(descriptor(PrimitiveTypeName.BINARY), ValuesType.VALUES) + instanceof SymbolTableValuesReader); + for (PrimitiveTypeName type : new PrimitiveTypeName[] { + PrimitiveTypeName.INT32, PrimitiveTypeName.INT64, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY + }) { + assertThrows( + ParquetDecodingException.class, + () -> Encoding.FSST.getValuesReader(descriptor(type), ValuesType.VALUES)); + } + } + + /** + * A page the encoding makes bigger is written plain instead, and the values still come back. + * + *

Single bytes with four-byte offsets: the codes are as long as the values and the offsets cost + * what a plain page's lengths cost, so the nine-byte header alone decides it. That is the + * configuration to fall back from, and it is also the argument against writing offsets plain — the + * same page with packed offsets is smaller than plain and keeps the encoding. + */ + @Test + public void aPageTheEncodingWouldGrowIsWrittenPlain() throws IOException { + List values = new ArrayList<>(); + for (int i = 0; i < 256; i++) { + values.add(Binary.fromConstantByteArray(new byte[] {(byte) i})); + } + + assertEquals(Encoding.PLAIN, fallbackEncodingFor(values, OffsetEncoding.PLAIN)); + assertEquals(Encoding.FSST, fallbackEncodingFor(values, OffsetEncoding.DELTA_BINARY_PACKED)); + } + + /** Runs a page through the fallback wrapper and reads it back with whatever encoding it chose. */ + private static Encoding fallbackEncodingFor(List values, OffsetEncoding offsetEncoding) throws IOException { + SymbolTableRelay relay = new SymbolTableRelay(); + Encoding encoding; + byte[] body; + try (FallbackValuesWriter writer = FallbackValuesWriter.of( + writer(relay, offsetEncoding), + new PlainValuesWriter(SLAB_SIZE, PAGE_SIZE, HeapByteBufferAllocator.getInstance()))) { + for (Binary value : values) { + writer.writeBytes(value); + } + body = writer.getBytes().toByteArray(); + encoding = writer.getEncoding(); + } + + ValuesReader reader = + encoding == Encoding.PLAIN ? new BinaryPlainValuesReader() : new SymbolTableValuesReader(relay); + reader.initFromPage(values.size(), ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); + for (Binary value : values) { + assertEquals(value, reader.readBytes()); + } + return encoding; + } + + /** Replaying buffered values into another writer must not depend on the encoding having run. */ + @Test + public void fallingBackReplaysEveryValue() throws IOException { + List values = binaries("alpha", "", "beta", "gamma"); + SymbolTableRelay relay = new SymbolTableRelay(); + List replayed = new ArrayList<>(); + ValuesWriter collector = new CollectingValuesWriter(replayed); + try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + for (Binary value : values) { + writer.writeBytes(value); + } + writer.fallBackAllValuesTo(collector); + } + assertEquals(values, replayed); + assertEquals(0, relay.publishCount); + } + + @Test + public void aTableIsPublishedOnceAndRebuiltFromItsBytes() throws IOException { + SymbolTableRelay relay = new SymbolTableRelay(); + try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + for (int i = 0; i < 200; i++) { + writer.writeBytes(Binary.fromString("measurement-" + i)); + } + writer.getBytes(); + } + assertEquals(SymbolTableType.FSST_8, relay.type); + SymbolTable first = relay.getSymbolTable(); + assertEquals(SymbolTableType.FSST_8, first.type()); + assertTrue("a table was trained", first.symbolCount() > 0); + assertEquals(first.symbolCount(), relay.getSymbolTable().symbolCount()); + } + + @Test + public void theWriterReportsWhatItIsHoldingAndReleasesItOnReset() throws IOException { + SymbolTableRelay relay = new SymbolTableRelay(); + try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + assertEquals(0, writer.getBufferedSize()); + writer.writeBytes(Binary.fromString("alpha")); + assertEquals(5 + 4, writer.getBufferedSize()); + writer.getBytes(); + writer.reset(); + assertEquals(0, writer.getBufferedSize()); + assertTrue(writer.getAllocatedSize() > 0); + assertTrue(writer.memUsageString("x").startsWith("x ")); + assertEquals(SymbolTableType.FSST_8, writer.symbolTableType()); + // The only fallback question is asked once, and never by the writer itself. + assertTrue(writer.isCompressionSatisfying(100, 99)); + assertFalse(writer.isCompressionSatisfying(100, 100)); + assertFalse(writer.shouldFallBack()); + } + } + + private static byte[] allByteValues() { + byte[] all = new byte[256]; + for (int i = 0; i < all.length; i++) { + all[i] = (byte) i; + } + return all; + } + + private static ColumnDescriptor descriptor(PrimitiveTypeName type) { + return new ColumnDescriptor(new String[] {"column"}, type, 0, 0); + } + + /** Records what a fallback replay hands it, so the replay can be compared value by value. */ + private static final class CollectingValuesWriter extends ValuesWriter { + + private final List values; + + CollectingValuesWriter(List values) { + this.values = values; + } + + @Override + public void writeBytes(Binary v) { + // Copy, because a replay is allowed to hand over a view of a buffer it will reuse. + values.add(Binary.fromString(new String(v.getBytes(), StandardCharsets.UTF_8))); + } + + @Override + public long getBufferedSize() { + return 0; + } + + @Override + public BytesInput getBytes() { + return BytesInput.empty(); + } + + @Override + public Encoding getEncoding() { + return Encoding.PLAIN; + } + + @Override + public void reset() { + values.clear(); + } + + @Override + public long getAllocatedSize() { + return 0; + } + + @Override + public String memUsageString(String prefix) { + return prefix; + } + } +} From b385edad69a4bf873ffaecb2d45fca1eea8ca9ae Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 19:56:12 +0000 Subject: [PATCH 04/10] Let a column ask for the symbol table encoding Turning the encoding on is a per-column property, off by default, and only BINARY columns honour it. The writer factories give the symbol table the first attempt at a column the dictionary has given up on, and hand back the columns whose codes come out no smaller than the values, so the writer stack is dictionary, then symbol table, then plain or delta byte array. The offsets into a page's code stream default to delta packing rather than four plain bytes a value. On text the encoding halves gets to roughly twenty bytes a value, so plain offsets are about a fifth of the page and delta packing takes them to near one byte; plain offsets are worth having only for a reader that wants them addressable without decoding. A writer built from the format's own settings gets a sink that refuses the table, because the format has nowhere to keep it and the pages would not be readable. It fails at the first page, while the failure can still name the reason. For the same reason converting the encoding to a footer value throws, which is why the two tests that convert every encoding value now skip it. --- .../parquet/column/ParquetProperties.java | 89 +++++++++++++++++++ .../factory/DefaultV1ValuesWriterFactory.java | 11 ++- .../factory/DefaultV2ValuesWriterFactory.java | 11 ++- .../factory/DefaultValuesWriterFactory.java | 27 ++++++ .../values/symboltable/SymbolTables.java | 21 +++++ .../parquet/column/TestParquetProperties.java | 58 ++++++++++++ .../DefaultValuesWriterFactoryTest.java | 65 ++++++++++++++ .../SymbolTableValuesRoundTripTest.java | 18 ++++ .../converter/ParquetMetadataConverter.java | 8 ++ .../parquet/hadoop/ParquetOutputFormat.java | 21 +++++ .../TestParquetMetadataConverter.java | 15 ++++ 11 files changed, 336 insertions(+), 8 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index 8fe45e01ef..6c1b4f2ec4 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -39,6 +39,8 @@ import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.column.values.symboltable.SymbolTableType; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.schema.MessageType; @@ -51,6 +53,8 @@ public class ParquetProperties { public static final int DEFAULT_DICTIONARY_PAGE_SIZE = DEFAULT_PAGE_SIZE; public static final boolean DEFAULT_IS_DICTIONARY_ENABLED = true; public static final boolean DEFAULT_IS_BYTE_STREAM_SPLIT_ENABLED = false; + public static final boolean DEFAULT_IS_FSST_ENABLED = false; + public static final OffsetEncoding DEFAULT_SYMBOL_TABLE_OFFSET_ENCODING = OffsetEncoding.DELTA_BINARY_PACKED; public static final WriterVersion DEFAULT_WRITER_VERSION = WriterVersion.PARQUET_1_0; public static final boolean DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK = true; public static final int DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK = 100; @@ -133,6 +137,8 @@ public static WriterVersion fromString(String name) { private final int pageRowCountLimit; private final boolean pageWriteChecksumEnabled; private final ColumnProperty byteStreamSplitEnabled; + private final ColumnProperty fsstEnabled; + private final OffsetEncoding symbolTableOffsetEncoding; private final Map extraMetaData; private final ColumnProperty statistics; private final ColumnProperty sizeStatistics; @@ -167,6 +173,8 @@ private ParquetProperties(Builder builder) { this.pageRowCountLimit = builder.pageRowCountLimit; this.pageWriteChecksumEnabled = builder.pageWriteChecksumEnabled; this.byteStreamSplitEnabled = builder.byteStreamSplitEnabled.build(); + this.fsstEnabled = builder.fsstEnabled.build(); + this.symbolTableOffsetEncoding = builder.symbolTableOffsetEncoding; this.extraMetaData = builder.extraMetaData; this.statistics = builder.statistics.build(); this.sizeStatistics = builder.sizeStatistics.build(); @@ -264,6 +272,37 @@ public boolean isByteStreamSplitEnabled(ColumnDescriptor column) { } } + /** + * Whether a symbol table encoding may be used for this column, which is only ever true for BINARY. + * + *

Off by default, and it has to stay off by default until the format carries a symbol table: + * see parquet-format issue #531. + */ + public boolean isFsstEnabled(ColumnDescriptor column) { + return getSymbolTableType(column) != null; + } + + /** + * The symbol table representation to write this column with, or null to not use one. + * + *

One encoding covers every representation, so this is what decides the width of a code and how a + * byte that no symbol covers is escaped, and it is where a choice between representations attaches + * once there is more than one to choose from. Only single-byte codes are implemented. + */ + public SymbolTableType getSymbolTableType(ColumnDescriptor column) { + switch (column.getPrimitiveType().getPrimitiveTypeName()) { + case BINARY: + return fsstEnabled.getValue(column) ? SymbolTableType.FSST_8 : null; + default: + return null; + } + } + + /** How the per-value offsets into a symbol table page's code stream are written. */ + public OffsetEncoding getSymbolTableOffsetEncoding() { + return symbolTableOffsetEncoding; + } + public ByteBufferAllocator getAllocator() { return allocator; } @@ -455,6 +494,8 @@ public static class Builder { private int pageRowCountLimit = DEFAULT_PAGE_ROW_COUNT_LIMIT; private boolean pageWriteChecksumEnabled = DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED; private final ColumnProperty.Builder byteStreamSplitEnabled; + private final ColumnProperty.Builder fsstEnabled; + private OffsetEncoding symbolTableOffsetEncoding = DEFAULT_SYMBOL_TABLE_OFFSET_ENCODING; private Map extraMetaData = new HashMap<>(); private final ColumnProperty.Builder statistics; private final ColumnProperty.Builder sizeStatistics; @@ -468,6 +509,7 @@ private Builder() { DEFAULT_IS_BYTE_STREAM_SPLIT_ENABLED ? ByteStreamSplitMode.FLOATING_POINT : ByteStreamSplitMode.NONE); + fsstEnabled = ColumnProperty.builder().withDefaultValue(DEFAULT_IS_FSST_ENABLED); bloomFilterEnabled = ColumnProperty.builder().withDefaultValue(DEFAULT_BLOOM_FILTER_ENABLED); bloomFilterNDVs = ColumnProperty.builder().withDefaultValue(null); bloomFilterFPPs = ColumnProperty.builder().withDefaultValue(DEFAULT_BLOOM_FILTER_FPP); @@ -504,6 +546,8 @@ private Builder(ParquetProperties toCopy) { this.numBloomFilterCandidates = ColumnProperty.builder(toCopy.numBloomFilterCandidates); this.maxBloomFilterBytes = toCopy.maxBloomFilterBytes; this.byteStreamSplitEnabled = ColumnProperty.builder(toCopy.byteStreamSplitEnabled); + this.fsstEnabled = ColumnProperty.builder(toCopy.fsstEnabled); + this.symbolTableOffsetEncoding = toCopy.symbolTableOffsetEncoding; this.extraMetaData = toCopy.extraMetaData; this.statistics = ColumnProperty.builder(toCopy.statistics); this.statisticsEnabled = toCopy.statisticsEnabled; @@ -573,6 +617,51 @@ public Builder withByteStreamSplitEncoding(String columnPath, boolean enable) { return this; } + /** + * Enable or disable the symbol table encoding for BINARY columns. + * + *

The encoding is not ratified and no file written with it is portable yet, because the format + * has nowhere to put the symbol table a column chunk's pages are compressed against: see + * parquet-format issue #531. Turning it on without supplying a values writer factory that knows + * where to keep the table fails when the first page is written rather than writing a file that + * cannot be read. + * + * @param enable whether the symbol table encoding should be enabled + * @return this builder for method chaining. + */ + public Builder withFsstEncoding(boolean enable) { + this.fsstEnabled.withDefaultValue(enable); + return this; + } + + /** + * Enable or disable the symbol table encoding for the specified column. + * + * @param columnPath the path of the column (dot-string) + * @param enable whether the symbol table encoding should be enabled + * @return this builder for method chaining. + */ + public Builder withFsstEncoding(String columnPath, boolean enable) { + this.fsstEnabled.withValue(columnPath, enable); + return this; + } + + /** + * Set how the per-value offsets into a symbol table page's code stream are written. + * + *

Delta packing by default. Writing them plain costs four bytes a value, which on text that + * the encoding halves is around a fifth of the page, so it gives away much of the ratio the + * encoding is there for; it is worth having only for a reader that wants the offsets addressable + * without decoding them. + * + * @param offsetEncoding how to write the offset section + * @return this builder for method chaining. + */ + public Builder withSymbolTableOffsetEncoding(OffsetEncoding offsetEncoding) { + this.symbolTableOffsetEncoding = Objects.requireNonNull(offsetEncoding, "offsetEncoding cannot be null"); + return this; + } + /** * Enable or disable BYTE_STREAM_SPLIT encoding for FLOAT, DOUBLE, INT32, INT64 and FIXED_LEN_BYTE_ARRAY columns. * diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java index e0d12c2878..f56ce95c64 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java @@ -93,10 +93,13 @@ private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { } private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter( - parquetProperties.getInitialSlabSize(), - parquetProperties.getPageSizeThreshold(), - parquetProperties.getAllocator()); + ValuesWriter fallbackWriter = DefaultValuesWriterFactory.symbolTableWriterWithFallBack( + path, + parquetProperties, + new PlainValuesWriter( + parquetProperties.getInitialSlabSize(), + parquetProperties.getPageSizeThreshold(), + parquetProperties.getAllocator())); return DefaultValuesWriterFactory.dictWriterWithFallBack( path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java index c50b4e49c5..8185ad588a 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java @@ -105,10 +105,13 @@ private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { } private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new DeltaByteArrayWriter( - parquetProperties.getInitialSlabSize(), - parquetProperties.getPageSizeThreshold(), - parquetProperties.getAllocator()); + ValuesWriter fallbackWriter = DefaultValuesWriterFactory.symbolTableWriterWithFallBack( + path, + parquetProperties, + new DeltaByteArrayWriter( + parquetProperties.getInitialSlabSize(), + parquetProperties.getPageSizeThreshold(), + parquetProperties.getAllocator())); return DefaultValuesWriterFactory.dictWriterWithFallBack( path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java index 4c03e6b65e..4cc4e05c80 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java @@ -25,6 +25,9 @@ import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter; import org.apache.parquet.column.values.fallback.FallbackValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTableType; +import org.apache.parquet.column.values.symboltable.SymbolTableValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTables; /** * Handles ValuesWriter creation statically based on the types of the columns and the writer version. @@ -103,6 +106,30 @@ static DictionaryValuesWriter dictionaryWriter( } } + /** + * Wraps a writer so that a symbol table encoding gets the first attempt at the column, falling back + * to the given writer for a chunk whose codes do not come out smaller than the values. + * + *

Returns the writer unchanged when the column is not configured for a symbol table, which is the + * default for every column. + */ + static ValuesWriter symbolTableWriterWithFallBack( + ColumnDescriptor path, ParquetProperties properties, ValuesWriter writerToFallBackTo) { + SymbolTableType type = properties.getSymbolTableType(path); + if (type == null) { + return writerToFallBackTo; + } + return FallbackValuesWriter.of( + new SymbolTableValuesWriter( + type, + SymbolTables.rejectingSink(), + properties.getSymbolTableOffsetEncoding(), + properties.getInitialSlabSize(), + properties.getPageSizeThreshold(), + properties.getAllocator()), + writerToFallBackTo); + } + static ValuesWriter dictWriterWithFallBack( ColumnDescriptor path, ParquetProperties parquetProperties, diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java index 3ea11380b9..b35ab6f0be 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java @@ -21,6 +21,7 @@ import org.apache.parquet.column.values.symboltable.fsst.Fsst8SymbolTable; import org.apache.parquet.column.values.symboltable.fsst.FsstTrainer; import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.ParquetEncodingException; /** * The one place that maps a symbol table representation to the code that implements it. @@ -43,6 +44,26 @@ public static SymbolTableTrainer trainer(SymbolTableType type) { } } + /** + * A sink that refuses the table instead of storing it. + * + *

What a writer gets when nothing has told it where the table should go, which is every writer + * built from the format's own metadata today: the format has no place for a symbol table, so a file + * written with the encoding could not be read back. Refusing at the first page fails while the + * failure still names the reason, rather than producing a file whose pages nothing can decode. + * + *

Supplying a sink that does store the table is what the encoding is waiting on. Until the format + * carries one, a caller that has somewhere to put it can build the writer itself and install it with + * {@code ParquetProperties.Builder.withValuesWriterFactory}. + */ + public static SymbolTableSink rejectingSink() { + return (type, body) -> { + throw new ParquetEncodingException("Cannot write a symbol table encoded column: the format has nowhere " + + "to keep the chunk's symbol table, so the pages would not be readable. See parquet-format " + + "issue #531."); + }; + } + /** * Rebuilds a table from a serialized body. * diff --git a/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java b/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java index 6d51f67cb0..109ef64375 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java @@ -24,6 +24,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.column.values.symboltable.SymbolTableType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.BeforeEach; @@ -122,6 +124,62 @@ public void withCompressionCodec_nullCodec_throwsNullPointerException() { .hasMessage("codec cannot be null"); } + @Test + public void fsst_isOffUnlessAskedFor() { + ParquetProperties props = ParquetProperties.builder().build(); + assertThat(props.isFsstEnabled(colA)).isFalse(); + assertThat(props.getSymbolTableType(colA)).isNull(); + } + + @Test + public void fsst_appliesToBinaryColumnsOnly() { + ParquetProperties props = + ParquetProperties.builder().withFsstEncoding(true).build(); + assertThat(props.getSymbolTableType(colA)).isEqualTo(SymbolTableType.FSST_8); + assertThat(props.isFsstEnabled(colB)).isFalse(); + assertThat(props.isFsstEnabled(colC)).isFalse(); + } + + @Test + public void fsst_canBeSetForOneColumn() { + ParquetProperties props = + ParquetProperties.builder().withFsstEncoding("col_a", true).build(); + assertThat(props.isFsstEnabled(colA)).isTrue(); + assertThat(ParquetProperties.builder() + .withFsstEncoding(true) + .withFsstEncoding("col_a", false) + .build() + .isFsstEnabled(colA)) + .isFalse(); + } + + @Test + public void symbolTableOffsets_areDeltaPackedUnlessAskedOtherwise() { + assertThat(ParquetProperties.builder().build().getSymbolTableOffsetEncoding()) + .isEqualTo(OffsetEncoding.DELTA_BINARY_PACKED); + assertThat(ParquetProperties.builder() + .withSymbolTableOffsetEncoding(OffsetEncoding.PLAIN) + .build() + .getSymbolTableOffsetEncoding()) + .isEqualTo(OffsetEncoding.PLAIN); + assertThatThrownBy(() -> ParquetProperties.builder().withSymbolTableOffsetEncoding(null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("offsetEncoding cannot be null"); + } + + @Test + public void copyBuilder_preservesSymbolTableSettings() { + ParquetProperties original = ParquetProperties.builder() + .withFsstEncoding("col_a", true) + .withSymbolTableOffsetEncoding(OffsetEncoding.PLAIN) + .build(); + + ParquetProperties copy = ParquetProperties.copy(original).build(); + + assertThat(copy.isFsstEnabled(colA)).isTrue(); + assertThat(copy.getSymbolTableOffsetEncoding()).isEqualTo(OffsetEncoding.PLAIN); + } + @Test public void copyBuilder_preservesColumnCodecAndLevel() { ParquetProperties original = ParquetProperties.builder() diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java index 436c13834f..c242f1c96e 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java @@ -48,6 +48,7 @@ import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; import org.apache.parquet.column.values.plain.PlainValuesWriter; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTableValuesWriter; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; @@ -631,6 +632,70 @@ private void testFloatingPoint_WithByteStreamSplitAndDictionary( PlainValuesWriter.class); } + @Test + public void testBinary_WithSymbolTable_AndDictionary() { + // The symbol table takes the columns the dictionary gives up on, and hands back the ones whose codes + // are no smaller than the values, so it sits between the two. + for (WriterVersion version : WriterVersion.values()) { + ValuesWriter writer = getDefaultFactory(ParquetProperties.builder() + .withWriterVersion(version) + .withFsstEncoding(true) + .build()) + .newValuesWriter(createColumnDescriptor(BINARY)); + validateWriterType(writer, FallbackValuesWriter.class); + FallbackValuesWriter outer = (FallbackValuesWriter) writer; + validateWriterType(outer.initialWriter, PlainBinaryDictionaryValuesWriter.class); + validateNestedSymbolTableWriter(outer.fallBackWriter, version); + } + } + + @Test + public void testBinary_WithSymbolTable_WithoutDictionary() { + for (WriterVersion version : WriterVersion.values()) { + ValuesWriter writer = getDefaultFactory(ParquetProperties.builder() + .withWriterVersion(version) + .withDictionaryEncoding(false) + .withFsstEncoding(true) + .build()) + .newValuesWriter(createColumnDescriptor(BINARY)); + validateNestedSymbolTableWriter(writer, version); + } + } + + @Test + public void testBinary_WithSymbolTable_PerColumn() { + ParquetProperties properties = ParquetProperties.builder() + .withDictionaryEncoding(false) + .withFsstEncoding("colA", true) + .build(); + ValuesWriterFactory factory = getDefaultFactory(properties); + validateNestedSymbolTableWriter( + factory.newValuesWriter(createColumnDescriptor(BINARY, "colA")), WriterVersion.PARQUET_1_0); + doTestValueWriter(createColumnDescriptor(BINARY, "colB"), properties, PlainValuesWriter.class); + } + + @Test + public void testSymbolTable_LeavesOtherTypesAlone() { + // The encoding is defined for byte arrays, so turning it on for every column must not reach a column + // it cannot encode. + ParquetProperties properties = ParquetProperties.builder() + .withDictionaryEncoding(false) + .withFsstEncoding(true) + .build(); + doTestValueWriter(createColumnDescriptor(INT32), properties, PlainValuesWriter.class); + doTestValueWriter(createColumnDescriptor(FLOAT), properties, PlainValuesWriter.class); + doTestValueWriter(createColumnDescriptor(BOOLEAN), properties, BooleanPlainValuesWriter.class); + } + + private void validateNestedSymbolTableWriter(ValuesWriter writer, WriterVersion version) { + validateWriterType(writer, FallbackValuesWriter.class); + FallbackValuesWriter fallback = (FallbackValuesWriter) writer; + validateWriterType(fallback.initialWriter, SymbolTableValuesWriter.class); + validateWriterType( + fallback.fallBackWriter, + version == WriterVersion.PARQUET_1_0 ? PlainValuesWriter.class : DeltaByteArrayWriter.class); + } + private void validateFactory( ValuesWriterFactory factory, PrimitiveTypeName typeName, diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java index aea8edabd5..bb57da2d3a 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java @@ -44,6 +44,7 @@ import org.apache.parquet.column.values.plain.PlainValuesWriter; import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.junit.Test; @@ -317,6 +318,23 @@ public void aReaderWithNoTableSaysSoRatherThanFailingLater() { assertTrue(thrown.getMessage().contains("#531")); } + @Test + public void aWriterWithNowhereToPutItsTableSaysSoBeforeWritingAPage() throws IOException { + // What a writer built from the format's own settings gets, because the format has no place for a + // symbol table. Refusing at the first page beats writing pages nothing can decode. + try (SymbolTableValuesWriter writer = new SymbolTableValuesWriter( + SymbolTableType.FSST_8, + SymbolTables.rejectingSink(), + OffsetEncoding.DELTA_BINARY_PACKED, + SLAB_SIZE, + PAGE_SIZE, + HeapByteBufferAllocator.getInstance())) { + writer.writeBytes(Binary.fromString("http://example.com/a")); + ParquetEncodingException thrown = assertThrows(ParquetEncodingException.class, writer::getBytes); + assertTrue(thrown.getMessage().contains("#531")); + } + } + @Test public void theEncodingHandsOutTheReaderForBinaryOnly() { assertTrue( diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index f6ee73bbc0..2a3816d78d 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -127,6 +127,7 @@ import org.apache.parquet.internal.hadoop.metadata.IndexReference; import org.apache.parquet.io.InvalidFileOffsetException; import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.ColumnOrder.ColumnOrderName; import org.apache.parquet.schema.GroupType; @@ -767,6 +768,13 @@ public org.apache.parquet.column.Encoding getEncoding(Encoding encoding) { } public Encoding getEncoding(org.apache.parquet.column.Encoding encoding) { + if (encoding == org.apache.parquet.column.Encoding.FSST) { + // The format has no FSST value and no place for the symbol table a chunk's pages are compressed + // against, so a footer cannot describe such a column: see parquet-format issue #531. + throw new ParquetEncodingException( + "Encoding FSST cannot be written to a Parquet footer: the format does not define it yet. " + + "See parquet-format issue #531."); + } return Encoding.valueOf(encoding.name()); } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 4db288f455..62cf3f157f 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; import org.apache.parquet.crypto.FileEncryptionProperties; import org.apache.parquet.hadoop.ParquetFileWriter.Mode; import org.apache.parquet.hadoop.api.WriteSupport; @@ -83,6 +84,13 @@ * # To enable/disable BYTE_STREAM_SPLIT encoding * parquet.enable.bytestreamsplit=false # true to enable BYTE_STREAM_SPLIT encoding * + * # To enable/disable the FSST symbol table encoding for BINARY columns. Not ratified: no file + * # written with it is readable, see parquet-format issue #531 + * parquet.enable.fsst=false # true to enable the symbol table encoding + * + * # How a symbol table page's per-value offsets are written: PLAIN or DELTA_BINARY_PACKED + * parquet.fsst.offset.encoding=DELTA_BINARY_PACKED + * * # To enable/disable summary metadata aggregation at the end of a MR job * # The default is true (enabled) * parquet.enable.summary-metadata=true # false to disable summary aggregation @@ -141,6 +149,8 @@ public static enum JobSummaryLevel { public static final String DICTIONARY_PAGE_SIZE = "parquet.dictionary.page.size"; public static final String ENABLE_DICTIONARY = "parquet.enable.dictionary"; public static final String ENABLE_BYTE_STREAM_SPLIT = "parquet.enable.bytestreamsplit"; + public static final String ENABLE_FSST = "parquet.enable.fsst"; + public static final String FSST_OFFSET_ENCODING = "parquet.fsst.offset.encoding"; public static final String VALIDATION = "parquet.validation"; public static final String WRITER_VERSION = "parquet.writer.version"; public static final String MEMORY_POOL_RATIO = "parquet.memory.pool.ratio"; @@ -288,6 +298,15 @@ public static boolean getByteStreamSplitEnabled(Configuration configuration) { ENABLE_BYTE_STREAM_SPLIT, ParquetProperties.DEFAULT_IS_BYTE_STREAM_SPLIT_ENABLED); } + public static boolean getFsstEnabled(Configuration configuration) { + return configuration.getBoolean(ENABLE_FSST, ParquetProperties.DEFAULT_IS_FSST_ENABLED); + } + + public static OffsetEncoding getFsstOffsetEncoding(Configuration configuration) { + return OffsetEncoding.valueOf( + configuration.get(FSST_OFFSET_ENCODING, ParquetProperties.DEFAULT_SYMBOL_TABLE_OFFSET_ENCODING.name())); + } + public static int getMinRowCountForPageSizeCheck(Configuration configuration) { return configuration.getInt( MIN_ROW_COUNT_FOR_PAGE_SIZE_CHECK, ParquetProperties.DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK); @@ -522,6 +541,8 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .withDictionaryPageSize(getDictionaryPageSize(conf)) .withDictionaryEncoding(getEnableDictionary(conf)) .withByteStreamSplitEncoding(getByteStreamSplitEnabled(conf)) + .withFsstEncoding(getFsstEnabled(conf)) + .withSymbolTableOffsetEncoding(getFsstOffsetEncoding(conf)) .withWriterVersion(getWriterVersion(conf)) .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index f5222de828..e841e2db4c 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -127,6 +127,7 @@ import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder; +import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.ColumnOrder; import org.apache.parquet.schema.LogicalTypeAnnotation; @@ -542,10 +543,21 @@ public void testLogicalToConvertedTypeConversion() { .isEqualTo(ConvertedType.MAP_KEY_VALUE); } + @Test + public void testFsstCannotBeWrittenToAFooter() { + assertThatThrownBy(() -> new ParquetMetadataConverter().getEncoding(org.apache.parquet.column.Encoding.FSST)) + .isInstanceOf(ParquetEncodingException.class) + .hasMessageContaining("#531"); + } + @Test public void testEnumEquivalence() { ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); for (org.apache.parquet.column.Encoding encoding : org.apache.parquet.column.Encoding.values()) { + if (encoding == org.apache.parquet.column.Encoding.FSST) { + // The format has no value to convert it to yet: see testFsstCannotBeWrittenToAFooter. + continue; + } assertThat(parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding))) .isEqualTo(encoding); } @@ -832,6 +844,9 @@ public void testEncodingsOrder() { Set columnEncodings = new HashSet<>(Arrays.asList(org.apache.parquet.column.Encoding.values())); + // The format has no FSST value, so it has no ordinal to order and cannot be converted at all: + // see testFsstCannotBeWrittenToAFooter. + columnEncodings.remove(org.apache.parquet.column.Encoding.FSST); // Assert that the encodings are returned in ascending ordinal order List formatEncodings = From f9a73503e0d0ae30c131dbe4fa67ced32b6942d5 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 20:20:36 +0000 Subject: [PATCH 05/10] Assert with JUnit 5 and AssertJ, as the rest of the module does parquet-column's tests have been migrated off JUnit 4, and an enforcer rule bans the old imports outright: only the four benchmark classes are exempt, because they still need @Rule. These four files were written against JUnit 4 and only ever run with the enforcer skipped, so the ban was never checked against them. Converted mechanically, then the comparisons that had become a boolean plus a message were rewritten as the assertion they were describing - isLessThanOrEqualTo, isBetween, isPositive - so a failure prints both sides instead of "expected true". --- .../symboltable/SymbolTablePayloadTest.java | 74 +++++++------- .../SymbolTableValuesRoundTripTest.java | 97 +++++++++---------- .../fsst/FsstCodecRoundTripTest.java | 59 +++++------ .../fsst/FsstReferenceComparisonTest.java | 26 +++-- 4 files changed, 126 insertions(+), 130 deletions(-) diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java index 2a31eb1aad..9c14ef9bbb 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTablePayloadTest.java @@ -18,10 +18,8 @@ */ package org.apache.parquet.column.values.symboltable; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -33,7 +31,7 @@ import org.apache.parquet.bytes.HeapByteBufferAllocator; import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; import org.apache.parquet.io.ParquetDecodingException; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * The data page body's framing: what a writer produces byte for byte, and what a reader refuses. @@ -56,7 +54,7 @@ private static byte[] write(OffsetEncoding offsetEncoding, byte[]... values) thr for (byte[] value : values) { writer.addValue(value, 0, value.length); } - assertEquals(values.length, writer.valueCount()); + assertThat(writer.valueCount()).isEqualTo(values.length); return writer.getBytes().toByteArray(); } } @@ -88,7 +86,7 @@ public void writesThePlainLayoutByteForByte() throws IOException { expected.putInt(4); expected.putInt(6); expected.put(bytes(1, 2, 3, 4, 5, 6)); - assertArrayEquals(expected.array(), body); + assertThat(body).isEqualTo(expected.array()); } @Test @@ -96,11 +94,12 @@ public void writesTheDeltaLayoutWithTheSameHeaderShape() throws IOException { byte[] body = write(OffsetEncoding.DELTA_BINARY_PACKED, bytes(1, 2, 3), bytes(4), bytes(5, 6)); ByteBuffer header = ByteBuffer.wrap(body, 0, 9).order(ByteOrder.LITTLE_ENDIAN); - assertEquals(1, header.get() & 0xFF); - assertEquals(3, header.getInt()); + assertThat(header.get() & 0xFF).isEqualTo(1); + assertThat(header.getInt()).isEqualTo(3); int offsetSectionSize = header.getInt(); - assertEquals(body.length - 9 - 6, offsetSectionSize); - assertArrayEquals(bytes(1, 2, 3, 4, 5, 6), java.util.Arrays.copyOfRange(body, body.length - 6, body.length)); + assertThat(offsetSectionSize).isEqualTo(body.length - 9 - 6); + assertThat(java.util.Arrays.copyOfRange(body, body.length - 6, body.length)) + .isEqualTo(bytes(1, 2, 3, 4, 5, 6)); } @Test @@ -109,11 +108,11 @@ public void roundTripsBothOffsetEncodings() throws IOException { byte[] body = write(offsetEncoding, bytes(1, 2, 3), bytes(4), bytes(5, 6)); SymbolTablePayload payload = parse(body, 3); - assertEquals(3, payload.valueCount()); + assertThat(payload.valueCount()).isEqualTo(3); List values = valuesOf(payload); - assertArrayEquals(bytes(1, 2, 3), values.get(0)); - assertArrayEquals(bytes(4), values.get(1)); - assertArrayEquals(bytes(5, 6), values.get(2)); + assertThat(values.get(0)).isEqualTo(bytes(1, 2, 3)); + assertThat(values.get(1)).isEqualTo(bytes(4)); + assertThat(values.get(2)).isEqualTo(bytes(5, 6)); } } @@ -126,12 +125,12 @@ public void roundTripsValuesWithNoCodes() throws IOException { byte[] body = write(offsetEncoding, bytes(), bytes(7), bytes(), bytes()); SymbolTablePayload payload = parse(body, 4); - assertEquals(4, payload.valueCount()); + assertThat(payload.valueCount()).isEqualTo(4); List values = valuesOf(payload); - assertEquals(0, values.get(0).length); - assertArrayEquals(bytes(7), values.get(1)); - assertEquals(0, values.get(2).length); - assertEquals(0, values.get(3).length); + assertThat(values.get(0)).isEmpty(); + assertThat(values.get(1)).isEqualTo(bytes(7)); + assertThat(values.get(2)).isEmpty(); + assertThat(values.get(3)).isEmpty(); } } @@ -139,8 +138,8 @@ public void roundTripsValuesWithNoCodes() throws IOException { public void aPageWithNoValuesIsJustAHeader() throws IOException { for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { byte[] body = write(offsetEncoding); - assertEquals(SymbolTablePayload.HEADER_SIZE, body.length); - assertEquals(0, parse(body, 0).valueCount()); + assertThat(body.length).isEqualTo(SymbolTablePayload.HEADER_SIZE); + assertThat(parse(body, 0).valueCount()).isEqualTo(0); } } @@ -158,37 +157,38 @@ public void deltaOffsetsCostFarLessThanPlainOnesOnAPageOfShortValues() throws IO int delta = write(OffsetEncoding.DELTA_BINARY_PACKED, values).length; int codeBytes = values.length * 4; - assertEquals(9 + values.length * 4 + codeBytes, plain); - assertTrue( - "delta offsets should cost a small fraction of plain ones, but the payloads were " + delta + " and " - + plain, - delta - codeBytes < (plain - codeBytes) / 8); + assertThat(plain).isEqualTo(9 + values.length * 4 + codeBytes); + assertThat(delta - codeBytes) + .as("delta offsets should cost a small fraction of plain ones, but the payloads were " + delta + " and " + + plain) + .isLessThan((plain - codeBytes) / 8); } @Test public void rejectsABodyShorterThanTheHeader() { - ParquetDecodingException thrown = assertThrows(ParquetDecodingException.class, () -> parse(new byte[8], 1)); - assertTrue(thrown.getMessage().contains("shorter than its 9-byte header")); + assertThatThrownBy(() -> parse(new byte[8], 1)) + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("shorter than its 9-byte header"); } @Test public void rejectsAnUnknownOffsetEncoding() throws IOException { byte[] body = write(OffsetEncoding.PLAIN, bytes(1)); body[0] = 2; - assertThrows(ParquetDecodingException.class, () -> parse(body, 1)); + assertThatThrownBy(() -> parse(body, 1)).isInstanceOf(ParquetDecodingException.class); } @Test public void rejectsMoreValuesThanThePageHeaderDeclares() throws IOException { byte[] body = write(OffsetEncoding.PLAIN, bytes(1), bytes(2)); - assertThrows(ParquetDecodingException.class, () -> parse(body, 1)); + assertThatThrownBy(() -> parse(body, 1)).isInstanceOf(ParquetDecodingException.class); } @Test public void rejectsAnOffsetSectionLongerThanTheBody() throws IOException { byte[] body = write(OffsetEncoding.PLAIN, bytes(1)); putInt(body, 5, 1000); - assertThrows(ParquetDecodingException.class, () -> parse(body, 1)); + assertThatThrownBy(() -> parse(body, 1)).isInstanceOf(ParquetDecodingException.class); } @Test @@ -196,7 +196,7 @@ public void rejectsAPlainOffsetSectionOfTheWrongLength() throws IOException { byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); // Claim one value's worth of offsets while the section holds two. putInt(body, 1, 1); - assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + assertThatThrownBy(() -> parse(body, 2)).isInstanceOf(ParquetDecodingException.class); } @Test @@ -204,21 +204,21 @@ public void rejectsOffsetsThatGoBackwards() throws IOException { byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); putInt(body, 9, 3); // the first value ends past where the second one does putInt(body, 13, 1); - assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + assertThatThrownBy(() -> parse(body, 2)).isInstanceOf(ParquetDecodingException.class); } @Test public void rejectsOffsetsThatLeaveCodeBytesUnreachable() throws IOException { byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); putInt(body, 13, 2); // the last value's codes are dropped - assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + assertThatThrownBy(() -> parse(body, 2)).isInstanceOf(ParquetDecodingException.class); } @Test public void rejectsOffsetsThatRunPastTheCodeSection() throws IOException { byte[] body = write(OffsetEncoding.PLAIN, bytes(1, 2), bytes(3)); putInt(body, 13, 4); - assertThrows(ParquetDecodingException.class, () -> parse(body, 2)); + assertThatThrownBy(() -> parse(body, 2)).isInstanceOf(ParquetDecodingException.class); } @Test @@ -230,7 +230,7 @@ public void rejectsAnEmptyPageCarryingAnOffsetSection() throws IOException { putInt(header, 5, 4); body.write(header); body.write(new byte[4]); - assertThrows(ParquetDecodingException.class, () -> parse(body.toByteArray(), 0)); + assertThatThrownBy(() -> parse(body.toByteArray(), 0)).isInstanceOf(ParquetDecodingException.class); } private static void putInt(byte[] destination, int position, int value) { diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java index bb57da2d3a..2d20e18fff 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java @@ -18,11 +18,8 @@ */ package org.apache.parquet.column.values.symboltable; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.IOException; import java.nio.ByteBuffer; @@ -47,7 +44,7 @@ import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * The writer and reader as a pair: values in, the same values out, over the seams that a symbol table @@ -121,12 +118,12 @@ private static List> roundTrip(List> pages, OffsetEnco for (Binary value : page) { writer.writeBytes(value); } - assertEquals(Encoding.FSST, writer.getEncoding()); + assertThat(writer.getEncoding()).isEqualTo(Encoding.FSST); bodies.add(writer.getBytes().toByteArray()); writer.reset(); } } - assertEquals("one table for the whole chunk", 1, relay.publishCount); + assertThat(relay.publishCount).as("one table for the whole chunk").isEqualTo(1); SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); List> read = new ArrayList<>(); @@ -144,10 +141,9 @@ private static List> roundTrip(List> pages, OffsetEnco private static void assertRoundTrips(List values) throws IOException { for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { - assertEquals( - "offsets " + offsetEncoding, - Arrays.asList(values), - roundTrip(Arrays.asList(values), offsetEncoding)); + assertThat(roundTrip(Arrays.asList(values), offsetEncoding)) + .as("offsets " + offsetEncoding) + .isEqualTo(Arrays.asList(values)); } } @@ -235,7 +231,9 @@ public void oneTableTrainedOnTheFirstPageServesLaterPages() throws IOException { pages.add(values); } for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { - assertEquals("offsets " + offsetEncoding, pages, roundTrip(pages, offsetEncoding)); + assertThat(roundTrip(pages, offsetEncoding)) + .as("offsets " + offsetEncoding) + .isEqualTo(pages); } } @@ -249,14 +247,14 @@ public void resetDictionaryTrainsAgain() throws IOException { } writer.getBytes(); writer.reset(); - assertEquals(1, relay.publishCount); + assertThat(relay.publishCount).isEqualTo(1); writer.resetDictionary(); for (Binary value : binaries("zeta", "zenith", "zephyr")) { writer.writeBytes(value); } writer.getBytes(); - assertEquals(2, relay.publishCount); + assertThat(relay.publishCount).isEqualTo(2); } } @@ -277,7 +275,7 @@ public void skipReachesTheSameValuesAsReading() throws IOException { reader.initFromPage(values.size(), ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); reader.skip(start); for (int i = start; i < values.size(); i++) { - assertEquals("from " + start + " at " + i, values.get(i), reader.readBytes()); + assertThat(reader.readBytes()).as("from " + start + " at " + i).isEqualTo(values.get(i)); } } @@ -288,7 +286,7 @@ public void skipReachesTheSameValuesAsReading() throws IOException { if (i % 2 == 0) { reader.skip(); } else { - assertEquals(values.get(i), reader.readBytes()); + assertThat(reader.readBytes()).isEqualTo(values.get(i)); } } } @@ -304,18 +302,17 @@ public void readingPastTheEndOfAPageIsRejected() throws IOException { SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); reader.initFromPage(1, ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); reader.readBytes(); - assertThrows(ParquetDecodingException.class, reader::readBytes); - assertThrows(ParquetDecodingException.class, reader::skip); + assertThatThrownBy(reader::readBytes).isInstanceOf(ParquetDecodingException.class); + assertThatThrownBy(reader::skip).isInstanceOf(ParquetDecodingException.class); } @Test public void aReaderWithNoTableSaysSoRatherThanFailingLater() { SymbolTableValuesReader reader = new SymbolTableValuesReader(); - assertNull(reader.symbolTableType()); - ParquetDecodingException thrown = assertThrows( - ParquetDecodingException.class, - () -> reader.initFromPage(1, ByteBufferInputStream.wrap(ByteBuffer.wrap(new byte[9])))); - assertTrue(thrown.getMessage().contains("#531")); + assertThat(reader.symbolTableType()).isNull(); + assertThatThrownBy(() -> reader.initFromPage(1, ByteBufferInputStream.wrap(ByteBuffer.wrap(new byte[9])))) + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("#531"); } @Test @@ -330,22 +327,21 @@ public void aWriterWithNowhereToPutItsTableSaysSoBeforeWritingAPage() throws IOE PAGE_SIZE, HeapByteBufferAllocator.getInstance())) { writer.writeBytes(Binary.fromString("http://example.com/a")); - ParquetEncodingException thrown = assertThrows(ParquetEncodingException.class, writer::getBytes); - assertTrue(thrown.getMessage().contains("#531")); + assertThatThrownBy(writer::getBytes) + .isInstanceOf(ParquetEncodingException.class) + .hasMessageContaining("#531"); } } @Test public void theEncodingHandsOutTheReaderForBinaryOnly() { - assertTrue( - Encoding.FSST.getValuesReader(descriptor(PrimitiveTypeName.BINARY), ValuesType.VALUES) - instanceof SymbolTableValuesReader); + assertThat(Encoding.FSST.getValuesReader(descriptor(PrimitiveTypeName.BINARY), ValuesType.VALUES)) + .isInstanceOf(SymbolTableValuesReader.class); for (PrimitiveTypeName type : new PrimitiveTypeName[] { PrimitiveTypeName.INT32, PrimitiveTypeName.INT64, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY }) { - assertThrows( - ParquetDecodingException.class, - () -> Encoding.FSST.getValuesReader(descriptor(type), ValuesType.VALUES)); + assertThatThrownBy(() -> Encoding.FSST.getValuesReader(descriptor(type), ValuesType.VALUES)) + .isInstanceOf(ParquetDecodingException.class); } } @@ -364,8 +360,9 @@ public void aPageTheEncodingWouldGrowIsWrittenPlain() throws IOException { values.add(Binary.fromConstantByteArray(new byte[] {(byte) i})); } - assertEquals(Encoding.PLAIN, fallbackEncodingFor(values, OffsetEncoding.PLAIN)); - assertEquals(Encoding.FSST, fallbackEncodingFor(values, OffsetEncoding.DELTA_BINARY_PACKED)); + assertThat(fallbackEncodingFor(values, OffsetEncoding.PLAIN)).isEqualTo(Encoding.PLAIN); + assertThat(fallbackEncodingFor(values, OffsetEncoding.DELTA_BINARY_PACKED)) + .isEqualTo(Encoding.FSST); } /** Runs a page through the fallback wrapper and reads it back with whatever encoding it chose. */ @@ -387,7 +384,7 @@ private static Encoding fallbackEncodingFor(List values, OffsetEncoding encoding == Encoding.PLAIN ? new BinaryPlainValuesReader() : new SymbolTableValuesReader(relay); reader.initFromPage(values.size(), ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); for (Binary value : values) { - assertEquals(value, reader.readBytes()); + assertThat(reader.readBytes()).isEqualTo(value); } return encoding; } @@ -405,8 +402,8 @@ public void fallingBackReplaysEveryValue() throws IOException { } writer.fallBackAllValuesTo(collector); } - assertEquals(values, replayed); - assertEquals(0, relay.publishCount); + assertThat(replayed).isEqualTo(values); + assertThat(relay.publishCount).isEqualTo(0); } @Test @@ -418,30 +415,30 @@ public void aTableIsPublishedOnceAndRebuiltFromItsBytes() throws IOException { } writer.getBytes(); } - assertEquals(SymbolTableType.FSST_8, relay.type); + assertThat(relay.type).isEqualTo(SymbolTableType.FSST_8); SymbolTable first = relay.getSymbolTable(); - assertEquals(SymbolTableType.FSST_8, first.type()); - assertTrue("a table was trained", first.symbolCount() > 0); - assertEquals(first.symbolCount(), relay.getSymbolTable().symbolCount()); + assertThat(first.type()).isEqualTo(SymbolTableType.FSST_8); + assertThat(first.symbolCount()).as("a table was trained").isPositive(); + assertThat(relay.getSymbolTable().symbolCount()).isEqualTo(first.symbolCount()); } @Test public void theWriterReportsWhatItIsHoldingAndReleasesItOnReset() throws IOException { SymbolTableRelay relay = new SymbolTableRelay(); try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { - assertEquals(0, writer.getBufferedSize()); + assertThat(writer.getBufferedSize()).isEqualTo(0); writer.writeBytes(Binary.fromString("alpha")); - assertEquals(5 + 4, writer.getBufferedSize()); + assertThat(writer.getBufferedSize()).isEqualTo(5 + 4); writer.getBytes(); writer.reset(); - assertEquals(0, writer.getBufferedSize()); - assertTrue(writer.getAllocatedSize() > 0); - assertTrue(writer.memUsageString("x").startsWith("x ")); - assertEquals(SymbolTableType.FSST_8, writer.symbolTableType()); + assertThat(writer.getBufferedSize()).isEqualTo(0); + assertThat(writer.getAllocatedSize()).isPositive(); + assertThat(writer.memUsageString("x")).startsWith("x "); + assertThat(writer.symbolTableType()).isEqualTo(SymbolTableType.FSST_8); // The only fallback question is asked once, and never by the writer itself. - assertTrue(writer.isCompressionSatisfying(100, 99)); - assertFalse(writer.isCompressionSatisfying(100, 100)); - assertFalse(writer.shouldFallBack()); + assertThat(writer.isCompressionSatisfying(100, 99)).isTrue(); + assertThat(writer.isCompressionSatisfying(100, 100)).isFalse(); + assertThat(writer.shouldFallBack()).isFalse(); } } diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java index 47ad302aa9..76b93ff5ac 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstCodecRoundTripTest.java @@ -18,9 +18,7 @@ */ package org.apache.parquet.column.values.symboltable.fsst; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -33,7 +31,7 @@ import org.apache.parquet.column.values.symboltable.SymbolTableType; import org.apache.parquet.column.values.symboltable.TrainedSymbolTable; import org.apache.parquet.column.values.symboltable.ValueBuffer; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * Round trips the FSST codec over inputs chosen to reach the places the port could be wrong: the @@ -56,9 +54,9 @@ private static Result roundTrip(List values) throws IOException { // serialization and the renumbering are both under test. byte[] serialized = table.serialize().toByteArray(); SymbolTable reread = Fsst8SymbolTable.deserialize(serialized, 0, serialized.length); - assertEquals(SymbolTableType.FSST_8, reread.type()); - assertEquals(table.symbolCount(), reread.symbolCount()); - assertArrayEquals(serialized, reread.serialize().toByteArray()); + assertThat(reread.type()).isEqualTo(SymbolTableType.FSST_8); + assertThat(reread.symbolCount()).isEqualTo(table.symbolCount()); + assertThat(reread.serialize().toByteArray()).isEqualTo(serialized); CodeStreamDecoder decoder = reread.decoder(); int codeBytes = 0; @@ -67,16 +65,17 @@ private static Result roundTrip(List values) throws IOException { byte[] value = values.get(i); byte[] codes = new byte[encoder.maxCompressedLength(value.length)]; int codeLength = encoder.compress(buffer.data(), buffer.offset(i), buffer.length(i), codes, 0); - assertTrue("compressed past the declared bound", codeLength <= codes.length); + assertThat(codeLength).as("compressed past the declared bound").isLessThanOrEqualTo(codes.length); - assertEquals( - "expandedLength disagrees with expand on value " + i, - value.length, - decoder.expandedLength(codes, 0, codeLength)); + assertThat(decoder.expandedLength(codes, 0, codeLength)) + .as("expandedLength disagrees with expand on value " + i) + .isEqualTo(value.length); byte[] expanded = new byte[value.length]; int written = decoder.expand(codes, 0, codeLength, expanded, 0); - assertEquals("wrong expanded length for value " + i, value.length, written); - assertArrayEquals("value " + i + " did not survive the round trip", value, expanded); + assertThat(written).as("wrong expanded length for value " + i).isEqualTo(value.length); + assertThat(expanded) + .as("value " + i + " did not survive the round trip") + .isEqualTo(value); codeBytes += codeLength; rawBytes += value.length; @@ -112,12 +111,14 @@ public void compressesRepetitiveTextAndGetsItBack() throws IOException { .getBytes(StandardCharsets.UTF_8)); } Result result = roundTrip(values); - assertTrue("the table should hold symbols", result.table.symbolCount() > 0); + assertThat(result.table.symbolCount()) + .as("the table should hold symbols") + .isPositive(); // A corpus this repetitive is the case the encoding exists for; if it does not shrink here the // trainer is not finding the shared substrings. - assertTrue( - "expected well under half the bytes, got " + result.codeBytes + " of " + result.rawBytes, - result.codeBytes * 2 < result.rawBytes); + assertThat(result.codeBytes * 2) + .as("expected well under half the bytes, got " + result.codeBytes + " of " + result.rawBytes) + .isLessThan(result.rawBytes); } @Test @@ -158,7 +159,9 @@ public void handlesIncompressibleBytes() throws IOException { values.add(value); } Result result = roundTrip(values); - assertTrue("escaping should cost bytes, not save them", result.codeBytes >= result.rawBytes); + assertThat(result.codeBytes) + .as("escaping should cost bytes, not save them") + .isGreaterThanOrEqualTo(result.rawBytes); } @Test @@ -195,9 +198,9 @@ public void trainsOnACorpusLargerThanTheSample() throws IOException { values.add(builder.toString().getBytes(StandardCharsets.UTF_8)); } Result result = roundTrip(values); - assertTrue( - "expected well under half the bytes, got " + result.codeBytes + " of " + result.rawBytes, - result.codeBytes * 2 < result.rawBytes); + assertThat(result.codeBytes * 2) + .as("expected well under half the bytes, got " + result.codeBytes + " of " + result.rawBytes) + .isLessThan(result.rawBytes); } @Test @@ -212,19 +215,19 @@ public void symbolTableStaysWithinItsFormatLimits() throws IOException { values.add(value); } Result result = roundTrip(values); - assertTrue(result.table.symbolCount() <= FsstCodes.MAX_SYMBOLS); + assertThat(result.table.symbolCount()).isLessThanOrEqualTo(FsstCodes.MAX_SYMBOLS); for (int code = 0; code < result.table.symbolCount(); code++) { int length = result.table.symbolLength(code); - assertTrue("symbol " + code + " has length " + length, length >= 1 && length <= 8); + assertThat(length).as("symbol " + code).isBetween(1, 8); } // The serialized table must be in length order, which is what lets a reader rebuild it from the // length histogram alone. for (int code = 1; code < result.table.symbolCount(); code++) { - assertTrue( - "symbols are not in length order at code " + code, - result.table.symbolLength(code - 1) <= result.table.symbolLength(code)); + assertThat(result.table.symbolLength(code - 1)) + .as("symbols are not in length order at code " + code) + .isLessThanOrEqualTo(result.table.symbolLength(code)); } int size = (int) result.table.serialize().size(); - assertTrue("serialized table is " + size + " bytes", size >= 9 && size <= 2049); + assertThat(size).as("serialized table size").isBetween(9, 2049); } } diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java index 1c3d632d03..09ff85c6cd 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/fsst/FsstReferenceComparisonTest.java @@ -18,8 +18,7 @@ */ package org.apache.parquet.column.values.symboltable.fsst; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.assertj.core.api.Assertions.assertThat; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -32,7 +31,7 @@ import org.apache.parquet.column.values.symboltable.CodeStreamEncoder; import org.apache.parquet.column.values.symboltable.TrainedSymbolTable; import org.apache.parquet.column.values.symboltable.ValueBuffer; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * Requires this implementation to produce the same symbol table and the same code bytes as the @@ -197,16 +196,14 @@ private void check(String corpus, List values, String expectedDigest) th for (byte[] value : values) { buffer.add(value, 0, value.length); } - assertEquals( - corpus + ": the generated corpus does not match the one the fixtures were made from", - expectedDigest, - sha256(buffer.data(), buffer.byteCount())); + assertThat(sha256(buffer.data(), buffer.byteCount())) + .as(corpus + ": the generated corpus does not match the one the fixtures were made from") + .isEqualTo(expectedDigest); TrainedSymbolTable trained = new FsstTrainer().train(buffer); - assertArrayEquals( - corpus + ": symbol table differs from the reference implementation", - resource(corpus + ".table"), - trained.table().serialize().toByteArray()); + assertThat(trained.table().serialize().toByteArray()) + .as(corpus + ": symbol table differs from the reference implementation") + .isEqualTo(resource(corpus + ".table")); CodeStreamEncoder encoder = trained.encoder(); ByteArrayOutputStream codes = new ByteArrayOutputStream(); @@ -215,10 +212,9 @@ private void check(String corpus, List values, String expectedDigest) th int length = encoder.compress(buffer.data(), buffer.offset(i), buffer.length(i), output, 0); codes.write(output, 0, length); } - assertArrayEquals( - corpus + ": code bytes differ from the reference implementation", - resource(corpus + ".codes"), - codes.toByteArray()); + assertThat(codes.toByteArray()) + .as(corpus + ": code bytes differ from the reference implementation") + .isEqualTo(resource(corpus + ".codes")); } private static byte[] resource(String name) throws IOException { From e64fad241089a6c85bb7b2025d2c57232ddc4096 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 20:21:02 +0000 Subject: [PATCH 06/10] Decode symbol table pages written by another implementation The page layout in this package is only worth having if it is the same layout everyone else writes. Nothing in the tree proved that: every test so far encoded with this code and decoded with this code, which passes just as well for a private format. The C++ implementation's interop file supplies the missing side. Its pages are lifted out and committed as fixtures, and the reader is required to reproduce what the file's plain reference columns hold - so the fixture does not depend on any FSST decoder being correct, including this one. Five cases: high cardinality values over fourteen pages sharing one table, zero-length values and short pages, the escape code, and a chunk whose offsets are stored plainly rather than packed. Both offset section encodings appear, checked from the page bytes rather than through the reader, so a reader that ignored the mode byte could not hide it. The expected values are kept as digests rather than as payloads: the values themselves would have been fifty kilobytes in a module whose test resources total under three hundred. The header of expected.txt records where the file came from, how the fixture is laid out, and how the digest is computed, so it can be regenerated. The 16-bit columns are skipped. Nothing reads them yet, and the table body they carry is a different shape from the one the format currently describes. --- .gitattributes | 1 + .../symboltable/SymbolTableInteropTest.java | 284 ++++++++++++++++++ .../fsst/interop/escapes-binary.pages | Bin 0 -> 2116 bytes .../test/resources/fsst/interop/escapes.pages | Bin 0 -> 2458 bytes .../test/resources/fsst/interop/expected.txt | 74 +++++ .../fsst/interop/nulls-and-empties.pages | Bin 0 -> 3241 bytes .../fsst/interop/plain-offsets.pages | Bin 0 -> 2320 bytes .../test/resources/fsst/interop/urls.pages | Bin 0 -> 6936 bytes pom.xml | 3 + 9 files changed, 362 insertions(+) create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java create mode 100644 parquet-column/src/test/resources/fsst/interop/escapes-binary.pages create mode 100644 parquet-column/src/test/resources/fsst/interop/escapes.pages create mode 100644 parquet-column/src/test/resources/fsst/interop/expected.txt create mode 100644 parquet-column/src/test/resources/fsst/interop/nulls-and-empties.pages create mode 100644 parquet-column/src/test/resources/fsst/interop/plain-offsets.pages create mode 100644 parquet-column/src/test/resources/fsst/interop/urls.pages diff --git a/.gitattributes b/.gitattributes index 759dadcbe5..a4810ececb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -23,3 +23,4 @@ core.autocrlf=false # FSST test fixtures are byte streams, not text: line-ending normalization would corrupt them. parquet-column/src/test/resources/fsst/*.table binary parquet-column/src/test/resources/fsst/*.codes binary +parquet-column/src/test/resources/fsst/interop/*.pages binary diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java new file mode 100644 index 0000000000..9b131ad78a --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.io.api.Binary; +import org.junit.jupiter.api.Test; + +/** + * Requires this implementation to decode symbol table pages another implementation wrote. + * + *

The round trip and reference comparison tests in this package both start from bytes this code + * produced: one proves the decoder undoes the encoder, the other proves the trainer picks the same + * symbols as the reference implementation. Neither can catch a page layout that is self-consistent + * and different from everyone else's -- a swapped header field, an off-by-one in the offset section, + * an escape read as a code. Only bytes from a foreign writer catch that, which is what these + * fixtures are. + * + *

They come from the C++ implementation's Parquet interop file, and their provenance, layout and + * per-case coverage are documented in the header of {@code src/test/resources/fsst/interop/ + * expected.txt}. The expected values there were taken from that file's conventionally-encoded + * columns, which hold the same values as the encoded ones, so nothing in the fixtures depends on any + * FSST decoder being right. + */ +public class SymbolTableInteropTest { + + private static final String DIRECTORY = "/fsst/interop/"; + + /** One case's symbol table and the bodies of its data pages, as the fixture files hold them. */ + private static final class Chunk { + final SymbolTableType type; + final byte[] table; + final List pages; + + Chunk(SymbolTableType type, byte[] table, List pages) { + this.type = type; + this.table = table; + this.pages = pages; + } + + SymbolTableValuesReader reader() { + SymbolTable symbolTable = SymbolTables.deserialize(type, table, 0, table.length); + return new SymbolTableValuesReader(() -> symbolTable); + } + } + + /** What one line of {@code expected.txt} asserts: how many values, and which ones. */ + private static final class Expectation { + final int pageCount; + final int count; + final String digest; + + Expectation(int pageCount, int count, String digest) { + this.pageCount = pageCount; + this.count = count; + this.digest = digest; + } + } + + private static final List CASES = + List.of("urls", "nulls-and-empties", "escapes", "escapes-binary", "plain-offsets"); + + @Test + public void everyCaseDecodesToWhatTheReferenceColumnsHold() throws IOException { + Map expected = readExpectations(); + for (String name : CASES) { + Chunk chunk = readChunk(name); + Expectation whole = expectation(expected, name); + assertThat(chunk.pages.size()).as(name + ": page count").isEqualTo(whole.pageCount); + + SymbolTableValuesReader reader = chunk.reader(); + List all = new ArrayList<>(); + for (int page = 0; page < chunk.pages.size(); page++) { + String pageName = name + "." + page; + Expectation pageExpectation = expectation(expected, pageName); + List values = decodePage(reader, chunk.pages.get(page), pageExpectation.count); + assertThat(digest(values)).as(pageName).isEqualTo(pageExpectation.digest); + all.addAll(values); + } + assertThat(all.size()).as(name + ": value count").isEqualTo(whole.count); + assertThat(digest(all)).as(name).isEqualTo(whole.digest); + } + } + + @Test + public void bothOffsetSectionEncodingsAreCovered() throws IOException { + // A writer with few values per page has nothing to gain from packing the offsets, so the file + // contains pages of each kind and a reader has to tolerate both. Read from the page bodies + // rather than through the reader, so that a reader which ignored the byte could not hide it. + Map pages = new HashMap<>(); + for (String name : CASES) { + for (byte[] page : readChunk(name).pages) { + OffsetEncoding encoding = page[0] == 0 ? OffsetEncoding.PLAIN : OffsetEncoding.DELTA_BINARY_PACKED; + assertThat(page[0]).as(name + ": offset encoding byte").isIn((byte) 0, (byte) 1); + pages.merge(encoding, 1, Integer::sum); + } + } + assertThat((int) pages.get(OffsetEncoding.PLAIN)) + .as("pages with a plain offset array") + .isEqualTo(4); + assertThat((int) pages.get(OffsetEncoding.DELTA_BINARY_PACKED)) + .as("pages with a packed offset array") + .isEqualTo(26); + } + + @Test + public void theReaderLearnsTheRepresentationFromTheTable() throws IOException { + Chunk chunk = readChunk("urls"); + SymbolTableValuesReader reader = chunk.reader(); + reader.initFromPage(30, stream(chunk.pages.get(0))); + assertThat(reader.symbolTableType()).isEqualTo(SymbolTableType.FSST_8); + } + + @Test + public void skippingLandsOnTheSameValueAsReadingWould() throws IOException { + Chunk chunk = readChunk("escapes"); + byte[] page = chunk.pages.get(0); + int count = expectation(readExpectations(), "escapes.0").count; + List straight = decodePage(chunk.reader(), page, count); + + for (int skipped = 0; skipped <= count; skipped++) { + SymbolTableValuesReader reader = chunk.reader(); + reader.initFromPage(count, stream(page)); + reader.skip(skipped); + for (int i = skipped; i < count; i++) { + assertThat(reader.readBytes()) + .as("skipped " + skipped + ", value " + i) + .isEqualTo(Binary.fromConstantByteArray(straight.get(i))); + } + } + } + + private static List decodePage(SymbolTableValuesReader reader, byte[] page, int count) throws IOException { + reader.initFromPage(count, stream(page)); + List values = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + values.add(reader.readBytes().getBytes()); + } + return values; + } + + private static ByteBufferInputStream stream(byte[] page) { + return ByteBufferInputStream.wrap(ByteBuffer.wrap(page)); + } + + private static String digest(List values) { + MessageDigest sha256; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + for (byte[] value : values) { + sha256.update(new byte[] { + (byte) (value.length >>> 24), + (byte) (value.length >>> 16), + (byte) (value.length >>> 8), + (byte) value.length + }); + sha256.update(value); + } + StringBuilder hex = new StringBuilder(); + for (byte b : sha256.digest()) { + hex.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return hex.toString(); + } + + private static Chunk readChunk(String name) throws IOException { + byte[] bytes = resource(name + ".pages"); + int pos = 0; + SymbolTableType type = SymbolTableType.fromTypeValue(bytes[pos++]); + int tableLength = readInt(bytes, pos); + pos += 4; + byte[] table = new byte[tableLength]; + System.arraycopy(bytes, pos, table, 0, tableLength); + pos += tableLength; + int pageCount = readInt(bytes, pos); + pos += 4; + List pages = new ArrayList<>(pageCount); + for (int i = 0; i < pageCount; i++) { + int length = readInt(bytes, pos); + pos += 4; + byte[] page = new byte[length]; + System.arraycopy(bytes, pos, page, 0, length); + pos += length; + pages.add(page); + } + assertThat(pos).as(name + ": bytes left over in the fixture").isEqualTo(bytes.length); + return new Chunk(type, table, pages); + } + + private static int readInt(byte[] bytes, int pos) { + return ((bytes[pos] & 0xFF) << 24) + | ((bytes[pos + 1] & 0xFF) << 16) + | ((bytes[pos + 2] & 0xFF) << 8) + | (bytes[pos + 3] & 0xFF); + } + + private static Map readExpectations() throws IOException { + Map expectations = new HashMap<>(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(open("expected.txt"), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + String[] fields = line.split(" "); + if (fields.length == 4) { + // A whole chunk: name, page count, value count, digest. + expectations.put( + fields[0], + new Expectation(Integer.parseInt(fields[1]), Integer.parseInt(fields[2]), fields[3])); + } else { + // One page of a chunk: name, value count, digest. + expectations.put(fields[0], new Expectation(-1, Integer.parseInt(fields[1]), fields[2])); + } + } + } + return expectations; + } + + private static Expectation expectation(Map expected, String name) { + Expectation expectation = expected.get(name); + if (expectation == null) { + throw new AssertionError("No expectation for " + name + " in " + DIRECTORY + "expected.txt"); + } + return expectation; + } + + private static byte[] resource(String name) throws IOException { + try (InputStream in = open(name)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + } + + private static InputStream open(String name) { + InputStream in = SymbolTableInteropTest.class.getResourceAsStream(DIRECTORY + name); + if (in == null) { + throw new UncheckedIOException(new IOException("Missing test resource " + DIRECTORY + name)); + } + return in; + } +} diff --git a/parquet-column/src/test/resources/fsst/interop/escapes-binary.pages b/parquet-column/src/test/resources/fsst/interop/escapes-binary.pages new file mode 100644 index 0000000000000000000000000000000000000000..9b3625002ec828ac7eceaa60b099c22974ccb4ef GIT binary patch literal 2116 zcmZvcOLyB;6ouv3vL)NGql@iBxFn$1q)ien;tCXy>V6he{X?o+g)X>6+ z|G~Ti{s6y%aTf!aGA-|CDf>od#=zP~*420R-8zyW2%d|}hQ8tv1vP9~<{3L3?>~Ip zZnk#!F6N7!#?-!lK5{0PPd2@de~wE3u1}R{b;g85RH4-oYt;R;ItR0yqgBAla*DGpXWpQrc;{(KY%tBkPXV5Rd=urfu8 zgSEYBQ5-B$tW$i#h82>zgls zq6k=W(oc~wJ4mG|f<5XP=#&Ws5fH#cr9>eZ({ecqAu&<$QV7nZicBH!YP&ZCVWsvz z1MuzF&)y%NJ^k|OIm=RL0S79D&`?DXp)os7lqrOUg*r~rFsoK5goZgJ5ylhBK=>2F zqYb0q_L@Oq^z%z)3NP3}rbJ=Dsy>_|i9{+-VWgT?C_H6{n4B;-uc;J-c_~?-AYf}- z>l6e`DflS}n4V5i5U`3)K@Kb9?=icb(kKXLRoO>DIMaPz3WBB<>J$LYytYPx5j)H( z6yQn7B??@@L<%g}Z81*)z{D(0d9Eo6OxQ8zE-+=cvKa~hwQ-{+p6t)qkyL_?*qv$> zI%kc79|~Bhl0yMnH3|hT8s!DdC9_btQ_@Kku(!Eppa8T~#yxv5=$B*^*sr2u%^o}^ zqzVdflX4ox0*w<;a7&Vc0tx!6ItqX{Fd3*|R@I;Y`zrmH3UK8zFr5iI(DP7m z{RNH#SI+pvlkSWSy|}rHZ-Eyvcj30kEp&n1*g_6nVD)4X1+Y04*zTAOb6nw8gM%)p zW&YHCwN1uBwqHiBFoy9KtfY#F(pFW`VJYao9 zkcZViB9N230CjEans_n*waiTqKrQN+o)0_{juCf`B+L=2bNsLbUvvIa4bTvSfdb)V iv)K9o9mPx)#hBgUb1=YbDPgh)Xn1{{_d&z5F8vEf{q>6g literal 0 HcmV?d00001 diff --git a/parquet-column/src/test/resources/fsst/interop/escapes.pages b/parquet-column/src/test/resources/fsst/interop/escapes.pages new file mode 100644 index 0000000000000000000000000000000000000000..44548edf57d0fc9c89c6cbe71d268b266c7ee2cd GIT binary patch literal 2458 zcmZ8i%Wm6N5G951O{b!&H}#_RNaC0fFW+3ylh{boqV}Re7ySVv$R+`#2H6!|CHpM0 zPyZnPihf5H`3?6B#Q>5G_|E0k+*j^WLlw@I=2X9}$ zdjIXsCfdZJ5JH50DsmBtSR{TXGLia4z7>I=Z^R~i`^NXN7s3~jFJfQhuW_{H#ecWG z-h{jcyhOYtykxw@yrjGcUJ71vUfy4C|MItN;ksIV|N5QJhqdsx^?BiMu3xdES|s%* z{c-T^!>8|`KW>BiJE~pPuPDnPQ1}bdAhxh4@S|1B5M^*o3^%6*3bJ)jT3I zTEQG*H`C6u(!*sA`oOGN$WVc5-oa%}BTew8L0b05gEJN10gQPjY2gYP>i8i+#t;z% zoM|b0-H%j@mpD;X(h8ZO!`&S2CCw&PbQoJwv)>+^nFgsr=Vdv84BJmCnRJk3gVgA* zQ8it?EKf*{mq<+`wdg>q(998rWsxpWrc=_h&Wm!OgA$zsO0+6Ud}%cu+4Wk@FAlh1 zc~0|7>-f0gwti4T4{#lFMet-a$VwSB%^)jg&;X-VtY8|yOHfD`H3 zg0nNp&RBEbs2w1z3E3gi8bC4OC{2~@2+<%4JB6hQWZZrMIP3y>PAsyyJaDau<W{_T9^&4?1a>ItrQfL>o*wqF#o(+^9Wf8AeAhgR zE$YP#^1jU;Q-Mw9fCgpg3%;RKKa81>psgui- zU;)(vO@PGR96!M3NZ^K{&?rMwFDx4IjFCX`Q3xa^8jX_WopCy1C$>tXl-mU~8FAnP zgGL3n1HN-WgULA-xH^F35u-x}ZRBIpIJ^_(IAJV6;cqjU`o~Y0TV&sRETjzfNMHh-7goZ_yGpa`KSR!$)PO}D&T#&7&iwo|nx^T|jmCQo zP1B4<-w)2h2hlJYjVG})y?=W1*4=wS+z-Mq2$Lung^3@Gf_^lNqa^YD_{@o%2e=3FinCe38N$!C&47~5e^f7l7vBW=ET0A1b!0w_&)JR_#F3rtRLMhK7Id| zbBYz6dv`-W9QPq2ZVZbXlj7<-_wRi5^_|mUagB>>ewU|}FEA~xVR0Q7*HJM)DB?*G zhqPY)d{D&W;#yqJ&@K}ld#lIz7UKVrZpK%%3l}u)ug|rYTlMjI=b<*XzkKTa+`Rqb z{a>1oUtIe1EqfLi|2mkCKT|g=NoUWPX)AW|qYp)>xAk_q^eA+6CtbMva@5+hW?JT2 zG+x=O%(VGb#@)Rd!bpb4$LaoObs6<~>5>^TYS}F=Y|8lXFx@aI)bFP&R*^%40ivpm z8g_#}Sd;O=0YnT^Y1&Ps;L*BAN9n?o>kduizwlK4Z4>#gJd)BKvpUl@9;wAD(=m-f_uE*KSEFj!IC(I7zm)p@;=Y@~{L&%pgg>geGDMs&Hq! zKD%;qu|!D>EMxZZ$wiBjIItWv!IGk5%i<4MB-*qss4iTKRFABxm_n+H@|P4B_c2PU=v;$WT_vU9AYDvI={QU`5h;zDe4lD0)q}qyspNAzFCtoc z$L4mPM^sYZ#15}(5P z)k#rzO2WN$(um<+lwfqoTNT@Cr`MTIvH|?OX1bK^+PR?(orFi+&~v7fa1Vw?6Ahh| zTlOx@!n#uC&_b;l(@?3$S*8w5zpBnd%TjMFZHvlyqk0NOPZgc2{w zyT1@!@seEnsrkqAfzfjx{r$Lg+x%zg{Ak{IHNEql_R9YA+w<4f>n-(?2vBEC5zhBP zY6Uc~8l%(@z#5~JYe8!TGvIp36w1O7p#qtp_kvmE1mHVk1UA9%4O3v#1;FP_femjL z_xO?#$^^nOh592HzGlQWj_@T@Wa9{*Ga{QHHexD~6B`EJp&|vbGp4v!5POFt8{FXw zR-w8$vGEfp_q-ssQK3E0k&P)N%aIMMDL!>%$+c*W|pJV!8MDwH#L&InyPAdF#BiwAs^FOFkWSKo)>i(is` zmcJzNC-d)5etXk>cH>EF_gQdNduayO#C!3-kNn)k8P0oFr}O~v4O7$u#g`r-4;0sy zo&sLW&QB%^a02CPPXSJ_91SoP;RMY$URRQY(_GA;JvhK|1!IhZKN^zFYJvxWwEXNHp0B1Q)dWu{*%00!dBUqX@PtfW= DvO9pn literal 0 HcmV?d00001 diff --git a/parquet-column/src/test/resources/fsst/interop/plain-offsets.pages b/parquet-column/src/test/resources/fsst/interop/plain-offsets.pages new file mode 100644 index 0000000000000000000000000000000000000000..f1b61fa3d42dd124598645b42319d46a17894e2e GIT binary patch literal 2320 zcmY*a+j8Q{6%`Vwb;(KQA*rM)g&=^hSb**VjLl$U#x}m&slwOks*wa3=z-D939-%i zlJCh|UXl;V!>P)zoNNJ}G7rs`M637SYpuORQB>$Z|A-a@iU|pS@MT-EbS-bHIb)D@ z`o&Ba_q^ukucYS*oIGnyt%*ZCI9`GnJfbYnrYZmZ6wA zOSSTOJEtkOqN|o>=(=GVvZ**L1_{!-3!OTclTNlCNJJIKnSxzYM-*KVctdf5rnR zJ2}pOJw=@!XisB)8kK3(rcs|pa~iE_G^UZFs4zv*+f)Rueh=$2tpCB<4N=s;VXe>< z_0yk!^Xs>te*N^fZ(oIB8kKN*=`Vl$`=^l7SropL8}1B;6ycs+na$l&fAC(c;V7f@ z2XXs$tKJ_(Q4ULokA#x=JshIJ0%?EOI*p&K3jB9SbBfan96s+v&hFv1 z8hTaEmK4F^HsS5P)-rT(QF7TDRIm?yRQiLM)-H2Tp#TQ z1R?zwRuzQwPgsdrF^NNo6`XMHfm|1RH4PGn4V*+lZ*&&N)*i{Z(c4^r=XD%MG$0g7 z;TQmNwd)z^j$HP_b1$S3VVm^M1G%)CdmuNd>WbB;8Ctnvvf^Z9IEOCb=mB6>ZXp&F za4lqUbQj{nZH5Dqx4LU%Af+Am2zGN3m#IS$ro*oRO>6BGB)t;FW`O}V5bq$Kd+qy; zX2Zj%5OH7ynRKLE6>jwS9G#>8yVyb9>8vYeAIS}0#_`A_X&k$A=pIBYICgP{S9&Pz-EvD%*n(#Pc~Yfo15_UuJWjm>QpF~h z1*n!`(<`JrPl59B6NzzO+QIf~)R$oMiF*Wy zfdtFWiSR@kB6oWhV9F?1+J1!+Bvk$(p%%#J7g#lr&%a_5aO!o{pQKe-B11It>5@EKlm}Q(;1|_6iBM|3?!?VC#h~_qMd?!fyeiO%@$<#bNZ&SpcC% z!M1$tiu-0sRe$kY=E27AOSEo9A6p%1)OJrRxegs zObT3Uy;C1o!?d+o$<7VmANOcw1ajU5TU-PJ>^5@>3E{(F180<^dt6{V>Bt<8hbJ@1 z3EcvWz~x>?wm_=etfU>We8u;8r2 zw<-aCAOiF8Lr;ZI*~imv;IPSTlL2$T7Bn2sj)F$!frcx-y$3TAEjSgh3e)9TFwcvM QI}g=J3r!A%N7TFj107X#x&QzG literal 0 HcmV?d00001 diff --git a/parquet-column/src/test/resources/fsst/interop/urls.pages b/parquet-column/src/test/resources/fsst/interop/urls.pages new file mode 100644 index 0000000000000000000000000000000000000000..6006ba28f022523d6aac912f661fca8f471a0d6b GIT binary patch literal 6936 zcmZu#S#KQKbuMw@$+WxKL$;m9%&4kjE<QY$~eVzI-ANn`E)*2a0;11wpc8tl1{Ri z4Rr_-5CCY5v2*-ScDD3sErOvyP+qz@CR!-P{x@teY{!^HZ*e&TQ>k=WcnOuWR8 z;e%J(iDA4h<#t}bF6Hqji9bdB$?~6rR~wtJ4!3(=u5E9YQYpO7$lsJBzhz;Df4(fG z)q9bY#;Z~a5=xGGBP$*CRT}TpvK>BpS;A|UAbV%zucM?XIkM7`_h}_T{yIvI?2?k- z8QIHGlCfi={2J!IUfVl>5u1Oww*PW(b9n7#c^JPBHxCcX2cI6|9sIDq@u|EofBFXp z<@L?O_1`~yeX!3Sc(q`kB+K0fB6@_ zNp1Z+Xqox)&o90A+lSQ(UHx62dtVJ*o9<#Z%1f*78qLvPetx+nrsh23WQ2!1Z7kC@ z>I1xPE6nR$qv3q<`Q`1n7GZq+&9D95m7nv61tsNZ`tNu;BD>eF@x(U7d*)eEq{kr7 zJf6Z}jel2nPv=p)bNr|LUj059`P)@3i4Iq6_`Y_+PPXV(`IkQtb2 zH%1YUkjR6QgDkzJE%@+gfp_3pvDP)hwJu)0rGv|u7^G2X)2YWJoGM-aAPjVbE2%iLT^^Dup0e`gf1j`6FuJdSLR%L0Db=?A9_>-eywu5&Aj$3O> zylN{3bKf<3sFNYHE#I^ruOJYZ@r_A}EU^YH*p%0;)2gypEb@$5>V^*iU7lSg2+?@& zqXC}U(XKG=QZJ)GfXFLzbcgrY<^cZ7g#do^0g~>+j_)pi@$teJ-&>hCM2il(2fvET zjT@AN7o%cE^f=lB{bJ{db&DRbl?rY~vrnnYw;t090x3CTL>oQu2R!8( zQG{{`UMPr5`l$z&y|0`Qsa=OQ%0Z))h9wbYr)P{)2xH*XGa{$ad1$v1%wDCZP(e0% zK!eI%2xp{zk9A?);Pg+Z8?aD{&C>%`OtRKDu2Y)R_ANf{Q72Sc6km){7`xa#fhbL8;uoZWDEsD2Z86dE$_P$vtLsQJbM72#8zny!(jVR&*;?dqEG zS{g>eVji3g0~tbDX8btXhM*zItGn^jLFFfAAIJTOC&Laf}Z(=G8`6O zhMA|-rIJgb%X~&THbq=r-Azylephkdpf!9J6amwrBBNG2?HY{(Y3Hi8A7vp%nA!~2 zkh9DzdXGWsP>Z3>5M9M2g6W%AsI{9-*kSm%=5vZZi;R0of6O=e7}34?WY@Wx;MV516ti}twm>fYi4q2 z)kAq~Jj0@Wv-LQea;y)R4d}B>&!7)-q`J#>0pg%` zb@wifz){Nl9F+pKtGgXE0boNwb@vAKak2=MT{C*z#V!MAT=N!{c$=V~@Xcs5!E?fg zo*6w!BDmV|uKAdv8R&i1`mq!fbP37X=8 zaH4BY(Nhq)PN?ecebs$vi23G2y2VQIJPd8Cr#U*nrf;@2+T&oa^HkiVNAQm(r2)DI zP@l^kAt{*BC7C5^eE}!wS>NoUTfCZ?SM>GGVOoLyI+;8()O2_~C#=j@cju@V>jU#b z^$8Am1&X4vQBbvFXP}Ke0LfaVE6{?ss#aF$ z8NXNYtHhd;4BDG;96W7kQL_+N(+8C9*ly@A}zwXf>Wy~O~f8c z@vTm}2|f&Z+saLv210_URbEgRd{xj z2}&aO(fqmAh6o5O@hwMHIBAutm1{H%O~H}Abga7m1y}~tF<=IZWPM*tR7kdRRzpxwH`kLGiyjxE0c=a zI9{-tc^^3;1XG!%1Td$YMbBDRCj`K*TIr(^-ch6HuFP;+0X{h@D$0OcI_M@8TNK4^ zP$&x%WDq^Rg&E;t>lTUGI73$}(dG*lz$yb1(UUH?4k-1k$0X-GBq=Xz%y5L*sjyin ze+d+a)by=>A@(i40l4(5|0jG4Ej=AS>0SEucOvho@I`7ktjnIBHA(|P5L|0qx@kd{ zYei0bfeNXn)>R5IUZgdsRvt(b8c>-Pskh%@Q<2E7#T5bAfZDz_uevfh09n^17np+- zAy)25D-w`#wK7HyysV?=Sre30r~*8usGHT4(^Iu_Uk+uem|G8N6hRjbcdfR1mi2r~ zi*1dkP)CvACXF-3{&Zr3ttCdgJM;Vp8?Im5vy8`AFpC829)EcXhfuI=b5H-oZ4e{`~=7k znn0FZ?`LS~{*6~RP$DH`U0aGk1njJW?2&Cj(wY)~>9Zt2Jqf_Jk z2;|vCy33~Q1#_LW0g-L2fp6cWSrvYcg#AL52T3P;nlca(utvF;(3~;aRH3#XQ3hX% z9VzNW`pPv!IYT4-;Wp&Eb~oYnWe1l%zrK)^>Oc~S{b1=sXKn6Jr>AvwbnW&UJj7h4 zKv-U)DfW@%oNGUk1G{Qx?Wa+3Mj9wfQ42yT5mdx%h(Il9o|3auf!K|5kUr&%zb zzCoy#!zWh}G%%oR-=-xNtpgEmpl-i)cUF#df)mdUAK?}u2(N3m*80H_%0r&rLn2k! z({4RRMb%}(vnQ!lE>QwXx7VqJI4Tup39Twn3@|Ly45lQr`UqEroWD%z_FZz=pF!+> zJ4a7At9C%O|8^m&Dowk;`rcQuj~D){hbDz;?G6e(=Y4|S-L-Gf45aH2xOVgyS7||) z=-Ibu02>A*S&lY`cpQYxx1%Sxo0BHqvmX;Kk)$f{OruNus4S8iG0URWA;<2Q>{E9v zcBGbp(4f04j}l-Z70n(eId{qJs2w>)CoKr{?5o^^LN}?2;b|Fbf=wfe1381>-<+= zq<{S*xdr;nc!A3x`6o7vsJsSMRLdn*{-vL=yJ!vp(2VTa!*WKJ&dd%qyWlz%n>o4z m2c73is2;&lL7mLXcL-%S;Nxm}fu>mBpx#u=dCDSrqW=q6A+6B> literal 0 HcmV?d00001 diff --git a/pom.xml b/pom.xml index 8bba194b48..ccddd23ecd 100644 --- a/pom.xml +++ b/pom.xml @@ -575,6 +575,9 @@ **/src/test/resources/fsst/*.table **/src/test/resources/fsst/*.codes + + **/src/test/resources/fsst/interop/*.pages + **/src/test/resources/fsst/interop/expected.txt From 2667916f4a06b31b4905032f56a7ca484f0ff1a4 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 22:01:42 +0000 Subject: [PATCH 07/10] Carry the symbol table from writer to reader through the column machinery Mirrors the dictionary page seam for seam: a SymbolTablePage alongside DictionaryPage, default methods on PageReader/PageWriter so no existing implementor breaks, and usesSymbolTable()/getSymbolTableBasedValuesReader on Encoding beside the dictionary equivalents. The publish moves off the writer's hot path and onto the chunk-finalize hook (toSymbolTablePageAndClose(), the toDictPageAndClose() analogue), which also fixes a real defect: FallbackValuesWriter.getBytes() used to call the initial writer's getBytes() - which published the table - before deciding whether to fall back, so a column that fell back to PLAIN or dictionary still published a table nobody would read. Publishing now only happens when the initial writer was actually used. SymbolTableSink and SymbolTables.rejectingSink() are retired: the page hook is a strictly better version of the same seam, and nothing depends on the old one since it was never wired to a real writer. ColumnReaderBase reads the symbol table page once per chunk and fails loudly if a column needs one that is not there, rather than guessing. --- .../org/apache/parquet/column/Encoding.java | 36 ++++++ .../parquet/column/impl/ColumnReaderBase.java | 11 ++ .../parquet/column/impl/ColumnWriterBase.java | 12 ++ .../parquet/column/page/PageReader.java | 7 ++ .../parquet/column/page/PageWriter.java | 10 ++ .../parquet/column/page/SymbolTablePage.java | 92 ++++++++++++++ .../parquet/column/values/ValuesWriter.java | 12 ++ .../factory/DefaultValuesWriterFactory.java | 2 - .../values/fallback/FallbackValuesWriter.java | 16 +++ .../values/symboltable/SymbolTableSink.java | 48 -------- .../values/symboltable/SymbolTableSource.java | 8 +- .../symboltable/SymbolTableValuesWriter.java | 22 ++-- .../values/symboltable/SymbolTables.java | 21 ---- .../column/page/mem/MemPageReader.java | 16 +++ .../parquet/column/page/mem/MemPageStore.java | 6 +- .../column/page/mem/MemPageWriter.java | 15 +++ .../SymbolTableValuesRoundTripTest.java | 116 +++++++++--------- 17 files changed, 308 insertions(+), 142 deletions(-) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/page/SymbolTablePage.java delete mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSink.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java b/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java index 64a172df26..d2a4aab0c3 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java @@ -54,6 +54,7 @@ import org.apache.parquet.column.values.plain.PlainValuesReader.LongPlainValuesReader; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesReader; import org.apache.parquet.column.values.rle.ZeroIntegerValuesReader; +import org.apache.parquet.column.values.symboltable.SymbolTable; import org.apache.parquet.column.values.symboltable.SymbolTableValuesReader; import org.apache.parquet.io.ParquetDecodingException; @@ -274,6 +275,20 @@ public ValuesReader getValuesReader(ColumnDescriptor descriptor, ValuesType valu } return new SymbolTableValuesReader(); } + + @Override + public boolean usesSymbolTable() { + return true; + } + + @Override + public ValuesReader getSymbolTableBasedValuesReader( + ColumnDescriptor descriptor, ValuesType valuesType, SymbolTable symbolTable) { + if (descriptor.getType() != BINARY) { + throw new ParquetDecodingException("Encoding FSST is only supported for type BINARY"); + } + return new SymbolTableValuesReader(() -> symbolTable); + } }; int getMaxLevel(ColumnDescriptor descriptor, ValuesType valuesType) { @@ -342,4 +357,25 @@ public ValuesReader getDictionaryBasedValuesReader( ColumnDescriptor descriptor, ValuesType valuesType, Dictionary dictionary) { throw new UnsupportedOperationException(this.name() + " is not dictionary based"); } + + /** + * @return whether this encoding requires a symbol table + */ + public boolean usesSymbolTable() { + return false; + } + + /** + * To read decoded values that require a symbol table + * + * @param descriptor the column to read + * @param valuesType the type of values + * @param symbolTable the symbol table for the chunk being read + * @return the proper values reader for the given column + * @throws UnsupportedOperationException if the encoding is not symbol table based + */ + public ValuesReader getSymbolTableBasedValuesReader( + ColumnDescriptor descriptor, ValuesType valuesType, SymbolTable symbolTable) { + throw new UnsupportedOperationException(this.name() + " is not symbol table based"); + } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderBase.java index 2b3e47116c..298f7de831 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderBase.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderBase.java @@ -39,9 +39,11 @@ import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.column.values.RequiresPreviousReader; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder; +import org.apache.parquet.column.values.symboltable.SymbolTable; import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.PrimitiveConverter; @@ -138,6 +140,7 @@ public double getDouble() { private final long totalValueCount; private final PageReader pageReader; private final Dictionary dictionary; + private final SymbolTable symbolTable; private IntIterator repetitionLevelColumn; private IntIterator definitionLevelColumn; @@ -457,6 +460,8 @@ void writeValue() { if (dictionary != null && converter.hasDictionarySupport()) { converter.setDictionary(dictionary); } + SymbolTablePage symbolTablePage = pageReader.readSymbolTablePage(); + this.symbolTable = symbolTablePage == null ? null : symbolTablePage.decode(); this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount <= 0) { throw new ParquetDecodingException("totalValueCount '" + totalValueCount + "' <= 0"); @@ -703,6 +708,12 @@ private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int + " as the dictionary was missing for encoding " + dataEncoding); } this.dataColumn = dataEncoding.getDictionaryBasedValuesReader(path, VALUES, dictionary); + } else if (dataEncoding.usesSymbolTable()) { + if (symbolTable == null) { + throw new ParquetDecodingException("could not read page in col " + path + + " as the symbol table was missing for encoding " + dataEncoding); + } + this.dataColumn = dataEncoding.getSymbolTableBasedValuesReader(path, VALUES, symbolTable); } else { this.dataColumn = dataEncoding.getValuesReader(path, VALUES); } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java index 408627404d..6f33de1ed6 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java @@ -25,6 +25,7 @@ import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageWriter; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.column.statistics.SizeStatistics; import org.apache.parquet.column.statistics.Statistics; import org.apache.parquet.column.statistics.geospatial.GeospatialStatistics; @@ -294,6 +295,17 @@ void finalizeColumnChunk() { dataColumn.resetDictionary(); } + final SymbolTablePage symbolTablePage = dataColumn.toSymbolTablePageAndClose(); + if (symbolTablePage != null) { + if (DEBUG) LOG.debug("write symbol table"); + try { + pageWriter.writeSymbolTablePage(symbolTablePage); + } catch (IOException e) { + throw new ParquetEncodingException("could not write symbol table page for " + path, e); + } + dataColumn.resetDictionary(); + } + collector.finalizeColumnChunk(); } catch (Throwable t) { statusManager.abort(); diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/PageReader.java b/parquet-column/src/main/java/org/apache/parquet/column/page/PageReader.java index 0b4321ca78..63c04c2af5 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/page/PageReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/page/PageReader.java @@ -28,6 +28,13 @@ public interface PageReader { */ DictionaryPage readDictionaryPage(); + /** + * @return the symbol table page in that chunk or null if none + */ + default SymbolTablePage readSymbolTablePage() { + return null; + } + /** * @return the total number of values in the column chunk */ diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java index 1d82db8c32..e184c8befd 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java @@ -226,6 +226,16 @@ default void writePageV2( */ void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOException; + /** + * writes a symbol table page + * + * @param symbolTablePage the symbol table page containing the table data + * @throws IOException if there was an exception while writing + */ + default void writeSymbolTablePage(SymbolTablePage symbolTablePage) throws IOException { + throw new UnsupportedOperationException("writeSymbolTablePage is not implemented"); + } + /** * @param prefix a prefix header to add at every line * @return a string presenting a summary of how memory is used diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/SymbolTablePage.java b/parquet-column/src/main/java/org/apache/parquet/column/page/SymbolTablePage.java new file mode 100644 index 0000000000..b38476df30 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/page/SymbolTablePage.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.page; + +import java.io.IOException; +import java.util.Objects; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.column.values.symboltable.SymbolTable; +import org.apache.parquet.column.values.symboltable.SymbolTableType; +import org.apache.parquet.column.values.symboltable.SymbolTables; +import org.apache.parquet.io.ParquetDecodingException; + +/** + * Data for a symbol table page. + * + *

A symbol table belongs to a column chunk the way a dictionary does: every page of the chunk is + * compressed against it, and it has to be readable before any of them. This page is that page's + * mirror, one per chunk instead of one per row of a data page. + */ +public class SymbolTablePage extends Page { + + private final BytesInput bytes; + private final SymbolTableType type; + + /** + * creates an uncompressed page + * + * @param bytes the content of the page + * @param type the symbol table representation + */ + public SymbolTablePage(BytesInput bytes, SymbolTableType type) { + this(bytes, (int) bytes.size(), type); + } + + /** + * creates a symbol table page + * + * @param bytes the (possibly compressed) content of the page + * @param uncompressedSize the size uncompressed + * @param type the symbol table representation + */ + public SymbolTablePage(BytesInput bytes, int uncompressedSize, SymbolTableType type) { + super(Math.toIntExact(bytes.size()), uncompressedSize); + this.bytes = Objects.requireNonNull(bytes, "bytes cannot be null"); + this.type = Objects.requireNonNull(type, "type cannot be null"); + } + + public BytesInput getBytes() { + return bytes; + } + + public SymbolTableType getType() { + return type; + } + + public SymbolTablePage copy() throws IOException { + return new SymbolTablePage(BytesInput.copy(bytes), getUncompressedSize(), type); + } + + /** + * @return the decoded symbol table + */ + public SymbolTable decode() { + try { + return SymbolTables.deserialize(type, bytes.toByteArray(), 0, (int) bytes.size()); + } catch (IOException e) { + throw new ParquetDecodingException("could not decode the symbol table", e); + } + } + + @Override + public String toString() { + return "SymbolTablePage [bytes.size=" + bytes.size() + ", type=" + type + ", uncompressedSize=" + + getUncompressedSize() + "]"; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriter.java index ecea4a7520..5ab4bdc45b 100755 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriter.java @@ -21,6 +21,7 @@ import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.io.api.Binary; /** @@ -72,6 +73,17 @@ public DictionaryPage toDictPageAndClose() { return null; } + /** + * Returns the symbol table generated by this writer if one was created. + * As part of this operation the table is closed and will not have + * any new values written into it. + * + * @return the symbol table page or null if not symbol table based + */ + public SymbolTablePage toSymbolTablePageAndClose() { + return null; + } + /** * reset the dictionary when a new block starts */ diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java index 4cc4e05c80..467737a5ca 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java @@ -27,7 +27,6 @@ import org.apache.parquet.column.values.fallback.FallbackValuesWriter; import org.apache.parquet.column.values.symboltable.SymbolTableType; import org.apache.parquet.column.values.symboltable.SymbolTableValuesWriter; -import org.apache.parquet.column.values.symboltable.SymbolTables; /** * Handles ValuesWriter creation statically based on the types of the columns and the writer version. @@ -122,7 +121,6 @@ static ValuesWriter symbolTableWriterWithFallBack( return FallbackValuesWriter.of( new SymbolTableValuesWriter( type, - SymbolTables.rejectingSink(), properties.getSymbolTableOffsetEncoding(), properties.getInitialSlabSize(), properties.getPageSizeThreshold(), diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/fallback/FallbackValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/fallback/FallbackValuesWriter.java index 41fe484f37..a9402b0d42 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/fallback/FallbackValuesWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/fallback/FallbackValuesWriter.java @@ -21,6 +21,7 @@ import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.column.values.RequiresFallback; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.io.api.Binary; @@ -51,6 +52,8 @@ public static A symbol table belongs to a column chunk rather than to a page: every page of the chunk is - * compressed against it, and it has to be readable before any of them. A values writer cannot write - * anything outside its own page, so it publishes the table here instead and something above it - * decides where the bytes go. - * - *

That indirection is the point. It is a test harness today, and a page written next to the - * dictionary page once the format carries one, and neither choice reaches the writer. - */ -public interface SymbolTableSink { - - /** - * Publishes the table a column chunk's pages are compressed against. - * - *

Called once per chunk, when the first page is compressed. A chunk that then abandons the - * encoding — because the codes did not come out smaller than the values — leaves a table behind - * that no page refers to, so whoever stores it should write it only if some page of the chunk was - * actually written with the encoding. - * - * @param type the representation, which a reader needs in order to interpret the body - * @param body the serialized table - */ - void putSymbolTable(SymbolTableType type, BytesInput body); -} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java index 6335e5594d..3f8dca74bd 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableSource.java @@ -21,9 +21,11 @@ /** * Where a reader gets the symbol table a column chunk's pages were compressed against. * - *

The counterpart of {@link SymbolTableSink}. The table arrives already deserialized, because - * which implementation to build from the bytes depends on the representation and that decision - * belongs in one place: {@link SymbolTables}. + *

The write side's counterpart is + * {@link org.apache.parquet.column.values.ValuesWriter#toSymbolTablePageAndClose()}, which hands the + * table to the column's {@link org.apache.parquet.column.page.PageWriter} once per chunk. The table + * here arrives already deserialized, because which implementation to build from the bytes depends + * on the representation and that decision belongs in one place: {@link SymbolTables}. */ public interface SymbolTableSource { diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java index 70b37440d9..1833df4905 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesWriter.java @@ -21,6 +21,7 @@ import org.apache.parquet.bytes.ByteBufferAllocator; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.column.values.RequiresFallback; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; @@ -37,11 +38,13 @@ *

What happens when

* *

Values are buffered raw as they arrive. The table is trained at the first {@link #getBytes()}, - * on that page's values, and then published through the {@link SymbolTableSink} and kept for the - * rest of the chunk: a table belongs to a column chunk, so a later page must not train its own. - * Training on the first page rather than on the whole chunk is deliberate — a page is already far - * more text than a trainer samples, and buffering the chunk to feed it would cost a second copy of - * the column. + * on that page's values, and kept for the rest of the chunk: a table belongs to a column chunk, so + * a later page must not train its own. It is handed off through {@link #toSymbolTablePageAndClose()}, + * called once per chunk after every page has been written, which is also what keeps a chunk that + * falls back to another encoding from leaving a table behind that no page refers to. Training on + * the first page rather than on the whole chunk is deliberate — a page is already far more text + * than a trainer samples, and buffering the chunk to feed it would cost a second copy of the + * column. * *

Buffering is the reason values are held at all: a trainer reads them in an order of its own * choosing and more than once, and a fallback to another encoding has to replay them. The cost is @@ -59,7 +62,6 @@ public class SymbolTableValuesWriter extends ValuesWriter implements RequiresFallback { private final SymbolTableType type; - private final SymbolTableSink sink; private final ValueBuffer values; private final SymbolTablePayloadWriter payload; @@ -80,13 +82,11 @@ public class SymbolTableValuesWriter extends ValuesWriter implements RequiresFal public SymbolTableValuesWriter( SymbolTableType type, - SymbolTableSink sink, OffsetEncoding offsetEncoding, int initialSlabSize, int pageSize, ByteBufferAllocator allocator) { this.type = type; - this.sink = sink; this.values = new ValueBuffer(); this.payload = new SymbolTablePayloadWriter(offsetEncoding, initialSlabSize, pageSize, allocator); } @@ -118,12 +118,16 @@ public long getBufferedSize() { public BytesInput getBytes() { if (trained == null) { trained = SymbolTables.trainer(type).train(values); - sink.putSymbolTable(type, trained.table().serialize()); } compressBufferedValues(); return payload.getBytes(); } + @Override + public SymbolTablePage toSymbolTablePageAndClose() { + return trained == null ? null : new SymbolTablePage(trained.table().serialize(), type); + } + @Override public Encoding getEncoding() { return Encoding.FSST; diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java index b35ab6f0be..3ea11380b9 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/symboltable/SymbolTables.java @@ -21,7 +21,6 @@ import org.apache.parquet.column.values.symboltable.fsst.Fsst8SymbolTable; import org.apache.parquet.column.values.symboltable.fsst.FsstTrainer; import org.apache.parquet.io.ParquetDecodingException; -import org.apache.parquet.io.ParquetEncodingException; /** * The one place that maps a symbol table representation to the code that implements it. @@ -44,26 +43,6 @@ public static SymbolTableTrainer trainer(SymbolTableType type) { } } - /** - * A sink that refuses the table instead of storing it. - * - *

What a writer gets when nothing has told it where the table should go, which is every writer - * built from the format's own metadata today: the format has no place for a symbol table, so a file - * written with the encoding could not be read back. Refusing at the first page fails while the - * failure still names the reason, rather than producing a file whose pages nothing can decode. - * - *

Supplying a sink that does store the table is what the encoding is waiting on. Until the format - * carries one, a caller that has somewhere to put it can build the writer itself and install it with - * {@code ParquetProperties.Builder.withValuesWriterFactory}. - */ - public static SymbolTableSink rejectingSink() { - return (type, body) -> { - throw new ParquetEncodingException("Cannot write a symbol table encoded column: the format has nowhere " - + "to keep the chunk's symbol table, so the pages would not be readable. See parquet-format " - + "issue #531."); - }; - } - /** * Rebuilds a table from a serialized body. * diff --git a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageReader.java b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageReader.java index c44a7b644c..0805c4c5cb 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageReader.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageReader.java @@ -23,6 +23,7 @@ import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.io.ParquetDecodingException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,12 +34,22 @@ public class MemPageReader implements PageReader { private final long totalValueCount; private final Iterator pages; private final DictionaryPage dictionaryPage; + private final SymbolTablePage symbolTablePage; public MemPageReader(long totalValueCount, Iterator pages, DictionaryPage dictionaryPage) { + this(totalValueCount, pages, dictionaryPage, null); + } + + public MemPageReader( + long totalValueCount, + Iterator pages, + DictionaryPage dictionaryPage, + SymbolTablePage symbolTablePage) { super(); this.pages = Objects.requireNonNull(pages, "pages cannot be null"); this.totalValueCount = totalValueCount; this.dictionaryPage = dictionaryPage; + this.symbolTablePage = symbolTablePage; } @Override @@ -61,4 +72,9 @@ public DataPage readPage() { public DictionaryPage readDictionaryPage() { return dictionaryPage; } + + @Override + public SymbolTablePage readSymbolTablePage() { + return symbolTablePage; + } } diff --git a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageStore.java b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageStore.java index f5b66fd88b..8981371222 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageStore.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageStore.java @@ -62,7 +62,11 @@ public PageReader getPageReader(ColumnDescriptor descriptor) { } List pages = new ArrayList<>(pageWriter.getPages()); LOG.debug("initialize page reader with {} values and {} pages", pageWriter.getTotalValueCount(), pages.size()); - return new MemPageReader(pageWriter.getTotalValueCount(), pages.iterator(), pageWriter.getDictionaryPage()); + return new MemPageReader( + pageWriter.getTotalValueCount(), + pages.iterator(), + pageWriter.getDictionaryPage(), + pageWriter.getSymbolTablePage()); } @Override diff --git a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java index 1594c119fd..8a4710d2e4 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java @@ -30,6 +30,7 @@ import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageWriter; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.column.statistics.SizeStatistics; import org.apache.parquet.column.statistics.Statistics; import org.apache.parquet.column.statistics.geospatial.GeospatialStatistics; @@ -42,6 +43,7 @@ public class MemPageWriter implements PageWriter { private final List pages = new ArrayList<>(); private DictionaryPage dictionaryPage; + private SymbolTablePage symbolTablePage; private long memSize = 0; private long totalValueCount = 0; @@ -157,6 +159,10 @@ public DictionaryPage getDictionaryPage() { return dictionaryPage; } + public SymbolTablePage getSymbolTablePage() { + return symbolTablePage; + } + public long getTotalValueCount() { return totalValueCount; } @@ -180,6 +186,15 @@ public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOExceptio dictionaryPage.getDictionarySize()); } + @Override + public void writeSymbolTablePage(SymbolTablePage symbolTablePage) throws IOException { + if (this.symbolTablePage != null) { + throw new ParquetEncodingException("Only one symbol table page per block"); + } + this.memSize += symbolTablePage.getBytes().size(); + this.symbolTablePage = symbolTablePage.copy(); + } + @Override public String memUsageString(String prefix) { return String.format("%s %,d bytes", prefix, memSize); diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java index 2d20e18fff..47f7a1eec0 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableValuesRoundTripTest.java @@ -34,6 +34,7 @@ import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.ValuesType; +import org.apache.parquet.column.page.SymbolTablePage; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.fallback.FallbackValuesWriter; @@ -41,7 +42,6 @@ import org.apache.parquet.column.values.plain.PlainValuesWriter; import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; import org.apache.parquet.io.ParquetDecodingException; -import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.junit.jupiter.api.Test; @@ -61,27 +61,19 @@ public class SymbolTableValuesRoundTripTest { private static final int PAGE_SIZE = 1 << 20; /** - * Stands in for wherever the serialized table ends up. + * Stands in for wherever the deserialized {@link SymbolTablePage} ends up. * - *

Both halves of the seam, so a test can hand the writer's own table straight back to the - * reader. Deserializing on every call is deliberate: it means the reader is decoding against a table - * rebuilt from bytes rather than against the trainer's own object. + *

Deserializing on every {@link #getSymbolTable()} call is deliberate: it means the reader is + * decoding against a table rebuilt from bytes rather than against the trainer's own object. */ - private static final class SymbolTableRelay implements SymbolTableSink, SymbolTableSource { + private static final class SymbolTableRelay implements SymbolTableSource { private SymbolTableType type; private byte[] body; - private int publishCount; - @Override - public void putSymbolTable(SymbolTableType type, BytesInput body) { - this.type = type; - try { - this.body = body.toByteArray(); - } catch (IOException e) { - throw new AssertionError(e); - } - this.publishCount++; + void receive(SymbolTablePage page) throws IOException { + this.type = page.getType(); + this.body = page.getBytes().toByteArray(); } @Override @@ -90,14 +82,9 @@ public SymbolTable getSymbolTable() { } } - private static SymbolTableValuesWriter writer(SymbolTableSink sink, OffsetEncoding offsetEncoding) { + private static SymbolTableValuesWriter writer(OffsetEncoding offsetEncoding) { return new SymbolTableValuesWriter( - SymbolTableType.FSST_8, - sink, - offsetEncoding, - SLAB_SIZE, - PAGE_SIZE, - HeapByteBufferAllocator.getInstance()); + SymbolTableType.FSST_8, offsetEncoding, SLAB_SIZE, PAGE_SIZE, HeapByteBufferAllocator.getInstance()); } private static List binaries(String... values) { @@ -113,7 +100,7 @@ private static List> roundTrip(List> pages, OffsetEnco throws IOException { SymbolTableRelay relay = new SymbolTableRelay(); List bodies = new ArrayList<>(); - try (SymbolTableValuesWriter writer = writer(relay, offsetEncoding)) { + try (SymbolTableValuesWriter writer = writer(offsetEncoding)) { for (List page : pages) { for (Binary value : page) { writer.writeBytes(value); @@ -122,8 +109,10 @@ private static List> roundTrip(List> pages, OffsetEnco bodies.add(writer.getBytes().toByteArray()); writer.reset(); } + SymbolTablePage tablePage = writer.toSymbolTablePageAndClose(); + assertThat(tablePage).as("one table for the whole chunk").isNotNull(); + relay.receive(tablePage); } - assertThat(relay.publishCount).as("one table for the whole chunk").isEqualTo(1); SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); List> read = new ArrayList<>(); @@ -240,21 +229,29 @@ public void oneTableTrainedOnTheFirstPageServesLaterPages() throws IOException { /** A new chunk trains a new table, which is what a row group boundary needs. */ @Test public void resetDictionaryTrainsAgain() throws IOException { - SymbolTableRelay relay = new SymbolTableRelay(); - try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + try (SymbolTableValuesWriter writer = writer(OffsetEncoding.DELTA_BINARY_PACKED)) { for (Binary value : binaries("alpha", "alphabet", "alpine")) { writer.writeBytes(value); } writer.getBytes(); writer.reset(); - assertThat(relay.publishCount).isEqualTo(1); + SymbolTablePage first = writer.toSymbolTablePageAndClose(); + assertThat(first).as("the first chunk trained a table").isNotNull(); writer.resetDictionary(); + assertThat(writer.toSymbolTablePageAndClose()) + .as("nothing has been trained yet for the new chunk") + .isNull(); + for (Binary value : binaries("zeta", "zenith", "zephyr")) { writer.writeBytes(value); } writer.getBytes(); - assertThat(relay.publishCount).isEqualTo(2); + SymbolTablePage second = writer.toSymbolTablePageAndClose(); + assertThat(second).as("the second chunk trained its own table").isNotNull(); + assertThat(second.getBytes().toByteArray()) + .as("a table trained on different values") + .isNotEqualTo(first.getBytes().toByteArray()); } } @@ -263,11 +260,12 @@ public void skipReachesTheSameValuesAsReading() throws IOException { List values = binaries("alpha", "", "alphabet", "beta", "betamax", "gamma", "gamma-ray", "delta"); SymbolTableRelay relay = new SymbolTableRelay(); byte[] body; - try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + try (SymbolTableValuesWriter writer = writer(OffsetEncoding.DELTA_BINARY_PACKED)) { for (Binary value : values) { writer.writeBytes(value); } body = writer.getBytes().toByteArray(); + relay.receive(writer.toSymbolTablePageAndClose()); } for (int start = 0; start < values.size(); start++) { @@ -295,9 +293,10 @@ public void skipReachesTheSameValuesAsReading() throws IOException { public void readingPastTheEndOfAPageIsRejected() throws IOException { SymbolTableRelay relay = new SymbolTableRelay(); byte[] body; - try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + try (SymbolTableValuesWriter writer = writer(OffsetEncoding.DELTA_BINARY_PACKED)) { writer.writeBytes(Binary.fromString("one")); body = writer.getBytes().toByteArray(); + relay.receive(writer.toSymbolTablePageAndClose()); } SymbolTableValuesReader reader = new SymbolTableValuesReader(relay); reader.initFromPage(1, ByteBufferInputStream.wrap(ByteBuffer.wrap(body))); @@ -315,24 +314,6 @@ public void aReaderWithNoTableSaysSoRatherThanFailingLater() { .hasMessageContaining("#531"); } - @Test - public void aWriterWithNowhereToPutItsTableSaysSoBeforeWritingAPage() throws IOException { - // What a writer built from the format's own settings gets, because the format has no place for a - // symbol table. Refusing at the first page beats writing pages nothing can decode. - try (SymbolTableValuesWriter writer = new SymbolTableValuesWriter( - SymbolTableType.FSST_8, - SymbolTables.rejectingSink(), - OffsetEncoding.DELTA_BINARY_PACKED, - SLAB_SIZE, - PAGE_SIZE, - HeapByteBufferAllocator.getInstance())) { - writer.writeBytes(Binary.fromString("http://example.com/a")); - assertThatThrownBy(writer::getBytes) - .isInstanceOf(ParquetEncodingException.class) - .hasMessageContaining("#531"); - } - } - @Test public void theEncodingHandsOutTheReaderForBinaryOnly() { assertThat(Encoding.FSST.getValuesReader(descriptor(PrimitiveTypeName.BINARY), ValuesType.VALUES)) @@ -352,6 +333,9 @@ public void theEncodingHandsOutTheReaderForBinaryOnly() { * what a plain page's lengths cost, so the nine-byte header alone decides it. That is the * configuration to fall back from, and it is also the argument against writing offsets plain — the * same page with packed offsets is smaller than plain and keeps the encoding. + * + *

Also the regression test for a fixed publish-timing defect: a chunk that falls back must not + * leave a symbol table page behind that no page refers to. */ @Test public void aPageTheEncodingWouldGrowIsWrittenPlain() throws IOException { @@ -365,19 +349,35 @@ public void aPageTheEncodingWouldGrowIsWrittenPlain() throws IOException { .isEqualTo(Encoding.FSST); } - /** Runs a page through the fallback wrapper and reads it back with whatever encoding it chose. */ + /** + * Runs a page through the fallback wrapper, reads it back with whatever encoding it chose, and + * checks that a symbol table page was published if and only if the encoding kept FSST. + */ private static Encoding fallbackEncodingFor(List values, OffsetEncoding offsetEncoding) throws IOException { SymbolTableRelay relay = new SymbolTableRelay(); Encoding encoding; byte[] body; + SymbolTablePage tablePage; try (FallbackValuesWriter writer = FallbackValuesWriter.of( - writer(relay, offsetEncoding), + writer(offsetEncoding), new PlainValuesWriter(SLAB_SIZE, PAGE_SIZE, HeapByteBufferAllocator.getInstance()))) { for (Binary value : values) { writer.writeBytes(value); } body = writer.getBytes().toByteArray(); encoding = writer.getEncoding(); + tablePage = writer.toSymbolTablePageAndClose(); + } + + if (encoding == Encoding.PLAIN) { + assertThat(tablePage) + .as("a chunk that fell back publishes no table") + .isNull(); + } else { + assertThat(tablePage) + .as("a chunk that kept the encoding publishes its table") + .isNotNull(); + relay.receive(tablePage); } ValuesReader reader = @@ -393,29 +393,30 @@ private static Encoding fallbackEncodingFor(List values, OffsetEncoding @Test public void fallingBackReplaysEveryValue() throws IOException { List values = binaries("alpha", "", "beta", "gamma"); - SymbolTableRelay relay = new SymbolTableRelay(); List replayed = new ArrayList<>(); ValuesWriter collector = new CollectingValuesWriter(replayed); - try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + try (SymbolTableValuesWriter writer = writer(OffsetEncoding.DELTA_BINARY_PACKED)) { for (Binary value : values) { writer.writeBytes(value); } writer.fallBackAllValuesTo(collector); } assertThat(replayed).isEqualTo(values); - assertThat(relay.publishCount).isEqualTo(0); } @Test public void aTableIsPublishedOnceAndRebuiltFromItsBytes() throws IOException { SymbolTableRelay relay = new SymbolTableRelay(); - try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + SymbolTablePage tablePage; + try (SymbolTableValuesWriter writer = writer(OffsetEncoding.DELTA_BINARY_PACKED)) { for (int i = 0; i < 200; i++) { writer.writeBytes(Binary.fromString("measurement-" + i)); } writer.getBytes(); + tablePage = writer.toSymbolTablePageAndClose(); } - assertThat(relay.type).isEqualTo(SymbolTableType.FSST_8); + assertThat(tablePage.getType()).isEqualTo(SymbolTableType.FSST_8); + relay.receive(tablePage); SymbolTable first = relay.getSymbolTable(); assertThat(first.type()).isEqualTo(SymbolTableType.FSST_8); assertThat(first.symbolCount()).as("a table was trained").isPositive(); @@ -424,8 +425,7 @@ public void aTableIsPublishedOnceAndRebuiltFromItsBytes() throws IOException { @Test public void theWriterReportsWhatItIsHoldingAndReleasesItOnReset() throws IOException { - SymbolTableRelay relay = new SymbolTableRelay(); - try (SymbolTableValuesWriter writer = writer(relay, OffsetEncoding.DELTA_BINARY_PACKED)) { + try (SymbolTableValuesWriter writer = writer(OffsetEncoding.DELTA_BINARY_PACKED)) { assertThat(writer.getBufferedSize()).isEqualTo(0); writer.writeBytes(Binary.fromString("alpha")); assertThat(writer.getBufferedSize()).isEqualTo(5 + 4); From 6494e81f6f18dc787b5d10997d0e2013d1303087 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 22:01:51 +0000 Subject: [PATCH 08/10] Add an end-to-end test for the symbol table transport Values through the column machinery, not just through the values writer/reader pair: a MemPageStore, a real ColumnWriteStoreV1, a real ColumnReadStoreImpl. Covers a high-cardinality column that keeps the encoding and publishes a table, a multi-page chunk sharing one table, nulls and empty strings on an optional column, a column that expands and falls back with no table published, skip() and re-reading a column from the same store, both offset encodings, two row groups each training their own table, and the negative control the plan asked for: strip the symbol table out of the page reader and confirm the column reader refuses to proceed rather than silently decoding as if the encoding were something else. --- .../symboltable/SymbolTableEndToEndTest.java | 360 ++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableEndToEndTest.java diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableEndToEndTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableEndToEndTest.java new file mode 100644 index 0000000000..17c5183e41 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableEndToEndTest.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ColumnReader; +import org.apache.parquet.column.ColumnWriter; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.column.impl.ColumnReadStoreImpl; +import org.apache.parquet.column.impl.ColumnWriteStoreV1; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.mem.MemPageReader; +import org.apache.parquet.column.page.mem.MemPageStore; +import org.apache.parquet.column.page.mem.MemPageWriter; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.example.DummyRecordConverter; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.jupiter.api.Test; + +/** + * Values through the column machinery, not just through the values writer/reader pair: a + * {@link MemPageStore}, a real {@link ColumnWriteStoreV1}, a real {@link ColumnReadStoreImpl}, the + * transport built in {@code column/page} and {@code column/impl} carrying the table between them. + * + *

{@link SymbolTableValuesRoundTripTest} already covers the writer and reader in isolation; what + * only this level can catch is a table that never reaches the writer's chunk-finalize hook, or a + * reader built before the table it needs exists. + */ +public class SymbolTableEndToEndTest { + + private static final ColumnDescriptor REQUIRED_BINARY = + requiredBinaryColumn().getColumnDescription(new String[] {"foo", "bar"}); + + private static MessageType requiredBinaryColumn() { + return MessageTypeParser.parseMessageType("message msg { required group foo { required binary bar; } }"); + } + + private static List binaries(String... values) { + List result = new ArrayList<>(); + for (String value : values) { + result.add(Binary.fromString(value)); + } + return result; + } + + /** Each word repeated enough times that FSST beats the delta-byte-array fallback it competes with. */ + private static List repeatedBinaries(String... words) { + List result = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + for (String word : words) { + result.add(Binary.fromString(word)); + } + } + return result; + } + + private static ColumnWriteStoreV1 fsstWriteStore(MemPageStore memPageStore) { + return fsstWriteStore(memPageStore, OffsetEncoding.DELTA_BINARY_PACKED); + } + + private static ColumnWriteStoreV1 fsstWriteStore(MemPageStore memPageStore, OffsetEncoding offsetEncoding) { + return new ColumnWriteStoreV1( + memPageStore, + ParquetProperties.builder() + .withDictionaryEncoding(false) + .withFsstEncoding(true) + .withSymbolTableOffsetEncoding(offsetEncoding) + .build()); + } + + private static void writeChunk(ColumnWriteStoreV1 store, ColumnDescriptor path, List values) { + ColumnWriter writer = store.getColumnWriter(path); + for (Binary value : values) { + writer.write(value, 0, 0); + store.endRecord(); + } + store.flush(); + } + + private static ColumnReader columnReader(MemPageStore memPageStore, ColumnDescriptor path, MessageType schema) { + return new ColumnReadStoreImpl(memPageStore, new DummyRecordConverter(schema).getRootConverter(), schema, null) + .getColumnReader(path); + } + + /** Reads every value of a required column, in order. */ + private static List readBinaries(MemPageStore memPageStore, ColumnDescriptor path, MessageType schema) { + ColumnReader reader = columnReader(memPageStore, path, schema); + List read = new ArrayList<>(); + long count = reader.getTotalValueCount(); + for (long i = 0; i < count; i++) { + read.add(reader.getBinary()); + reader.consume(); + } + return read; + } + + private static MemPageWriter pageWriterFor(MemPageStore memPageStore, ColumnDescriptor path) { + return (MemPageWriter) memPageStore.getPageWriter(path); + } + + @Test + public void aHighCardinalityColumnKeepsTheEncodingAndPublishesATable() { + MessageType schema = requiredBinaryColumn(); + ColumnDescriptor path = REQUIRED_BINARY; + List values = new ArrayList<>(); + for (int i = 0; i < 2000; i++) { + values.add(Binary.fromString("https://example.com/catalogue/item/" + i + "?ref=newsletter")); + } + + MemPageStore memPageStore = new MemPageStore(values.size()); + writeChunk(fsstWriteStore(memPageStore), path, values); + + MemPageWriter pageWriter = pageWriterFor(memPageStore, path); + assertThat(pageWriter.getSymbolTablePage()) + .as("a chunk that wins with FSST publishes a table") + .isNotNull(); + for (DataPage page : pageWriter.getPages()) { + assertThat(((DataPageV1) page).getValueEncoding()).isEqualTo(Encoding.FSST); + } + + assertThat(readBinaries(memPageStore, path, schema)).isEqualTo(values); + } + + @Test + public void multiplePagesShareOneTable() { + MessageType schema = requiredBinaryColumn(); + ColumnDescriptor path = REQUIRED_BINARY; + String[] themes = {"warehouse-inventory", "flight-departure", "clinical-observation", "seismic-reading"}; + List values = new ArrayList<>(); + for (String theme : themes) { + for (int i = 0; i < 400; i++) { + values.add(Binary.fromString(theme + "/" + i)); + } + } + + MemPageStore memPageStore = new MemPageStore(values.size()); + ColumnWriteStoreV1 store = new ColumnWriteStoreV1( + memPageStore, + ParquetProperties.builder() + .withDictionaryEncoding(false) + .withFsstEncoding(true) + .withPageSize(2048) + .withMinRowCountForPageSizeCheck(1) + .build()); + writeChunk(store, path, values); + + MemPageWriter pageWriter = pageWriterFor(memPageStore, path); + assertThat(pageWriter.getPages().size()) + .as("the chunk crossed the page-size threshold") + .isGreaterThan(1); + assertThat(pageWriter.getSymbolTablePage()) + .as("one table for the whole chunk") + .isNotNull(); + + assertThat(readBinaries(memPageStore, path, schema)).isEqualTo(values); + } + + @Test + public void nullsAndEmptyStringsSurviveAnOptionalColumn() { + MessageType schema = MessageTypeParser.parseMessageType("message msg { optional binary foo; }"); + ColumnDescriptor path = schema.getColumns().get(0); + + List present = new ArrayList<>(); + for (int i = 0; i < 300; i++) { + present.add(Binary.fromString(i % 7 == 0 ? "" : "value-" + (i % 11))); + } + + MemPageStore memPageStore = new MemPageStore(present.size() * 2); + ColumnWriteStoreV1 store = fsstWriteStore(memPageStore); + ColumnWriter writer = store.getColumnWriter(path); + List expected = new ArrayList<>(); + for (int i = 0; i < present.size(); i++) { + if (i % 3 == 0) { + writer.writeNull(0, 0); + expected.add(null); + } else { + Binary value = present.get(i); + writer.write(value, 0, 1); + expected.add(value); + } + store.endRecord(); + } + store.flush(); + + ColumnReader reader = columnReader(memPageStore, path, schema); + List read = new ArrayList<>(); + long count = reader.getTotalValueCount(); + for (long i = 0; i < count; i++) { + read.add(reader.getCurrentDefinitionLevel() == 0 ? null : reader.getBinary()); + reader.consume(); + } + assertThat(read).isEqualTo(expected); + } + + @Test + public void aColumnThatExpandsFallsBackAndPublishesNoTable() { + MessageType schema = requiredBinaryColumn(); + ColumnDescriptor path = REQUIRED_BINARY; + List values = new ArrayList<>(); + for (int i = 0; i < 256; i++) { + values.add(Binary.fromConstantByteArray(new byte[] {(byte) i})); + } + + MemPageStore memPageStore = new MemPageStore(values.size()); + writeChunk(fsstWriteStore(memPageStore, OffsetEncoding.PLAIN), path, values); + + MemPageWriter pageWriter = pageWriterFor(memPageStore, path); + assertThat(pageWriter.getSymbolTablePage()) + .as("a chunk that fell back publishes no table") + .isNull(); + for (DataPage page : pageWriter.getPages()) { + assertThat(((DataPageV1) page).getValueEncoding()).isEqualTo(Encoding.PLAIN); + } + + assertThat(readBinaries(memPageStore, path, schema)).isEqualTo(values); + } + + @Test + public void aColumnCanBeReadTwiceAndSkippedAtSeveralPositions() { + MessageType schema = requiredBinaryColumn(); + ColumnDescriptor path = REQUIRED_BINARY; + List values = + binaries("alpha", "", "alphabet", "beta", "betamax", "gamma", "gamma-ray", "delta", "delta-force"); + + MemPageStore memPageStore = new MemPageStore(values.size()); + writeChunk(fsstWriteStore(memPageStore), path, values); + + // Read the whole column once, from the top. + assertThat(readBinaries(memPageStore, path, schema)).isEqualTo(values); + + // Read it again from the same store, skipping every other value this time. + ColumnReader reader = columnReader(memPageStore, path, schema); + for (int i = 0; i < values.size(); i++) { + if (i % 2 == 0) { + reader.skip(); + } else { + assertThat(reader.getBinary()).as("at " + i).isEqualTo(values.get(i)); + } + reader.consume(); + } + } + + @Test + public void bothOffsetEncodingsRoundTripTheSameValues() { + MessageType schema = requiredBinaryColumn(); + ColumnDescriptor path = REQUIRED_BINARY; + List values = binaries("one", "two", "three", "four", "five", "six", "seven"); + + for (OffsetEncoding offsetEncoding : OffsetEncoding.values()) { + MemPageStore memPageStore = new MemPageStore(values.size()); + writeChunk(fsstWriteStore(memPageStore, offsetEncoding), path, values); + + assertThat(readBinaries(memPageStore, path, schema)) + .as("offsets " + offsetEncoding) + .isEqualTo(values); + } + } + + /** + * Two independent chunks, each with its own {@link MemPageStore}, standing in for two row groups: + * {@code MemPageWriter.writeSymbolTablePage} rejects a second table on the same block, so a + * literal {@code resetDictionary()} mid-chunk cannot be driven through this substrate — but two + * chunks with disjoint vocabularies prove the same thing a row-group boundary needs, and more + * visibly: a table carried over into the wrong chunk decodes garbage, not just a bigger page. + */ + @Test + public void twoRowGroupsEachTrainTheirOwnTable() throws IOException { + MessageType schema = requiredBinaryColumn(); + ColumnDescriptor path = REQUIRED_BINARY; + List firstRowGroup = repeatedBinaries("alpha", "alphabet", "alpine", "alpaca"); + List secondRowGroup = repeatedBinaries("zeta", "zenith", "zephyr", "zodiac"); + + MemPageStore first = new MemPageStore(firstRowGroup.size()); + writeChunk(fsstWriteStore(first), path, firstRowGroup); + MemPageStore second = new MemPageStore(secondRowGroup.size()); + writeChunk(fsstWriteStore(second), path, secondRowGroup); + + assertThat(readBinaries(first, path, schema)).isEqualTo(firstRowGroup); + assertThat(readBinaries(second, path, schema)).isEqualTo(secondRowGroup); + + byte[] firstTable = + pageWriterFor(first, path).getSymbolTablePage().getBytes().toByteArray(); + byte[] secondTable = + pageWriterFor(second, path).getSymbolTablePage().getBytes().toByteArray(); + assertThat(secondTable).as("each row group trains its own table").isNotEqualTo(firstTable); + } + + /** + * The negative control the plan asks for: strip the symbol table out of the page reader the way a + * corrupt or truncated read might, and confirm the column reader refuses to guess rather than + * silently decoding as if the encoding were something else. + * + *

{@link org.apache.parquet.column.impl.ColumnReaderImpl}'s constructor calls {@code consume()} + * to prime the first value, so the failure happens at {@code getColumnReader(path)} itself, not on + * some later value read. + */ + @Test + public void aReaderCannotSilentlyProceedWhenTheTableIsMissing() { + MessageType schema = requiredBinaryColumn(); + ColumnDescriptor path = REQUIRED_BINARY; + List values = repeatedBinaries("alpha", "alphabet", "alpine", "alpaca"); + MemPageStore memPageStore = new MemPageStore(values.size()); + writeChunk(fsstWriteStore(memPageStore), path, values); + + MemPageWriter pageWriter = pageWriterFor(memPageStore, path); + assertThat(pageWriter.getSymbolTablePage()) + .as("the column actually used the encoding") + .isNotNull(); + List pages = pageWriter.getPages(); + + PageReadStore strippedStore = new PageReadStore() { + @Override + public PageReader getPageReader(ColumnDescriptor descriptor) { + Iterator iterator = new ArrayList<>(pages).iterator(); + return new MemPageReader( + pageWriter.getTotalValueCount(), iterator, pageWriter.getDictionaryPage(), null); + } + + @Override + public long getRowCount() { + return memPageStore.getRowCount(); + } + }; + + assertThatThrownBy(() -> new ColumnReadStoreImpl( + strippedStore, new DummyRecordConverter(schema).getRootConverter(), schema, null) + .getColumnReader(path)) + .isInstanceOf(ParquetDecodingException.class); + } +} From e41501ea41b2acc5bd12db98942676ccb319b257 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 22:02:00 +0000 Subject: [PATCH 09/10] Check the offset section in the encode direction against the fixtures The existing interop test only proves this implementation can read a packed offset section the C++ writer produced. It does not prove a packed section this implementation writes comes out byte-identical, since delta-binary-packing has framing choices a byte-compatible decoder does not have to make the same way. Decode each fixture page's packed offsets and re-encode them through the same DeltaBinaryPackingValuesWriterForInteger path SymbolTablePayloadWriter uses, then compare against the fixture's own offset bytes. All 26 packed-offset pages across the five interop cases re-encode byte-for-byte. --- .../symboltable/SymbolTableInteropTest.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java index 9b131ad78a..162ef9a8fd 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/SymbolTableInteropTest.java @@ -35,6 +35,10 @@ import java.util.List; import java.util.Map; import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.HeapByteBufferAllocator; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; import org.apache.parquet.io.api.Binary; import org.junit.jupiter.api.Test; @@ -163,6 +167,75 @@ public void skippingLandsOnTheSameValueAsReadingWould() throws IOException { } } + /** + * The offset section, in the encode direction: {@link #bothOffsetSectionEncodingsAreCovered} only + * proves this implementation can read a packed section the C++ writer produced. It does not prove + * a packed section this implementation writes would be the bytes another reader expects, since a + * delta-binary-packed stream has framing choices (miniblock count, bit widths) a byte-compatible + * decoder does not have to make the same way. Decoding each fixture's packed offsets and + * re-encoding them through the same path {@link SymbolTablePayloadWriter} uses settles that: if the + * two differ, our writer picked a different framing for the same values, not the same one. + */ + @Test + public void packedOffsetsReEncodeToTheFixturesOwnBytes() throws IOException { + boolean sawAPackedSection = false; + for (String name : CASES) { + Chunk chunk = readChunk(name); + for (int page = 0; page < chunk.pages.size(); page++) { + byte[] body = chunk.pages.get(page); + String label = name + "." + page; + int valueCount = readIntLittleEndian(body, 1); + int offsetSectionSize = readIntLittleEndian(body, 5); + if (body[0] != OffsetEncoding.DELTA_BINARY_PACKED.value() || offsetSectionSize == 0) { + continue; + } + sawAPackedSection = true; + + byte[] rawOffsets = new byte[offsetSectionSize]; + System.arraycopy(body, SymbolTablePayload.HEADER_SIZE, rawOffsets, 0, offsetSectionSize); + + int[] endOffsets = decodeDeltaPackedOffsets(rawOffsets, valueCount); + byte[] reEncoded = encodeDeltaPackedOffsets(endOffsets); + + assertThat(reEncoded).as(label + ": re-encoded offset bytes").isEqualTo(rawOffsets); + } + } + assertThat(sawAPackedSection) + .as("the fixtures actually exercise the packed offset path") + .isTrue(); + } + + private static int[] decodeDeltaPackedOffsets(byte[] rawOffsets, int valueCount) throws IOException { + DeltaBinaryPackingValuesReader reader = new DeltaBinaryPackingValuesReader(); + reader.initFromPage(valueCount, stream(rawOffsets)); + int[] endOffsets = new int[valueCount]; + for (int i = 0; i < valueCount; i++) { + endOffsets[i] = reader.readInteger(); + } + return endOffsets; + } + + private static byte[] encodeDeltaPackedOffsets(int[] endOffsets) throws IOException { + try (DeltaBinaryPackingValuesWriterForInteger writer = new DeltaBinaryPackingValuesWriterForInteger( + DeltaBinaryPackingValuesWriter.DEFAULT_NUM_BLOCK_VALUES, + DeltaBinaryPackingValuesWriter.DEFAULT_NUM_MINIBLOCKS, + 64 * 1024, + 64 * 1024, + HeapByteBufferAllocator.getInstance())) { + for (int endOffset : endOffsets) { + writer.writeInteger(endOffset); + } + return writer.getBytes().toByteArray(); + } + } + + private static int readIntLittleEndian(byte[] bytes, int pos) { + return (bytes[pos] & 0xFF) + | ((bytes[pos + 1] & 0xFF) << 8) + | ((bytes[pos + 2] & 0xFF) << 16) + | ((bytes[pos + 3] & 0xFF) << 24); + } + private static List decodePage(SymbolTableValuesReader reader, byte[] page, int count) throws IOException { reader.initFromPage(count, stream(page)); List values = new ArrayList<>(count); From 50dde2e7f7d3ae9d03465505237f08117d0edcff Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 8 Sep 2026 22:18:53 +0000 Subject: [PATCH 10/10] Benchmark FSST against the encodings a text column falls back to Compares FSST to DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY and RLE_DICTIONARY, with and without a zstd second pass, on a synthetic corpus with the shared structure (repeated URL templates and vocabulary) that lets each encoding actually exploit redundancy. Reports ratio, encode and decode separately. --- parquet-column/pom.xml | 6 + .../BenchmarkSymbolTableEncoding.java | 391 ++++++++++++++++++ 2 files changed, 397 insertions(+) create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/benchmark/BenchmarkSymbolTableEncoding.java diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index 8ca38ba4da..39e523e3b4 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -112,6 +112,12 @@ ${junit.version} test + + com.github.luben + zstd-jni + ${zstd-jni.version} + test + diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/benchmark/BenchmarkSymbolTableEncoding.java b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/benchmark/BenchmarkSymbolTableEncoding.java new file mode 100644 index 0000000000..1d3fc6bce8 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/symboltable/benchmark/BenchmarkSymbolTableEncoding.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.symboltable.benchmark; + +import com.carrotsearch.junitbenchmarks.BenchmarkOptions; +import com.carrotsearch.junitbenchmarks.BenchmarkRule; +import com.carrotsearch.junitbenchmarks.annotation.AxisRange; +import com.carrotsearch.junitbenchmarks.annotation.BenchmarkMethodChart; +import com.github.luben.zstd.Zstd; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.DirectByteBufferAllocator; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.values.ValuesReader; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.deltalengthbytearray.DeltaLengthByteArrayValuesReader; +import org.apache.parquet.column.values.deltalengthbytearray.DeltaLengthByteArrayValuesWriter; +import org.apache.parquet.column.values.deltastrings.DeltaByteArrayReader; +import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; +import org.apache.parquet.column.values.dictionary.DictionaryValuesReader; +import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter; +import org.apache.parquet.column.values.dictionary.PlainValuesDictionary.PlainBinaryDictionary; +import org.apache.parquet.column.values.symboltable.SymbolTable; +import org.apache.parquet.column.values.symboltable.SymbolTablePayload.OffsetEncoding; +import org.apache.parquet.column.values.symboltable.SymbolTableType; +import org.apache.parquet.column.values.symboltable.SymbolTableValuesReader; +import org.apache.parquet.column.values.symboltable.SymbolTableValuesWriter; +import org.apache.parquet.column.values.symboltable.SymbolTables; +import org.apache.parquet.io.api.Binary; +import org.junit.Rule; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.migrationsupport.rules.EnableRuleMigrationSupport; + +/** + * Compares FSST against the encodings a text column falls back to today, with and without a zstd + * second pass, since the layered configuration is the one a real column ever ships with. + * + *

Every {@code *WithoutZstd}/{@code *WithZstd} pair isolates encode from decode: the bytes a + * decode benchmark reads are built once outside the timed rounds, and the side channel each + * encoding needs (a symbol table or a dictionary page) is decoded once as well, matching how a real + * reader amortizes it across every page of a chunk. + * + *

Ratios are not a JUnitBenchmarks metric, so they are printed once, up front, rather than folded + * into a timed round. + */ +@EnableRuleMigrationSupport +@AxisRange(min = 0, max = 1) +@BenchmarkMethodChart(filePrefix = "benchmark-symboltable-encoding") +public class BenchmarkSymbolTableEncoding { + + @Rule + public org.junit.rules.TestRule benchmarkRun = new BenchmarkRule(); + + private static final int INITIAL_SLAB_SIZE = 64 * 1024; + private static final int PAGE_SIZE = 4 * 1024 * 1024; + + private static final Binary[] VALUES = buildCorpus(); + private static final long RAW_BYTES = totalRawBytes(VALUES); + + // FSST + private static final byte[] FSST_TABLE_BYTES; + private static final byte[] FSST_PAYLOAD_BYTES; + private static final byte[] FSST_PAYLOAD_ZSTD; + private static final SymbolTable FSST_TABLE; + + // DELTA_LENGTH_BYTE_ARRAY + private static final byte[] DLBA_PAYLOAD_BYTES; + private static final byte[] DLBA_PAYLOAD_ZSTD; + + // DELTA_BYTE_ARRAY + private static final byte[] DBA_PAYLOAD_BYTES; + private static final byte[] DBA_PAYLOAD_ZSTD; + + // RLE_DICTIONARY + private static final byte[] DICT_PAGE_BYTES; + private static final int DICT_SIZE; + private static final byte[] DICT_INDEX_BYTES; + private static final byte[] DICT_INDEX_ZSTD; + private static final DictionaryPage DICTIONARY_PAGE; + + static { + try { + SymbolTableValuesWriter fsstWriter = new SymbolTableValuesWriter( + SymbolTableType.FSST_8, + OffsetEncoding.DELTA_BINARY_PACKED, + INITIAL_SLAB_SIZE, + PAGE_SIZE, + new DirectByteBufferAllocator()); + writeAll(fsstWriter, VALUES); + FSST_PAYLOAD_BYTES = fsstWriter.getBytes().toByteArray(); + FSST_TABLE_BYTES = fsstWriter.toSymbolTablePageAndClose().getBytes().toByteArray(); + FSST_TABLE = SymbolTables.deserialize(SymbolTableType.FSST_8, FSST_TABLE_BYTES, 0, FSST_TABLE_BYTES.length); + FSST_PAYLOAD_ZSTD = Zstd.compress(FSST_PAYLOAD_BYTES); + + DeltaLengthByteArrayValuesWriter dlbaWriter = + new DeltaLengthByteArrayValuesWriter(INITIAL_SLAB_SIZE, PAGE_SIZE, new DirectByteBufferAllocator()); + writeAll(dlbaWriter, VALUES); + DLBA_PAYLOAD_BYTES = dlbaWriter.getBytes().toByteArray(); + DLBA_PAYLOAD_ZSTD = Zstd.compress(DLBA_PAYLOAD_BYTES); + + DeltaByteArrayWriter dbaWriter = + new DeltaByteArrayWriter(INITIAL_SLAB_SIZE, PAGE_SIZE, new DirectByteBufferAllocator()); + writeAll(dbaWriter, VALUES); + DBA_PAYLOAD_BYTES = dbaWriter.getBytes().toByteArray(); + DBA_PAYLOAD_ZSTD = Zstd.compress(DBA_PAYLOAD_BYTES); + + PlainBinaryDictionaryValuesWriter dictWriter = new PlainBinaryDictionaryValuesWriter( + Integer.MAX_VALUE, Encoding.RLE_DICTIONARY, Encoding.PLAIN, new DirectByteBufferAllocator()); + writeAll(dictWriter, VALUES); + DICT_INDEX_BYTES = dictWriter.getBytes().toByteArray(); + DictionaryPage dictPage = dictWriter.toDictPageAndClose(); + DICT_PAGE_BYTES = dictPage.getBytes().toByteArray(); + DICT_SIZE = dictPage.getDictionarySize(); + DICTIONARY_PAGE = new DictionaryPage(BytesInput.from(DICT_PAGE_BYTES), DICT_SIZE, Encoding.PLAIN); + DICT_INDEX_ZSTD = Zstd.compress(DICT_INDEX_BYTES); + + report(); + } catch (IOException e) { + throw new ExceptionInInitializerError(e); + } + } + + /** + * Text with the shared structure real string columns have: repeated URL templates and repeated + * vocabulary, the case FSST and dictionary encoding both exist for. Random alphanumeric data, used + * elsewhere in this module, would flatter nothing and make the comparison meaningless. + */ + private static Binary[] buildCorpus() { + String[] words = { + "alpha", + "bravo", + "charlie", + "delta", + "echo", + "foxtrot", + "golf", + "hotel", + "india", + "juliet", + "kilo", + "lima", + "mike", + "november", + "oscar", + "papa" + }; + Random random = new Random(42); + List values = new ArrayList<>(); + for (int i = 0; i < 15000; i++) { + values.add("https://www.example.com/products/widget-" + i + "/reviews?page=" + (i % 37) + "&sort=" + + (i % 5 == 0 ? "asc" : "desc")); + } + for (int i = 0; i < 15000; i++) { + StringBuilder builder = new StringBuilder(); + int wordCount = 6 + random.nextInt(6); + for (int j = 0; j < wordCount; j++) { + if (j > 0) { + builder.append(' '); + } + builder.append(words[random.nextInt(words.length)]); + } + builder.append(" order-id=").append(i); + values.add(builder.toString()); + } + Collections.shuffle(values, random); + Binary[] binaries = new Binary[values.size()]; + for (int i = 0; i < binaries.length; i++) { + binaries[i] = Binary.fromString(values.get(i)); + } + return binaries; + } + + private static long totalRawBytes(Binary[] values) { + long total = 0; + for (Binary value : values) { + total += value.length(); + } + return total; + } + + private static void writeAll(ValuesWriter writer, Binary[] values) { + for (Binary value : values) { + writer.writeBytes(value); + } + } + + private static void readAll(ValuesReader reader, ByteBufferInputStream stream, int count) throws IOException { + reader.initFromPage(count, stream); + for (int i = 0; i < count; i++) { + reader.readBytes(); + } + } + + private static void report() { + System.out.printf( + "%-20s %12s %14s %14s %8s %8s%n", "encoding", "raw", "encoded", "encoded+zstd", "ratio", "ratio+zstd"); + reportOne( + "FSST", + FSST_TABLE_BYTES.length + FSST_PAYLOAD_BYTES.length, + FSST_TABLE_BYTES.length + FSST_PAYLOAD_ZSTD.length); + reportOne("DELTA_LENGTH_BYTE_ARRAY", DLBA_PAYLOAD_BYTES.length, DLBA_PAYLOAD_ZSTD.length); + reportOne("DELTA_BYTE_ARRAY", DBA_PAYLOAD_BYTES.length, DBA_PAYLOAD_ZSTD.length); + reportOne( + "RLE_DICTIONARY", + DICT_PAGE_BYTES.length + DICT_INDEX_BYTES.length, + DICT_PAGE_BYTES.length + DICT_INDEX_ZSTD.length); + } + + private static void reportOne(String name, long encoded, long encodedZstd) { + System.out.printf( + "%-20s %12d %14d %14d %8.3f %8.3f%n", + name, RAW_BYTES, encoded, encodedZstd, (double) encoded / RAW_BYTES, (double) encodedZstd / RAW_BYTES); + } + + // FSST + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void fsstEncodeWithoutZstd() { + SymbolTableValuesWriter writer = new SymbolTableValuesWriter( + SymbolTableType.FSST_8, + OffsetEncoding.DELTA_BINARY_PACKED, + INITIAL_SLAB_SIZE, + PAGE_SIZE, + new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + writer.getBytes(); + writer.toSymbolTablePageAndClose(); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void fsstEncodeWithZstd() throws IOException { + SymbolTableValuesWriter writer = new SymbolTableValuesWriter( + SymbolTableType.FSST_8, + OffsetEncoding.DELTA_BINARY_PACKED, + INITIAL_SLAB_SIZE, + PAGE_SIZE, + new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + byte[] payload = writer.getBytes().toByteArray(); + byte[] table = writer.toSymbolTablePageAndClose().getBytes().toByteArray(); + Zstd.compress(payload); + Zstd.compress(table); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void fsstDecodeWithoutZstd() throws IOException { + SymbolTableValuesReader reader = new SymbolTableValuesReader(() -> FSST_TABLE); + readAll(reader, BytesInput.from(FSST_PAYLOAD_BYTES).toInputStream(), VALUES.length); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void fsstDecodeWithZstd() throws IOException { + byte[] payload = Zstd.decompress(FSST_PAYLOAD_ZSTD, FSST_PAYLOAD_BYTES.length); + SymbolTableValuesReader reader = new SymbolTableValuesReader(() -> FSST_TABLE); + readAll(reader, BytesInput.from(payload).toInputStream(), VALUES.length); + } + + // DELTA_LENGTH_BYTE_ARRAY + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaLengthByteArrayEncodeWithoutZstd() { + DeltaLengthByteArrayValuesWriter writer = + new DeltaLengthByteArrayValuesWriter(INITIAL_SLAB_SIZE, PAGE_SIZE, new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + writer.getBytes(); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaLengthByteArrayEncodeWithZstd() throws IOException { + DeltaLengthByteArrayValuesWriter writer = + new DeltaLengthByteArrayValuesWriter(INITIAL_SLAB_SIZE, PAGE_SIZE, new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + Zstd.compress(writer.getBytes().toByteArray()); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaLengthByteArrayDecodeWithoutZstd() throws IOException { + readAll( + new DeltaLengthByteArrayValuesReader(), + BytesInput.from(DLBA_PAYLOAD_BYTES).toInputStream(), + VALUES.length); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaLengthByteArrayDecodeWithZstd() throws IOException { + byte[] payload = Zstd.decompress(DLBA_PAYLOAD_ZSTD, DLBA_PAYLOAD_BYTES.length); + readAll(new DeltaLengthByteArrayValuesReader(), BytesInput.from(payload).toInputStream(), VALUES.length); + } + + // DELTA_BYTE_ARRAY + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaByteArrayEncodeWithoutZstd() { + DeltaByteArrayWriter writer = + new DeltaByteArrayWriter(INITIAL_SLAB_SIZE, PAGE_SIZE, new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + writer.getBytes(); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaByteArrayEncodeWithZstd() throws IOException { + DeltaByteArrayWriter writer = + new DeltaByteArrayWriter(INITIAL_SLAB_SIZE, PAGE_SIZE, new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + Zstd.compress(writer.getBytes().toByteArray()); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaByteArrayDecodeWithoutZstd() throws IOException { + readAll(new DeltaByteArrayReader(), BytesInput.from(DBA_PAYLOAD_BYTES).toInputStream(), VALUES.length); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void deltaByteArrayDecodeWithZstd() throws IOException { + byte[] payload = Zstd.decompress(DBA_PAYLOAD_ZSTD, DBA_PAYLOAD_BYTES.length); + readAll(new DeltaByteArrayReader(), BytesInput.from(payload).toInputStream(), VALUES.length); + } + + // RLE_DICTIONARY + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void rleDictionaryEncodeWithoutZstd() { + PlainBinaryDictionaryValuesWriter writer = new PlainBinaryDictionaryValuesWriter( + Integer.MAX_VALUE, Encoding.RLE_DICTIONARY, Encoding.PLAIN, new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + writer.getBytes(); + writer.toDictPageAndClose(); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void rleDictionaryEncodeWithZstd() throws IOException { + PlainBinaryDictionaryValuesWriter writer = new PlainBinaryDictionaryValuesWriter( + Integer.MAX_VALUE, Encoding.RLE_DICTIONARY, Encoding.PLAIN, new DirectByteBufferAllocator()); + writeAll(writer, VALUES); + byte[] index = writer.getBytes().toByteArray(); + byte[] dict = writer.toDictPageAndClose().getBytes().toByteArray(); + Zstd.compress(index); + Zstd.compress(dict); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void rleDictionaryDecodeWithoutZstd() throws IOException { + DictionaryValuesReader reader = new DictionaryValuesReader(new PlainBinaryDictionary(DICTIONARY_PAGE)); + readAll(reader, BytesInput.from(DICT_INDEX_BYTES).toInputStream(), VALUES.length); + } + + @BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 3) + @Test + public void rleDictionaryDecodeWithZstd() throws IOException { + byte[] index = Zstd.decompress(DICT_INDEX_ZSTD, DICT_INDEX_BYTES.length); + DictionaryValuesReader reader = new DictionaryValuesReader(new PlainBinaryDictionary(DICTIONARY_PAGE)); + readAll(reader, BytesInput.from(index).toInputStream(), VALUES.length); + } +}