From 0e28e8974d60919e311010e71276fe4abfe8b1a3 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 1 Sep 2026 11:16:10 +0200 Subject: [PATCH] HIVE-29866: Optimize LLAP IO cache encoding path for text tables with MultiDelimitSerDe --- .../encoded/VectorDeserializeOrcWriter.java | 53 ++++- .../TestVectorDeserializeOrcWriter.java | 73 +++++++ .../hive/serde2/lazy/LazySerDeParameters.java | 34 +++- .../lazy/fast/LazySimpleDeserializeRead.java | 107 ++++++++-- .../fast/TestLazySimpleDeserializeRead.java | 192 +++++++++++++++++- 5 files changed, 435 insertions(+), 24 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java index a55b64cdcbf1..7b4bde7f5bee 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.nio.charset.StandardCharsets; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -55,6 +56,7 @@ import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.MultiDelimitSerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; @@ -103,9 +105,16 @@ public static EncodingWriter create(InputFormat sourceIf, Deserializer ser List sourceIncludes, boolean[] cacheIncludes, int allocSize, ExecutorService encodeExecutor) throws IOException { // Vector SerDe can be disabled both on client and server side. + // MultiDelimitSerDe is accepted alongside LazySimpleSerDe: it's the same + // text-row shape, only its top-level FIELD_DELIM can span multiple bytes. + // The reader (LazySimpleDeserializeRead) picks the multi-byte scanner + // when LazySerDeParameters.setFieldDelimMulti has been called below — + // otherwise the specialized single-byte hot loop runs unchanged. + final boolean isLazySimple = serDe instanceof LazySimpleSerDe; + final boolean isMultiDelim = serDe instanceof MultiDelimitSerDe; if (!HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED) || !HiveConf.getBoolVar(jobConf, ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED) - || !(sourceIf instanceof TextInputFormat) || !(serDe instanceof LazySimpleSerDe)) { + || !(sourceIf instanceof TextInputFormat) || !(isLazySimple || isMultiDelim)) { return new DeserializerOrcWriter(serDe, sourceOi, allocSize); } Path path = splitPath.getFileSystem(jobConf).makeQualified(splitPath); @@ -121,6 +130,16 @@ public static EncodingWriter create(InputFormat sourceIf, Deserializer ser + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST); return new DeserializerOrcWriter(serDe, sourceOi, allocSize); } + // MultiDelimitSerDe's own row parser (LazyStruct.parseMultiDelimit) does + // NOT honour escape.delim for the top-level delimiter. Combining a + // multi-byte field delim with an escape char in the fast path would let + // it silently diverge from the slow path, so bail to DeserializerOrcWriter + // whenever both are configured on a MultiDelimit table. + if (isMultiDelim && tblProps.getProperty(serdeConstants.ESCAPE_CHAR) != null) { + LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter: MultiDelimitSerDe with " + + serdeConstants.ESCAPE_CHAR + " is not supported by the vectorized fast path"); + return new DeserializerOrcWriter(serDe, sourceOi, allocSize); + } for (StructField sf : sourceOi.getAllStructFieldRefs()) { Category c = sf.getFieldObjectInspector().getCategory(); if (c != Category.PRIMITIVE) { @@ -130,11 +149,13 @@ public static EncodingWriter create(InputFormat sourceIf, Deserializer ser } LlapIoImpl.LOG.info("Creating VertorDeserializeOrcWriter for " + path); return new VectorDeserializeOrcWriter( - jobConf, tblProps, sourceOi, sourceIncludes, cacheIncludes, allocSize, encodeExecutor); + jobConf, tblProps, sourceOi, sourceIncludes, cacheIncludes, allocSize, encodeExecutor, + isMultiDelim); } private VectorDeserializeOrcWriter(Configuration conf, Properties tblProps, StructObjectInspector sourceOi, - List sourceIncludes, boolean[] cacheIncludes, int allocSize, ExecutorService encodeExecutor) + List sourceIncludes, boolean[] cacheIncludes, int allocSize, ExecutorService encodeExecutor, + boolean isMultiDelim) throws IOException { super(sourceOi, allocSize); // See also: the usage of VectorDeserializeType, for binary. For now, we only want text. @@ -143,7 +164,8 @@ private VectorDeserializeOrcWriter(Configuration conf, Properties tblProps, Stru this.cacheIncludes = cacheIncludes; this.sourceBatch = vrbCtx.createVectorizedRowBatch(); deserializeRead = new LazySimpleDeserializeRead(vrbCtx.getRowColumnTypeInfos(), - vrbCtx.getRowdataTypePhysicalVariations(),/* useExternalBuffer */ true, createSerdeParams(conf, tblProps)); + vrbCtx.getRowdataTypePhysicalVariations(),/* useExternalBuffer */ true, + createSerdeParams(conf, tblProps, isMultiDelim)); vectorDeserializeRow = new VectorDeserializeRow(deserializeRead); int colCount = vrbCtx.getRowColumnTypeInfos().length; boolean[] includes = null; @@ -212,8 +234,10 @@ private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, fina .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); final String serde = tblProps.getProperty(serdeConstants.SERIALIZATION_LIB); final String inputFormat = tblProps.getProperty(hive_metastoreConstants.FILE_INPUT_FORMAT); - final boolean isTextFormat = inputFormat != null && inputFormat.equals(TextInputFormat.class.getName()) && - serde != null && serde.equals(LazySimpleSerDe.class.getName()); + final boolean isTextFormat = inputFormat != null && inputFormat.equals(TextInputFormat.class.getName()) + && serde != null + && (serde.equals(LazySimpleSerDe.class.getName()) + || serde.equals(MultiDelimitSerDe.class.getName())); List dataTypePhysicalVariations = new ArrayList<>(); if (isTextFormat) { StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(oi); @@ -245,9 +269,22 @@ private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, fina } private static LazySerDeParameters createSerdeParams( - Configuration conf, Properties tblProps) throws IOException { + Configuration conf, Properties tblProps, boolean isMultiDelim) throws IOException { try { - return new LazySerDeParameters(conf, tblProps, LazySimpleSerDe.class.getName()); + LazySerDeParameters params = + new LazySerDeParameters(conf, tblProps, LazySimpleSerDe.class.getName()); + // For MultiDelimitSerDe tables, if the field delimiter is > 1 byte, opt + // the reader into its multi-byte scan branch. LazySerDeParameters + // silently ignores single-byte values here, so misconfigured MultiDelim + // tables (e.g. FIELD_DELIM=",") still take the specialized single-byte + // fast path via separators[0]. + if (isMultiDelim) { + String rawDelim = tblProps.getProperty(serdeConstants.FIELD_DELIM); + if (rawDelim != null && rawDelim.length() > 1) { + params.setFieldDelimMulti(rawDelim.getBytes(StandardCharsets.UTF_8)); + } + } + return params; } catch (SerDeException e) { throw new IOException(e); } diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java index beca9fa7ed87..e330480ce6ba 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java @@ -19,10 +19,14 @@ package org.apache.hadoop.hive.llap.io.encoded; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Properties; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch; import org.apache.hadoop.hive.llap.io.decode.EncodedDataConsumer; @@ -30,13 +34,17 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.io.orc.WriterImpl; import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; import org.apache.hive.common.util.FixedSizedObjectPool; import org.apache.orc.impl.SchemaEvolution; import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; @@ -145,6 +153,71 @@ private static VectorDeserializeOrcWriter createOrcWriter( return orcWriter; } + // --- createSerdeParams: MultiDelimit routing → fieldDelimMulti wiring ----- + + private static LazySerDeParameters invokeCreateSerdeParams(Properties tblProps, + boolean isMultiDelim) throws Exception { + Method m = VectorDeserializeOrcWriter.class.getDeclaredMethod( + "createSerdeParams", Configuration.class, Properties.class, boolean.class); + m.setAccessible(true); + return (LazySerDeParameters) m.invoke(null, new Configuration(false), tblProps, isMultiDelim); + } + + /** + * When routing sees a MultiDelimitSerDe with a multi-byte field.delim, the + * LazySerDeParameters we hand to LazySimpleDeserializeRead must carry the + * raw delimiter bytes — that's the signal the reader uses to switch to its + * multi-byte scan branch (and hence keep the row on the vectorized LLAP + * fast path instead of falling back to DeserializerOrcWriter). + */ + @Test + public void testCreateSerdeParamsMultiDelimSetsFieldDelimMulti() throws Exception { + Properties tblProps = new Properties(); + tblProps.setProperty(serdeConstants.FIELD_DELIM, "~|"); + tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|"); + tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b"); + tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string"); + + LazySerDeParameters params = invokeCreateSerdeParams(tblProps, /*isMultiDelim*/ true); + assertArrayEquals("~|".getBytes(StandardCharsets.UTF_8), params.getFieldDelimMulti()); + } + + /** + * A LazySimpleSerDe table (isMultiDelim=false) must NEVER opt into the + * multi-byte scan branch, even if someone shoved a >1-byte value into + * field.delim — the slow path silently truncates to a single byte, and + * diverging here would produce different results between fast and slow + * paths for the same table. + */ + @Test + public void testCreateSerdeParamsLazySimpleNeverSetsFieldDelimMulti() throws Exception { + Properties tblProps = new Properties(); + tblProps.setProperty(serdeConstants.FIELD_DELIM, "~|"); + tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|"); + tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b"); + tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string"); + + LazySerDeParameters params = invokeCreateSerdeParams(tblProps, /*isMultiDelim*/ false); + assertNull(params.getFieldDelimMulti()); + } + + /** + * A MultiDelimit table whose field.delim is only one byte still takes the + * specialized single-byte fast path — setFieldDelimMulti drops sub-2-byte + * values, so separators[0] remains the source of truth. + */ + @Test + public void testCreateSerdeParamsMultiDelimWithSingleByteFallsThrough() throws Exception { + Properties tblProps = new Properties(); + tblProps.setProperty(serdeConstants.FIELD_DELIM, "|"); + tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "|"); + tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b"); + tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string"); + + LazySerDeParameters params = invokeCreateSerdeParams(tblProps, /*isMultiDelim*/ true); + assertNull(params.getFieldDelimMulti()); + } + private static EncodedDataConsumer createBlankEncodedDataConsumer() { return new EncodedDataConsumer(null, 1, null, null) { @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java index 9da6bf5e3c35..1db95d3b086f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java @@ -60,11 +60,18 @@ public class LazySerDeParameters implements LazyObjectInspectorParameters { private Properties tableProperties; private String serdeName; - // The list of bytes used for the separators in the column (a nested struct + // The list of bytes used for the separators in the column (a nested struct // such as Array> will use multiple separators). // The list of separators + escapeChar are the bytes required to be escaped. private byte[] separators; + // Populated only when the top-level field delimiter is longer than one byte + // (i.e. by MultiDelimitSerDe). Never populated for LazySimpleSerDe callers, + // so existing single-byte behaviour is preserved. The single-byte `separators[0]` + // slot is still set to the first byte of the multi-byte delim so that + // needsEscape[] and any legacy consumers of getSeparators() keep working. + private byte[] fieldDelimMulti; + private Text nullSequence; private TypeInfo rowTypeInfo; @@ -196,6 +203,31 @@ public byte[] getSeparators() { return separators; } + /** + * @return the raw (potentially multi-byte) top-level field delimiter, or + * {@code null} when the effective delimiter is a single byte (the + * value in {@code separators[0]} is authoritative in that case). + * + * Set by callers that know they're wiring up a multi-byte delimiter + * (e.g. LLAP's {@code VectorDeserializeOrcWriter} when the SerDe is + * {@code MultiDelimitSerDe}). Left {@code null} for plain LazySimpleSerDe so + * its single-byte fast path is preserved verbatim. + */ + public byte[] getFieldDelimMulti() { + return fieldDelimMulti; + } + + /** + * Opt-in switch for the multi-byte top-level delimiter. Only meaningful when + * {@code delim.length > 1}; shorter values are ignored (the {@code separators[0]} + * byte remains the source of truth). Never set this for a + * LazySimpleSerDe-backed table — its slow path still truncates to a single + * byte, so opting the fast path into multi-byte would silently diverge. + */ + public void setFieldDelimMulti(byte[] delim) { + this.fieldDelimMulti = (delim != null && delim.length > 1) ? delim : null; + } + public Text getNullSequence() { return nullSequence; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index 747d389db56a..3f496a356a48 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -198,6 +198,18 @@ public UnionComplexTypeHelper(Field complexField, Field[] fields) { private int[] startPositions; private final byte[] separators; + // Multi-byte top-level field delimiter (e.g. MultiDelimitSerDe's `~|`). + // null in the common single-byte case, in which case `separators[0]` is used + // and the row-scan hot loop keeps its per-byte specialization. Non-null only + // when the caller (LLAP's VectorDeserializeOrcWriter for MultiDelimitSerDe) + // explicitly opted in via LazySerDeParameters.setFieldDelimMulti. + private final byte[] fieldDelimMulti; + // Byte length "charged" to a top-level separator: 1 for the single-byte fast + // path, delim.length for multi-byte. Baked into (a) the Arrays.fill sentinel + // for missing/trailing fields and (b) the "length = next.start - this.start + // - sepLen" arithmetic in readField and getDetailedReadPositionString, so + // both paths share the same downstream code. + private final int topLevelSeparatorLen; private final boolean isEscaped; private final byte escapeChar; private final int[] escapeCounts; @@ -336,11 +348,23 @@ public LazySimpleDeserializeRead(TypeInfo[] typeInfos, startPositions = new int[count + 1]; this.separators = lazyParams.getSeparators(); + this.fieldDelimMulti = lazyParams.getFieldDelimMulti(); + this.topLevelSeparatorLen = (fieldDelimMulti == null) ? 1 : fieldDelimMulti.length; isEscaped = lazyParams.isEscaped(); if (isEscaped) { escapeChar = lazyParams.getEscapeChar(); escapeCounts = new int[count]; + // Multi-byte field delimiters with escape sequences are not implemented + // in this fast path — the escape/separator interaction would diverge + // from MultiDelimitSerDe's own row-based parser, which ignores escapes + // for the top-level delim. Callers (VectorDeserializeOrcWriter) route + // this combination to the DeserializerOrcWriter slow path instead. + if (fieldDelimMulti != null) { + throw new RuntimeException( + "Multi-byte field delimiter combined with escape.delim is not " + + "supported in the vectorized fast path."); + } } else { escapeChar = (byte) 0; escapeCounts = null; @@ -402,7 +426,7 @@ public String getDetailedReadPositionString() { sb.append(" at field start position "); sb.append(startPositions[currentTopLevelFieldIndex]); int currentFieldLength = startPositions[currentTopLevelFieldIndex + 1] - - startPositions[currentTopLevelFieldIndex] - 1; + startPositions[currentTopLevelFieldIndex] - topLevelSeparatorLen; sb.append(" for field length "); sb.append(currentFieldLength); } @@ -410,6 +434,23 @@ public String getDetailedReadPositionString() { return sb.toString(); } + /** + * Bytes at {@code buf[off..off+dlen)} equal to {@code delim[0..dlen)}? + * + * Caller has already checked {@code buf[off] == delim[0]}, so we start at + * index 1 — this is only ever invoked when the first byte matched, which + * keeps the multi-byte hot loop from paying for a tail compare on every + * mismatching input byte. + */ + private static boolean matchesAt(byte[] buf, int off, byte[] delim, int dlen) { + for (int i = 1; i < dlen; i++) { + if (buf[off + i] != delim[i]) { + return false; + } + } + return true; + } + /** * Parse the byte[] and fill each field. * @@ -432,20 +473,54 @@ private void topLevelParse() { * Optimize the loops by pulling special end cases and global decisions like isEscaped out! */ if (!isEscaped) { - while (fieldByteEnd < end) { - if (bytes[fieldByteEnd] == separator) { + if (fieldDelimMulti == null) { + // Single-byte fast path — hot on every row of a LazySimple table. + while (fieldByteEnd < end) { + if (bytes[fieldByteEnd] == separator) { + startPositions[fieldId++] = fieldByteBegin; + if (fieldId == fieldCount) { + break; + } + fieldByteBegin = ++fieldByteEnd; + } else { + fieldByteEnd++; + } + } + // End serves as final separator. + if (fieldByteEnd == end && fieldId < fieldCount) { startPositions[fieldId++] = fieldByteBegin; - if (fieldId == fieldCount) { - break; + } + } else { + // Multi-byte top-level delimiter (MultiDelimitSerDe path). We keep the + // "test the first byte, then verify the tail" idiom so the mismatching- + // byte case still costs a single load+compare — the tail memcmp only + // fires on a first-byte hit. + final byte[] delim = this.fieldDelimMulti; + final int dlen = delim.length; + final byte first = delim[0]; + final int scanEnd = end - dlen; // last index at which a full delim can start + while (fieldByteEnd <= scanEnd) { + if (bytes[fieldByteEnd] == first && matchesAt(bytes, fieldByteEnd, delim, dlen)) { + startPositions[fieldId++] = fieldByteBegin; + if (fieldId == fieldCount) { + // Malformed row with more delims than expected: leave fieldByteEnd + // where it is (matching single-byte-path semantics) and stop. + break; + } + fieldByteEnd += dlen; + fieldByteBegin = fieldByteEnd; + } else { + fieldByteEnd++; } - fieldByteBegin = ++fieldByteEnd; - } else { - fieldByteEnd++; } - } - // End serves as final separator. - if (fieldByteEnd == end && fieldId < fieldCount) { - startPositions[fieldId++] = fieldByteBegin; + // No trailing delim (single-byte parses "end as final separator" here). + // If we still owe fields, the remainder from fieldByteBegin..end is the + // last field; fast-forward fieldByteEnd so the Arrays.fill sentinel and + // isEndOfInputReached below both see the row as fully consumed. + if (fieldId < fieldCount) { + fieldByteEnd = end; + startPositions[fieldId++] = fieldByteBegin; + } } } else { final byte escapeChar = this.escapeChar; @@ -497,7 +572,11 @@ private void topLevelParse() { // For missing fields, their starting positions will all be the same, // which will make their lengths to be -1 and uncheckedGetField will // return these fields as NULLs. - Arrays.fill(startPositions, fieldId, startPositions.length, fieldByteEnd + 1); + // Charge the actual separator width (1 byte for LazySimple, delim.length + // for the multi-byte MultiDelimit path) so the "length = next - this - + // sepLen" arithmetic used by readField comes out right in both cases. + Arrays.fill(startPositions, fieldId, startPositions.length, + fieldByteEnd + topLevelSeparatorLen); } isEndOfInputReached = (fieldByteEnd == end); @@ -639,7 +718,7 @@ public boolean readField(int fieldIndex) throws IOException { currentTopLevelFieldIndex = fieldIndex; currentFieldStart = startPositions[fieldIndex]; - currentFieldLength = startPositions[fieldIndex + 1] - startPositions[fieldIndex] - 1; + currentFieldLength = startPositions[fieldIndex + 1] - startPositions[fieldIndex] - topLevelSeparatorLen; currentEscapeCount = (isEscaped ? escapeCounts[fieldIndex] : 0); return doReadField(fields[fieldIndex]); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java index 465a616fa936..da783e12d6f3 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java @@ -20,6 +20,7 @@ +import java.nio.charset.StandardCharsets; import java.util.Properties; import org.apache.hadoop.hive.conf.HiveConf; @@ -29,8 +30,11 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.Text; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.junit.Test; /** @@ -86,4 +90,190 @@ public void testEscaping() throws Exception { assertTrue("The escaped result is incorrect", field.compareTo(escaped) == 0); } + + // --- multi-byte field delimiter (MultiDelimitSerDe → LLAP fast path) ------ + + private static LazySerDeParameters multiDelimParams(String delim) throws Exception { + Properties props = new Properties(); + props.setProperty(serdeConstants.FIELD_DELIM, delim); + props.setProperty(serdeConstants.SERIALIZATION_FORMAT, delim); + LazySerDeParameters p = new LazySerDeParameters(new HiveConf(), props, + LazySimpleSerDe.class.getName()); + p.setFieldDelimMulti(delim.getBytes(StandardCharsets.UTF_8)); + return p; + } + + private static byte[] readStringField(LazySimpleDeserializeRead r) throws Exception { + assertTrue("expected non-null field", r.readNextField()); + int len = r.currentBytesLength; + byte[] out = new byte[len]; + System.arraycopy(r.currentBytes, r.currentBytesStart, out, 0, len); + return out; + } + + /** + * Three STRING columns separated by "~|" — the delimiter used by the BofA + * MultiDelimitSerDe workload that motivated this path. + */ + @Test + public void testMultiByteDelimThreeStringColumns() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "alpha~|beta~|gamma".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("alpha".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("beta".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("gamma".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertFalse("row should have exactly 3 fields", r.readNextField()); + } + + /** + * Mixed INT / STRING columns. Verifies that the length arithmetic charges + * the full delim.length (not 1) between fields so numeric parsing sees the + * correct field boundaries. + */ + @Test + public void testMultiByteDelimMixedTypes() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.intTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.longTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "42~|hello~|9876543210".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertTrue(r.readNextField()); + assertEquals(42, r.currentInt); + + assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), readStringField(r)); + + assertTrue(r.readNextField()); + assertEquals(9876543210L, r.currentLong); + } + + /** + * The first byte of the delimiter ("~") appearing standalone inside a field + * must NOT trigger a split — the tail-match on delim[1] rescues it. + */ + @Test + public void testMultiByteDelimFirstByteInsideField() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + // A bare '~' followed by non-'|' must stay inside the first field. + byte[] row = "foo~bar~|baz".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("foo~bar".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("baz".getBytes(StandardCharsets.UTF_8), readStringField(r)); + } + + /** + * The empty-tail case: last field extends to EOL and has zero length. + * startPositions must still tabulate length 0 (not -1), so NULL isn't + * spuriously returned. + */ + @Test + public void testMultiByteDelimEmptyLastField() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "one~|".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("one".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertTrue("empty last field should be non-null (zero-length)", r.readNextField()); + assertEquals(0, r.currentBytesLength); + } + + /** + * Missing trailing fields must land in the "startPositions filled with + * sentinel" branch and come back as NULL — same behavior as single-byte. + */ + @Test + public void testMultiByteDelimMissingTrailingFields() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "only-one".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("only-one".getBytes(StandardCharsets.UTF_8), readStringField(r)); + // Missing fields → NULL. + assertFalse(r.readNextField()); + assertFalse(r.readNextField()); + } + + /** + * A three-byte delimiter exercises the "delim length > 2" arithmetic — the + * -1 constant we replaced with topLevelSeparatorLen is easy to miss for + * dlen != 2. + */ + @Test + public void testMultiByteDelimThreeByteDelimiter() throws Exception { + LazySerDeParameters params = multiDelimParams("|~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "a|~|bb|~|ccc".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("a".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("bb".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("ccc".getBytes(StandardCharsets.UTF_8), readStringField(r)); + } + + /** + * Multi-byte delim combined with escape.delim is not supported in the fast + * path — LazySimpleDeserializeRead must reject the combination at + * construction time so we never silently diverge from + * MultiDelimitSerDe.parseMultiDelimit (which itself ignores escape at the + * top level). + */ + @Test + public void testMultiByteDelimRejectsEscape() throws Exception { + Properties props = new Properties(); + props.setProperty(serdeConstants.FIELD_DELIM, "~|"); + props.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|"); + props.setProperty(serdeConstants.ESCAPE_CHAR, "\\"); + LazySerDeParameters params = new LazySerDeParameters(new HiveConf(), props, + LazySimpleSerDe.class.getName()); + params.setFieldDelimMulti("~|".getBytes(StandardCharsets.UTF_8)); + + TypeInfo[] typeInfos = new TypeInfo[] { TypeInfoFactory.stringTypeInfo }; + try { + new LazySimpleDeserializeRead(typeInfos, null, true, params); + fail("expected RuntimeException for multi-byte delim + escape.delim"); + } catch (RuntimeException expected) { + assertTrue("unexpected message: " + expected.getMessage(), + expected.getMessage().contains("Multi-byte field delimiter")); + } + } }