diff --git a/data/files/multi_delim.txt b/data/files/multi_delim.txt new file mode 100644 index 000000000000..4180463f1f7f --- /dev/null +++ b/data/files/multi_delim.txt @@ -0,0 +1,6 @@ +1~|alpha~|100 +2~|beta~|200 +3~|gamma~|300 +4~|delta~|400 +5~|~|500 +6~|epsilon~|600 diff --git a/data/files/single_delim.txt b/data/files/single_delim.txt new file mode 100644 index 000000000000..c3e6bf72170b --- /dev/null +++ b/data/files/single_delim.txt @@ -0,0 +1,6 @@ +1|alpha|100 +2|beta|200 +3|gamma|300 +4|delta|400 +5||500 +6|epsilon|600 diff --git a/data/files/single_delim_comma.txt b/data/files/single_delim_comma.txt new file mode 100644 index 000000000000..195fff8cbb99 --- /dev/null +++ b/data/files/single_delim_comma.txt @@ -0,0 +1,6 @@ +1,alpha,100 +2,beta,200 +3,gamma,300 +4,delta,400 +5,,500 +6,epsilon,600 diff --git a/data/files/single_delim_escape.txt b/data/files/single_delim_escape.txt new file mode 100644 index 000000000000..94f8da9869e0 --- /dev/null +++ b/data/files/single_delim_escape.txt @@ -0,0 +1,6 @@ +1|alpha\|inner|100 +2|beta|200 +3|gamma\|more\|pipes|300 +4|delta|400 +5||500 +6|epsilon|600 diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 5b077e8f5bf5..70262dc565de 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -118,6 +118,7 @@ minillap.query.files=\ llap_io_cache.q,\ llap_nullscan.q,\ llap_stats.q,\ + llap_text_multi_delim.q,\ llap_udf.q,\ llapdecider.q,\ load_binary_data.q,\ diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java index a55b64cdcbf1..c7a331d400e2 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.nio.charset.StandardCharsets; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -55,6 +56,7 @@ import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.MultiDelimitSerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; @@ -103,38 +105,57 @@ public static EncodingWriter create(InputFormat sourceIf, Deserializer ser List sourceIncludes, boolean[] cacheIncludes, int allocSize, ExecutorService encodeExecutor) throws IOException { // Vector SerDe can be disabled both on client and server side. + // MultiDelimitSerDe is accepted alongside LazySimpleSerDe: it's the same + // text-row shape, only its top-level FIELD_DELIM can span multiple bytes. + // The reader (LazySimpleDeserializeRead) picks the multi-byte scanner + // when LazySerDeParameters.setFieldDelimMulti has been called below — + // otherwise the specialized single-byte hot loop runs unchanged. + final boolean isLazySimple = serDe instanceof LazySimpleSerDe; + final boolean isMultiDelim = serDe instanceof MultiDelimitSerDe; if (!HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED) || !HiveConf.getBoolVar(jobConf, ConfVars.LLAP_IO_ENCODE_VECTOR_SERDE_ENABLED) - || !(sourceIf instanceof TextInputFormat) || !(serDe instanceof LazySimpleSerDe)) { + || !(sourceIf instanceof TextInputFormat) || !(isLazySimple || isMultiDelim)) { return new DeserializerOrcWriter(serDe, sourceOi, allocSize); } Path path = splitPath.getFileSystem(jobConf).makeQualified(splitPath); PartitionDesc partDesc = HiveFileFormatUtils.getFromPathRecursively(parts, path, null); if (partDesc == null) { - LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter: no partition desc for " + path); + LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter: no partition desc for " + path); return new DeserializerOrcWriter(serDe, sourceOi, allocSize); } Properties tblProps = partDesc.getTableDesc().getProperties(); if ("true".equalsIgnoreCase(tblProps.getProperty( serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST))) { - LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter due to " + LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter due to " + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST); return new DeserializerOrcWriter(serDe, sourceOi, allocSize); } + // MultiDelimitSerDe's own row parser (LazyStruct.parseMultiDelimit) does + // NOT honour escape.delim for the top-level delimiter. Combining a + // multi-byte field delim with an escape char in the fast path would let + // it silently diverge from the slow path, so bail to DeserializerOrcWriter + // whenever both are configured on a MultiDelimit table. + if (isMultiDelim && tblProps.getProperty(serdeConstants.ESCAPE_CHAR) != null) { + LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter: MultiDelimitSerDe with " + + serdeConstants.ESCAPE_CHAR + " is not supported by the vectorized fast path"); + return new DeserializerOrcWriter(serDe, sourceOi, allocSize); + } for (StructField sf : sourceOi.getAllStructFieldRefs()) { Category c = sf.getFieldObjectInspector().getCategory(); if (c != Category.PRIMITIVE) { - LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter: " + c + " is not supported"); + LlapIoImpl.LOG.info("Not using VectorDeserializeOrcWriter: " + c + " is not supported"); return new DeserializerOrcWriter(serDe, sourceOi, allocSize); } } - LlapIoImpl.LOG.info("Creating VertorDeserializeOrcWriter for " + path); + LlapIoImpl.LOG.info("Creating VectorDeserializeOrcWriter for " + path); return new VectorDeserializeOrcWriter( - jobConf, tblProps, sourceOi, sourceIncludes, cacheIncludes, allocSize, encodeExecutor); + jobConf, tblProps, sourceOi, sourceIncludes, cacheIncludes, allocSize, encodeExecutor, + serDe); } private VectorDeserializeOrcWriter(Configuration conf, Properties tblProps, StructObjectInspector sourceOi, - List sourceIncludes, boolean[] cacheIncludes, int allocSize, ExecutorService encodeExecutor) + List sourceIncludes, boolean[] cacheIncludes, int allocSize, ExecutorService encodeExecutor, + Deserializer serDe) throws IOException { super(sourceOi, allocSize); // See also: the usage of VectorDeserializeType, for binary. For now, we only want text. @@ -143,7 +164,8 @@ private VectorDeserializeOrcWriter(Configuration conf, Properties tblProps, Stru this.cacheIncludes = cacheIncludes; this.sourceBatch = vrbCtx.createVectorizedRowBatch(); deserializeRead = new LazySimpleDeserializeRead(vrbCtx.getRowColumnTypeInfos(), - vrbCtx.getRowdataTypePhysicalVariations(),/* useExternalBuffer */ true, createSerdeParams(conf, tblProps)); + vrbCtx.getRowdataTypePhysicalVariations(),/* useExternalBuffer */ true, + createSerdeParams(conf, tblProps, serDe)); vectorDeserializeRow = new VectorDeserializeRow(deserializeRead); int colCount = vrbCtx.getRowColumnTypeInfos().length; boolean[] includes = null; @@ -212,8 +234,10 @@ private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, fina .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); final String serde = tblProps.getProperty(serdeConstants.SERIALIZATION_LIB); final String inputFormat = tblProps.getProperty(hive_metastoreConstants.FILE_INPUT_FORMAT); - final boolean isTextFormat = inputFormat != null && inputFormat.equals(TextInputFormat.class.getName()) && - serde != null && serde.equals(LazySimpleSerDe.class.getName()); + final boolean isTextFormat = inputFormat != null && inputFormat.equals(TextInputFormat.class.getName()) + && serde != null + && (serde.equals(LazySimpleSerDe.class.getName()) + || serde.equals(MultiDelimitSerDe.class.getName())); List dataTypePhysicalVariations = new ArrayList<>(); if (isTextFormat) { StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(oi); @@ -245,9 +269,22 @@ private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, fina } private static LazySerDeParameters createSerdeParams( - Configuration conf, Properties tblProps) throws IOException { + Configuration conf, Properties tblProps, Deserializer serDe) throws IOException { try { - return new LazySerDeParameters(conf, tblProps, LazySimpleSerDe.class.getName()); + LazySerDeParameters params = + new LazySerDeParameters(conf, tblProps, LazySimpleSerDe.class.getName()); + // For MultiDelimitSerDe tables, if the field delimiter is > 1 byte, opt + // the reader into its multi-byte scan branch. LazySerDeParameters + // silently ignores single-byte values here, so misconfigured MultiDelim + // tables (e.g. FIELD_DELIM=",") still take the specialized single-byte + // fast path via separators[0]. + if (serDe instanceof MultiDelimitSerDe) { + String rawDelim = tblProps.getProperty(serdeConstants.FIELD_DELIM); + if (rawDelim != null && rawDelim.length() > 1) { + params.setFieldDelimMulti(rawDelim.getBytes(StandardCharsets.UTF_8)); + } + } + return params; } catch (SerDeException e) { throw new IOException(e); } diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java index beca9fa7ed87..110d8348aa8b 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestVectorDeserializeOrcWriter.java @@ -19,10 +19,14 @@ package org.apache.hadoop.hive.llap.io.encoded; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Properties; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch; import org.apache.hadoop.hive.llap.io.decode.EncodedDataConsumer; @@ -30,13 +34,20 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.io.orc.WriterImpl; import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.MultiDelimitSerDe; +import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hive.common.util.FixedSizedObjectPool; import org.apache.orc.impl.SchemaEvolution; import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; @@ -145,6 +156,70 @@ private static VectorDeserializeOrcWriter createOrcWriter( return orcWriter; } + // --- createSerdeParams: MultiDelimit routing → fieldDelimMulti wiring ----- + + private static LazySerDeParameters invokeCreateSerdeParams(Properties tblProps, + Deserializer serDe) throws Exception { + Method m = VectorDeserializeOrcWriter.class.getDeclaredMethod( + "createSerdeParams", Configuration.class, Properties.class, Deserializer.class); + m.setAccessible(true); + return (LazySerDeParameters) m.invoke(null, new Configuration(false), tblProps, serDe); + } + + /** + * When routing sees a MultiDelimitSerDe with a multi-byte field.delim, the + * LazySerDeParameters we hand to LazySimpleDeserializeRead must carry the + * raw delimiter bytes — that's the signal the reader uses to switch to its + * multi-byte scan branch (and hence keep the row on the vectorized LLAP + * fast path instead of falling back to DeserializerOrcWriter). + */ + @Test + public void testCreateSerdeParamsMultiDelimSetsFieldDelimMulti() throws Exception { + Properties tblProps = new Properties(); + tblProps.setProperty(serdeConstants.FIELD_DELIM, "~|"); + tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|"); + tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b"); + tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string"); + + LazySerDeParameters params = invokeCreateSerdeParams(tblProps, new MultiDelimitSerDe()); + assertArrayEquals("~|".getBytes(StandardCharsets.UTF_8), params.getFieldDelimMulti()); + } + + /** + * A LazySimpleSerDe table must NEVER opt into the multi-byte scan branch, + * even if someone shoved a >1-byte value into field.delim — the slow path + * silently truncates to a single byte, and diverging here would produce + * different results between fast and slow paths for the same table. + */ + @Test + public void testCreateSerdeParamsLazySimpleNeverSetsFieldDelimMulti() throws Exception { + Properties tblProps = new Properties(); + tblProps.setProperty(serdeConstants.FIELD_DELIM, "~|"); + tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|"); + tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b"); + tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string"); + + LazySerDeParameters params = invokeCreateSerdeParams(tblProps, new LazySimpleSerDe()); + assertNull(params.getFieldDelimMulti()); + } + + /** + * A MultiDelimit table whose field.delim is only one byte still takes the + * specialized single-byte fast path — setFieldDelimMulti drops sub-2-byte + * values, so separators[0] remains the source of truth. + */ + @Test + public void testCreateSerdeParamsMultiDelimWithSingleByteFallsThrough() throws Exception { + Properties tblProps = new Properties(); + tblProps.setProperty(serdeConstants.FIELD_DELIM, "|"); + tblProps.setProperty(serdeConstants.SERIALIZATION_FORMAT, "|"); + tblProps.setProperty(serdeConstants.LIST_COLUMNS, "a,b"); + tblProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string:string"); + + LazySerDeParameters params = invokeCreateSerdeParams(tblProps, new MultiDelimitSerDe()); + assertNull(params.getFieldDelimMulti()); + } + private static EncodedDataConsumer createBlankEncodedDataConsumer() { return new EncodedDataConsumer(null, 1, null, null) { @Override diff --git a/ql/src/test/queries/clientpositive/llap_text_multi_delim.q b/ql/src/test/queries/clientpositive/llap_text_multi_delim.q new file mode 100644 index 000000000000..aa2cb87bc742 --- /dev/null +++ b/ql/src/test/queries/clientpositive/llap_text_multi_delim.q @@ -0,0 +1,129 @@ +-- Verify that after wiring MultiDelimitSerDe into VectorDeserializeOrcWriter, +-- the LazySimpleSerDe (single-byte) fast path still works unchanged across +-- several delimiter shapes, and the new MultiDelimitSerDe (multi-byte) path +-- produces identical results for equivalent data. +set hive.llap.io.enabled=true; +set hive.llap.io.encode.enabled=true; +set hive.llap.io.encode.vector.serde.enabled=true; +set hive.llap.io.encode.vector.serde.async.enabled=true; +set hive.fetch.task.conversion=none; + +-- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS lazy_simple_llap; +DROP TABLE IF EXISTS lazy_simple_comma_llap; +DROP TABLE IF EXISTS lazy_simple_escape_llap; +DROP TABLE IF EXISTS multi_delim_llap; +DROP TABLE IF EXISTS multi_delim_escape_llap; + +-- --------------------------------------------------------------------------- +-- LazySimpleSerDe: single-byte '|' — the classic hot loop through +-- VectorDeserializeOrcWriter → LazySimpleDeserializeRead. Nothing changed on +-- this path; the queries below assert it still parses cleanly and vectorizes. +-- --------------------------------------------------------------------------- +CREATE TABLE lazy_simple_llap(id INT, name STRING, val INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' +STORED AS TEXTFILE; + +DESCRIBE FORMATTED lazy_simple_llap; + +LOAD DATA LOCAL INPATH '../../data/files/single_delim.txt' + INTO TABLE lazy_simple_llap; + +SELECT * FROM lazy_simple_llap; +SELECT COUNT(*) FROM lazy_simple_llap; +SELECT SUM(val) FROM lazy_simple_llap; +SELECT COUNT(*) FROM lazy_simple_llap WHERE name IS NULL OR name = ''; + +-- --------------------------------------------------------------------------- +-- LazySimpleSerDe: a different single-byte delimiter (',') — proves the fast +-- path is not hardcoded to '|' and picks up whatever FIELD_DELIM says. +-- --------------------------------------------------------------------------- +CREATE TABLE lazy_simple_comma_llap(id INT, name STRING, val INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +STORED AS TEXTFILE; + +DESCRIBE FORMATTED lazy_simple_comma_llap; + +LOAD DATA LOCAL INPATH '../../data/files/single_delim_comma.txt' + INTO TABLE lazy_simple_comma_llap; + +SELECT * FROM lazy_simple_comma_llap; +SELECT COUNT(*) FROM lazy_simple_comma_llap; +SELECT SUM(val) FROM lazy_simple_comma_llap; + +-- --------------------------------------------------------------------------- +-- LazySimpleSerDe: '|' with escape.delim='\' — exercises the escape branch of +-- LazySimpleDeserializeRead (currentExternalBufferNeeded / copyToExternalBuffer) +-- through the LLAP encoder. Data contains '\|' inside fields which must be +-- unescaped to a literal '|', NOT treated as a field boundary. +-- --------------------------------------------------------------------------- +CREATE TABLE lazy_simple_escape_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ('field.delim'='|', 'escape.delim'='\\') +STORED AS TEXTFILE; + +DESCRIBE FORMATTED lazy_simple_escape_llap; + +LOAD DATA LOCAL INPATH '../../data/files/single_delim_escape.txt' + INTO TABLE lazy_simple_escape_llap; + +SELECT * FROM lazy_simple_escape_llap; +SELECT COUNT(*) FROM lazy_simple_escape_llap; +SELECT SUM(val) FROM lazy_simple_escape_llap; +-- The two fields with '|' inside them must survive as-is. +SELECT id, name FROM lazy_simple_escape_llap WHERE name LIKE '%|%'; + +-- --------------------------------------------------------------------------- +-- MultiDelimitSerDe: multi-byte '~|' — must now route through the same +-- VectorDeserializeOrcWriter path via the new fieldDelimMulti wiring. +-- --------------------------------------------------------------------------- +CREATE TABLE multi_delim_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.MultiDelimitSerDe' +WITH SERDEPROPERTIES ('field.delim'='~|') +STORED AS TEXTFILE; + +DESCRIBE FORMATTED multi_delim_llap; + +LOAD DATA LOCAL INPATH '../../data/files/multi_delim.txt' + INTO TABLE multi_delim_llap; + +SELECT * FROM multi_delim_llap; +SELECT COUNT(*) FROM multi_delim_llap; +SELECT SUM(val) FROM multi_delim_llap; + +-- Parity check: LazySimple '|' and MultiDelimit '~|' carry the same rows. +SELECT COUNT(*) FROM ( + SELECT id, name, val FROM lazy_simple_llap + EXCEPT + SELECT id, name, val FROM multi_delim_llap +) diff1; + +SELECT COUNT(*) FROM ( + SELECT id, name, val FROM multi_delim_llap + EXCEPT + SELECT id, name, val FROM lazy_simple_llap +) diff2; + +-- --------------------------------------------------------------------------- +-- MultiDelimitSerDe + escape.delim must NOT take the fast path — the router +-- falls back to DeserializerOrcWriter — but the query must still succeed and +-- return the same rows. +-- --------------------------------------------------------------------------- +CREATE TABLE multi_delim_escape_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.MultiDelimitSerDe' +WITH SERDEPROPERTIES ('field.delim'='~|', 'escape.delim'='\\') +STORED AS TEXTFILE; + +DESCRIBE FORMATTED multi_delim_escape_llap; + +LOAD DATA LOCAL INPATH '../../data/files/multi_delim.txt' + INTO TABLE multi_delim_escape_llap; + +SELECT COUNT(*) FROM multi_delim_escape_llap; + +DROP TABLE lazy_simple_llap; +DROP TABLE lazy_simple_comma_llap; +DROP TABLE lazy_simple_escape_llap; +DROP TABLE multi_delim_llap; +DROP TABLE multi_delim_escape_llap; diff --git a/ql/src/test/results/clientpositive/llap/llap_text_multi_delim.q.out b/ql/src/test/results/clientpositive/llap/llap_text_multi_delim.q.out new file mode 100644 index 000000000000..6c34fba7fd29 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/llap_text_multi_delim.q.out @@ -0,0 +1,577 @@ +PREHOOK: query: DROP TABLE IF EXISTS lazy_simple_llap +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS lazy_simple_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: DROP TABLE IF EXISTS lazy_simple_comma_llap +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS lazy_simple_comma_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: DROP TABLE IF EXISTS lazy_simple_escape_llap +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS lazy_simple_escape_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: DROP TABLE IF EXISTS multi_delim_llap +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS multi_delim_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: DROP TABLE IF EXISTS multi_delim_escape_llap +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS multi_delim_escape_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: CREATE TABLE lazy_simple_llap(id INT, name STRING, val INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@lazy_simple_llap +POSTHOOK: query: CREATE TABLE lazy_simple_llap(id INT, name STRING, val INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lazy_simple_llap +PREHOOK: query: DESCRIBE FORMATTED lazy_simple_llap +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@lazy_simple_llap +POSTHOOK: query: DESCRIBE FORMATTED lazy_simple_llap +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@lazy_simple_llap +# col_name data_type comment +id int +name string +val int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"name\":\"true\",\"val\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim | + serialization.format | +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/single_delim.txt' + INTO TABLE lazy_simple_llap +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@lazy_simple_llap +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/single_delim.txt' + INTO TABLE lazy_simple_llap +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@lazy_simple_llap +PREHOOK: query: SELECT * FROM lazy_simple_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM lazy_simple_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +1 alpha 100 +2 beta 200 +3 gamma 300 +4 delta 400 +5 500 +6 epsilon 600 +PREHOOK: query: SELECT COUNT(*) FROM lazy_simple_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM lazy_simple_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +6 +PREHOOK: query: SELECT SUM(val) FROM lazy_simple_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(val) FROM lazy_simple_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +2100 +PREHOOK: query: SELECT COUNT(*) FROM lazy_simple_llap WHERE name IS NULL OR name = '' +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM lazy_simple_llap WHERE name IS NULL OR name = '' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_llap +#### A masked pattern was here #### +1 +PREHOOK: query: CREATE TABLE lazy_simple_comma_llap(id INT, name STRING, val INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@lazy_simple_comma_llap +POSTHOOK: query: CREATE TABLE lazy_simple_comma_llap(id INT, name STRING, val INT) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lazy_simple_comma_llap +PREHOOK: query: DESCRIBE FORMATTED lazy_simple_comma_llap +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@lazy_simple_comma_llap +POSTHOOK: query: DESCRIBE FORMATTED lazy_simple_comma_llap +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@lazy_simple_comma_llap +# col_name data_type comment +id int +name string +val int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"name\":\"true\",\"val\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim , + serialization.format , +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/single_delim_comma.txt' + INTO TABLE lazy_simple_comma_llap +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@lazy_simple_comma_llap +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/single_delim_comma.txt' + INTO TABLE lazy_simple_comma_llap +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@lazy_simple_comma_llap +PREHOOK: query: SELECT * FROM lazy_simple_comma_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_comma_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM lazy_simple_comma_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_comma_llap +#### A masked pattern was here #### +1 alpha 100 +2 beta 200 +3 gamma 300 +4 delta 400 +5 500 +6 epsilon 600 +PREHOOK: query: SELECT COUNT(*) FROM lazy_simple_comma_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_comma_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM lazy_simple_comma_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_comma_llap +#### A masked pattern was here #### +6 +PREHOOK: query: SELECT SUM(val) FROM lazy_simple_comma_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_comma_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(val) FROM lazy_simple_comma_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_comma_llap +#### A masked pattern was here #### +2100 +PREHOOK: query: CREATE TABLE lazy_simple_escape_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ('field.delim'='|', 'escape.delim'='\\') +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@lazy_simple_escape_llap +POSTHOOK: query: CREATE TABLE lazy_simple_escape_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ('field.delim'='|', 'escape.delim'='\\') +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lazy_simple_escape_llap +PREHOOK: query: DESCRIBE FORMATTED lazy_simple_escape_llap +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@lazy_simple_escape_llap +POSTHOOK: query: DESCRIBE FORMATTED lazy_simple_escape_llap +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@lazy_simple_escape_llap +# col_name data_type comment +id int +name string +val int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"name\":\"true\",\"val\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + escape.delim \\ + field.delim | + serialization.format 1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/single_delim_escape.txt' + INTO TABLE lazy_simple_escape_llap +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@lazy_simple_escape_llap +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/single_delim_escape.txt' + INTO TABLE lazy_simple_escape_llap +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@lazy_simple_escape_llap +PREHOOK: query: SELECT * FROM lazy_simple_escape_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM lazy_simple_escape_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +1 alpha|inner 100 +2 beta 200 +3 gamma|more|pipes 300 +4 delta 400 +5 500 +6 epsilon 600 +PREHOOK: query: SELECT COUNT(*) FROM lazy_simple_escape_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM lazy_simple_escape_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +6 +PREHOOK: query: SELECT SUM(val) FROM lazy_simple_escape_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(val) FROM lazy_simple_escape_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +2100 +PREHOOK: query: SELECT id, name FROM lazy_simple_escape_llap WHERE name LIKE '%|%' +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT id, name FROM lazy_simple_escape_llap WHERE name LIKE '%|%' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_escape_llap +#### A masked pattern was here #### +1 alpha|inner +3 gamma|more|pipes +PREHOOK: query: CREATE TABLE multi_delim_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.MultiDelimitSerDe' +WITH SERDEPROPERTIES ('field.delim'='~|') +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@multi_delim_llap +POSTHOOK: query: CREATE TABLE multi_delim_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.MultiDelimitSerDe' +WITH SERDEPROPERTIES ('field.delim'='~|') +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@multi_delim_llap +PREHOOK: query: DESCRIBE FORMATTED multi_delim_llap +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@multi_delim_llap +POSTHOOK: query: DESCRIBE FORMATTED multi_delim_llap +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@multi_delim_llap +# col_name data_type comment +id int from deserializer +name string from deserializer +val int from deserializer + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"name\":\"true\",\"val\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.MultiDelimitSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim ~| + serialization.format 1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/multi_delim.txt' + INTO TABLE multi_delim_llap +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@multi_delim_llap +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/multi_delim.txt' + INTO TABLE multi_delim_llap +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@multi_delim_llap +PREHOOK: query: SELECT * FROM multi_delim_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM multi_delim_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +1 alpha 100 +2 beta 200 +3 gamma 300 +4 delta 400 +5 500 +6 epsilon 600 +PREHOOK: query: SELECT COUNT(*) FROM multi_delim_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM multi_delim_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +6 +PREHOOK: query: SELECT SUM(val) FROM multi_delim_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(val) FROM multi_delim_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +2100 +PREHOOK: query: SELECT COUNT(*) FROM ( + SELECT id, name, val FROM lazy_simple_llap + EXCEPT + SELECT id, name, val FROM multi_delim_llap +) diff1 +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_llap +PREHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM ( + SELECT id, name, val FROM lazy_simple_llap + EXCEPT + SELECT id, name, val FROM multi_delim_llap +) diff1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_llap +POSTHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +0 +PREHOOK: query: SELECT COUNT(*) FROM ( + SELECT id, name, val FROM multi_delim_llap + EXCEPT + SELECT id, name, val FROM lazy_simple_llap +) diff2 +PREHOOK: type: QUERY +PREHOOK: Input: default@lazy_simple_llap +PREHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM ( + SELECT id, name, val FROM multi_delim_llap + EXCEPT + SELECT id, name, val FROM lazy_simple_llap +) diff2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lazy_simple_llap +POSTHOOK: Input: default@multi_delim_llap +#### A masked pattern was here #### +0 +PREHOOK: query: CREATE TABLE multi_delim_escape_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.MultiDelimitSerDe' +WITH SERDEPROPERTIES ('field.delim'='~|', 'escape.delim'='\\') +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@multi_delim_escape_llap +POSTHOOK: query: CREATE TABLE multi_delim_escape_llap(id INT, name STRING, val INT) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.MultiDelimitSerDe' +WITH SERDEPROPERTIES ('field.delim'='~|', 'escape.delim'='\\') +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@multi_delim_escape_llap +PREHOOK: query: DESCRIBE FORMATTED multi_delim_escape_llap +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@multi_delim_escape_llap +POSTHOOK: query: DESCRIBE FORMATTED multi_delim_escape_llap +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@multi_delim_escape_llap +# col_name data_type comment +id int from deserializer +name string from deserializer +val int from deserializer + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"name\":\"true\",\"val\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize #Masked# +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.MultiDelimitSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + escape.delim \\ + field.delim ~| + serialization.format 1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/multi_delim.txt' + INTO TABLE multi_delim_escape_llap +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@multi_delim_escape_llap +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/multi_delim.txt' + INTO TABLE multi_delim_escape_llap +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@multi_delim_escape_llap +PREHOOK: query: SELECT COUNT(*) FROM multi_delim_escape_llap +PREHOOK: type: QUERY +PREHOOK: Input: default@multi_delim_escape_llap +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM multi_delim_escape_llap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@multi_delim_escape_llap +#### A masked pattern was here #### +6 +PREHOOK: query: DROP TABLE lazy_simple_llap +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@lazy_simple_llap +PREHOOK: Output: database:default +PREHOOK: Output: default@lazy_simple_llap +POSTHOOK: query: DROP TABLE lazy_simple_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@lazy_simple_llap +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lazy_simple_llap +PREHOOK: query: DROP TABLE lazy_simple_comma_llap +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@lazy_simple_comma_llap +PREHOOK: Output: database:default +PREHOOK: Output: default@lazy_simple_comma_llap +POSTHOOK: query: DROP TABLE lazy_simple_comma_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@lazy_simple_comma_llap +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lazy_simple_comma_llap +PREHOOK: query: DROP TABLE lazy_simple_escape_llap +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@lazy_simple_escape_llap +PREHOOK: Output: database:default +PREHOOK: Output: default@lazy_simple_escape_llap +POSTHOOK: query: DROP TABLE lazy_simple_escape_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@lazy_simple_escape_llap +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lazy_simple_escape_llap +PREHOOK: query: DROP TABLE multi_delim_llap +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@multi_delim_llap +PREHOOK: Output: database:default +PREHOOK: Output: default@multi_delim_llap +POSTHOOK: query: DROP TABLE multi_delim_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@multi_delim_llap +POSTHOOK: Output: database:default +POSTHOOK: Output: default@multi_delim_llap +PREHOOK: query: DROP TABLE multi_delim_escape_llap +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@multi_delim_escape_llap +PREHOOK: Output: database:default +PREHOOK: Output: default@multi_delim_escape_llap +POSTHOOK: query: DROP TABLE multi_delim_escape_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@multi_delim_escape_llap +POSTHOOK: Output: database:default +POSTHOOK: Output: default@multi_delim_escape_llap diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java index 9da6bf5e3c35..1db95d3b086f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java @@ -60,11 +60,18 @@ public class LazySerDeParameters implements LazyObjectInspectorParameters { private Properties tableProperties; private String serdeName; - // The list of bytes used for the separators in the column (a nested struct + // The list of bytes used for the separators in the column (a nested struct // such as Array> will use multiple separators). // The list of separators + escapeChar are the bytes required to be escaped. private byte[] separators; + // Populated only when the top-level field delimiter is longer than one byte + // (i.e. by MultiDelimitSerDe). Never populated for LazySimpleSerDe callers, + // so existing single-byte behaviour is preserved. The single-byte `separators[0]` + // slot is still set to the first byte of the multi-byte delim so that + // needsEscape[] and any legacy consumers of getSeparators() keep working. + private byte[] fieldDelimMulti; + private Text nullSequence; private TypeInfo rowTypeInfo; @@ -196,6 +203,31 @@ public byte[] getSeparators() { return separators; } + /** + * @return the raw (potentially multi-byte) top-level field delimiter, or + * {@code null} when the effective delimiter is a single byte (the + * value in {@code separators[0]} is authoritative in that case). + * + * Set by callers that know they're wiring up a multi-byte delimiter + * (e.g. LLAP's {@code VectorDeserializeOrcWriter} when the SerDe is + * {@code MultiDelimitSerDe}). Left {@code null} for plain LazySimpleSerDe so + * its single-byte fast path is preserved verbatim. + */ + public byte[] getFieldDelimMulti() { + return fieldDelimMulti; + } + + /** + * Opt-in switch for the multi-byte top-level delimiter. Only meaningful when + * {@code delim.length > 1}; shorter values are ignored (the {@code separators[0]} + * byte remains the source of truth). Never set this for a + * LazySimpleSerDe-backed table — its slow path still truncates to a single + * byte, so opting the fast path into multi-byte would silently diverge. + */ + public void setFieldDelimMulti(byte[] delim) { + this.fieldDelimMulti = (delim != null && delim.length > 1) ? delim : null; + } + public Text getNullSequence() { return nullSequence; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index 747d389db56a..bf991d017faa 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -71,6 +71,29 @@ * Some type values are by reference to either bytes in the deserialization buffer or to * other type specific buffers. So, those references are only valid until the next time set is * called. + * + *

Multi-byte top-level field delimiter

+ * By default this reader assumes a single-byte top-level FIELD_DELIM and runs a specialized + * byte-at-a-time hot loop against {@code separators[0]} — the LazySimpleSerDe fast path. + * + * A multi-byte top-level delimiter (e.g. MultiDelimitSerDe's {@code ~|}) is supported when, and + * only when, the caller opts in via + * {@link org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#setFieldDelimMulti(byte[])} on + * the {@code LazySerDeParameters} passed to the constructor. In that mode {@code topLevelParse()} + * takes a separate multi-byte scan branch and the specialized single-byte loop is not entered. + * + * Restrictions when opting into the multi-byte branch: + *
    + *
  • only the top-level field delimiter is multi-byte — nested COLLECTION_DELIM / + * MAPKEY_DELIM remain single-byte;
    • + *
  • ESCAPE_CHAR is rejected at construction (see {@code MultiDelimitSerDe.parseMultiDelimit} + * which itself ignores escape at the top level — enabling both here would silently + * diverge from the slow path);
    • + *
  • LAST_COLUMN_TAKES_REST is not honoured on this path (the LLAP router bails to + * DeserializerOrcWriter before we get here).
    • + *
+ * When {@code setFieldDelimMulti} is not called (or the value is shorter than 2 bytes), the + * single-byte fast path is unchanged. */ public final class LazySimpleDeserializeRead extends DeserializeRead { public static final Logger LOG = LoggerFactory.getLogger(LazySimpleDeserializeRead.class.getName()); @@ -198,6 +221,18 @@ public UnionComplexTypeHelper(Field complexField, Field[] fields) { private int[] startPositions; private final byte[] separators; + // Multi-byte top-level field delimiter (e.g. MultiDelimitSerDe's `~|`). + // null in the common single-byte case, in which case `separators[0]` is used + // and the row-scan hot loop keeps its per-byte specialization. Non-null only + // when the caller (LLAP's VectorDeserializeOrcWriter for MultiDelimitSerDe) + // explicitly opted in via LazySerDeParameters.setFieldDelimMulti. + private final byte[] fieldDelimMulti; + // Byte length "charged" to a top-level separator: 1 for the single-byte fast + // path, delim.length for multi-byte. Baked into (a) the Arrays.fill sentinel + // for missing/trailing fields and (b) the "length = next.start - this.start + // - sepLen" arithmetic in readField and getDetailedReadPositionString, so + // both paths share the same downstream code. + private final int topLevelSeparatorLen; private final boolean isEscaped; private final byte escapeChar; private final int[] escapeCounts; @@ -336,11 +371,23 @@ public LazySimpleDeserializeRead(TypeInfo[] typeInfos, startPositions = new int[count + 1]; this.separators = lazyParams.getSeparators(); + this.fieldDelimMulti = lazyParams.getFieldDelimMulti(); + this.topLevelSeparatorLen = (fieldDelimMulti == null) ? 1 : fieldDelimMulti.length; isEscaped = lazyParams.isEscaped(); if (isEscaped) { escapeChar = lazyParams.getEscapeChar(); escapeCounts = new int[count]; + // Multi-byte field delimiters with escape sequences are not implemented + // in this fast path — the escape/separator interaction would diverge + // from MultiDelimitSerDe's own row-based parser, which ignores escapes + // for the top-level delim. Callers (VectorDeserializeOrcWriter) route + // this combination to the DeserializerOrcWriter slow path instead. + if (fieldDelimMulti != null) { + throw new RuntimeException( + "Multi-byte field delimiter combined with escape.delim is not " + + "supported in the vectorized fast path."); + } } else { escapeChar = (byte) 0; escapeCounts = null; @@ -402,7 +449,7 @@ public String getDetailedReadPositionString() { sb.append(" at field start position "); sb.append(startPositions[currentTopLevelFieldIndex]); int currentFieldLength = startPositions[currentTopLevelFieldIndex + 1] - - startPositions[currentTopLevelFieldIndex] - 1; + startPositions[currentTopLevelFieldIndex] - topLevelSeparatorLen; sb.append(" for field length "); sb.append(currentFieldLength); } @@ -410,6 +457,23 @@ public String getDetailedReadPositionString() { return sb.toString(); } + /** + * Bytes at {@code buf[off..off+dlen)} equal to {@code delim[0..dlen)}? + * + * Caller has already checked {@code buf[off] == delim[0]}, so we start at + * index 1 — this is only ever invoked when the first byte matched, which + * keeps the multi-byte hot loop from paying for a tail compare on every + * mismatching input byte. + */ + private static boolean matchesAt(byte[] buf, int off, byte[] delim, int dlen) { + for (int i = 1; i < dlen; i++) { + if (buf[off + i] != delim[i]) { + return false; + } + } + return true; + } + /** * Parse the byte[] and fill each field. * @@ -432,20 +496,54 @@ private void topLevelParse() { * Optimize the loops by pulling special end cases and global decisions like isEscaped out! */ if (!isEscaped) { - while (fieldByteEnd < end) { - if (bytes[fieldByteEnd] == separator) { + if (fieldDelimMulti == null) { + // Single-byte fast path — hot on every row of a LazySimple table. + while (fieldByteEnd < end) { + if (bytes[fieldByteEnd] == separator) { + startPositions[fieldId++] = fieldByteBegin; + if (fieldId == fieldCount) { + break; + } + fieldByteBegin = ++fieldByteEnd; + } else { + fieldByteEnd++; + } + } + // End serves as final separator. + if (fieldByteEnd == end && fieldId < fieldCount) { startPositions[fieldId++] = fieldByteBegin; - if (fieldId == fieldCount) { - break; + } + } else { + // Multi-byte top-level delimiter (MultiDelimitSerDe path). We keep the + // "test the first byte, then verify the tail" idiom so the mismatching- + // byte case still costs a single load+compare — the tail memcmp only + // fires on a first-byte hit. + final byte[] delim = this.fieldDelimMulti; + final int dlen = delim.length; + final byte first = delim[0]; + final int scanEnd = end - dlen; // last index at which a full delim can start + while (fieldByteEnd <= scanEnd) { + if (bytes[fieldByteEnd] == first && matchesAt(bytes, fieldByteEnd, delim, dlen)) { + startPositions[fieldId++] = fieldByteBegin; + if (fieldId == fieldCount) { + // Malformed row with more delims than expected: leave fieldByteEnd + // where it is (matching single-byte-path semantics) and stop. + break; + } + fieldByteEnd += dlen; + fieldByteBegin = fieldByteEnd; + } else { + fieldByteEnd++; } - fieldByteBegin = ++fieldByteEnd; - } else { - fieldByteEnd++; } - } - // End serves as final separator. - if (fieldByteEnd == end && fieldId < fieldCount) { - startPositions[fieldId++] = fieldByteBegin; + // No trailing delim (single-byte parses "end as final separator" here). + // If we still owe fields, the remainder from fieldByteBegin..end is the + // last field; fast-forward fieldByteEnd so the Arrays.fill sentinel and + // isEndOfInputReached below both see the row as fully consumed. + if (fieldId < fieldCount) { + fieldByteEnd = end; + startPositions[fieldId++] = fieldByteBegin; + } } } else { final byte escapeChar = this.escapeChar; @@ -497,7 +595,11 @@ private void topLevelParse() { // For missing fields, their starting positions will all be the same, // which will make their lengths to be -1 and uncheckedGetField will // return these fields as NULLs. - Arrays.fill(startPositions, fieldId, startPositions.length, fieldByteEnd + 1); + // Charge the actual separator width (1 byte for LazySimple, delim.length + // for the multi-byte MultiDelimit path) so the "length = next - this - + // sepLen" arithmetic used by readField comes out right in both cases. + Arrays.fill(startPositions, fieldId, startPositions.length, + fieldByteEnd + topLevelSeparatorLen); } isEndOfInputReached = (fieldByteEnd == end); @@ -639,7 +741,7 @@ public boolean readField(int fieldIndex) throws IOException { currentTopLevelFieldIndex = fieldIndex; currentFieldStart = startPositions[fieldIndex]; - currentFieldLength = startPositions[fieldIndex + 1] - startPositions[fieldIndex] - 1; + currentFieldLength = startPositions[fieldIndex + 1] - startPositions[fieldIndex] - topLevelSeparatorLen; currentEscapeCount = (isEscaped ? escapeCounts[fieldIndex] : 0); return doReadField(fields[fieldIndex]); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java index 465a616fa936..c1fbbea5a0cb 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/fast/TestLazySimpleDeserializeRead.java @@ -20,6 +20,8 @@ +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.util.Properties; import org.apache.hadoop.hive.conf.HiveConf; @@ -29,8 +31,12 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.Text; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.junit.Test; /** @@ -86,4 +92,240 @@ public void testEscaping() throws Exception { assertTrue("The escaped result is incorrect", field.compareTo(escaped) == 0); } + + // --- multi-byte field delimiter (MultiDelimitSerDe → LLAP fast path) ------ + + private static LazySerDeParameters multiDelimParams(String delim) throws Exception { + Properties props = new Properties(); + props.setProperty(serdeConstants.FIELD_DELIM, delim); + props.setProperty(serdeConstants.SERIALIZATION_FORMAT, delim); + LazySerDeParameters p = new LazySerDeParameters(new HiveConf(), props, + LazySimpleSerDe.class.getName()); + p.setFieldDelimMulti(delim.getBytes(StandardCharsets.UTF_8)); + return p; + } + + private static byte[] readStringField(LazySimpleDeserializeRead r) throws Exception { + assertTrue("expected non-null field", r.readNextField()); + int len = r.currentBytesLength; + byte[] out = new byte[len]; + System.arraycopy(r.currentBytes, r.currentBytesStart, out, 0, len); + return out; + } + + /** + * Three STRING columns separated by "~|" — the delimiter used by the BofA + * MultiDelimitSerDe workload that motivated this path. + */ + @Test + public void testMultiByteDelimThreeStringColumns() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "alpha~|beta~|gamma".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("alpha".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("beta".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("gamma".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertFalse("row should have exactly 3 fields", r.readNextField()); + } + + /** + * Mixed INT / STRING columns. Verifies that the length arithmetic charges + * the full delim.length (not 1) between fields so numeric parsing sees the + * correct field boundaries. + */ + @Test + public void testMultiByteDelimMixedTypes() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.intTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.longTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "42~|hello~|9876543210".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertTrue(r.readNextField()); + assertEquals(42, r.currentInt); + + assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), readStringField(r)); + + assertTrue(r.readNextField()); + assertEquals(9876543210L, r.currentLong); + } + + /** + * The first byte of the delimiter ("~") appearing standalone inside a field + * must NOT trigger a split — the tail-match on delim[1] rescues it. + */ + @Test + public void testMultiByteDelimFirstByteInsideField() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + // A bare '~' followed by non-'|' must stay inside the first field. + byte[] row = "foo~bar~|baz".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("foo~bar".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("baz".getBytes(StandardCharsets.UTF_8), readStringField(r)); + } + + /** + * The empty-tail case: last field extends to EOL and has zero length. + * startPositions must still tabulate length 0 (not -1), so NULL isn't + * spuriously returned. + */ + @Test + public void testMultiByteDelimEmptyLastField() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "one~|".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("one".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertTrue("empty last field should be non-null (zero-length)", r.readNextField()); + assertEquals(0, r.currentBytesLength); + } + + /** + * Missing trailing fields must land in the "startPositions filled with + * sentinel" branch and come back as NULL — same behavior as single-byte. + */ + @Test + public void testMultiByteDelimMissingTrailingFields() throws Exception { + LazySerDeParameters params = multiDelimParams("~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "only-one".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("only-one".getBytes(StandardCharsets.UTF_8), readStringField(r)); + // Missing fields → NULL. + assertFalse(r.readNextField()); + assertFalse(r.readNextField()); + } + + /** + * A three-byte delimiter exercises the "delim length > 2" arithmetic. + */ + @Test + public void testMultiByteDelimThreeByteDelimiter() throws Exception { + LazySerDeParameters params = multiDelimParams("|~|"); + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "a|~|bb|~|ccc".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + + assertArrayEquals("a".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("bb".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("ccc".getBytes(StandardCharsets.UTF_8), readStringField(r)); + } + + /** + * Multi-byte delim combined with escape.delim is not supported in the fast + * path — LazySimpleDeserializeRead must reject the combination at + * construction time so we never silently diverge from + * MultiDelimitSerDe.parseMultiDelimit (which itself ignores escape at the + * top level). + */ + @Test + public void testMultiByteDelimRejectsEscape() throws Exception { + Properties props = new Properties(); + props.setProperty(serdeConstants.FIELD_DELIM, "~|"); + props.setProperty(serdeConstants.SERIALIZATION_FORMAT, "~|"); + props.setProperty(serdeConstants.ESCAPE_CHAR, "\\"); + LazySerDeParameters params = new LazySerDeParameters(new HiveConf(), props, + LazySimpleSerDe.class.getName()); + params.setFieldDelimMulti("~|".getBytes(StandardCharsets.UTF_8)); + + TypeInfo[] typeInfos = new TypeInfo[] { TypeInfoFactory.stringTypeInfo }; + try { + new LazySimpleDeserializeRead(typeInfos, null, true, params); + fail("expected RuntimeException for multi-byte delim + escape.delim"); + } catch (RuntimeException expected) { + assertTrue("unexpected message: " + expected.getMessage(), + expected.getMessage().contains("Multi-byte field delimiter")); + } + } + + /** + * When the table is a plain LazySimpleSerDe (setFieldDelimMulti never called + * on the params), LazySimpleDeserializeRead must select the specialized + * single-byte hot loop in {@code topLevelParse()}. The branch is gated on + * {@code fieldDelimMulti == null}, so we prove path selection by reflecting + * on the reader's internal state after a successful parse: + *
    + *
  • {@code fieldDelimMulti} is {@code null} — the {@code if} in + * {@code topLevelParse()} evaluated to the single-byte branch;
    • + *
  • {@code topLevelSeparatorLen == 1} — the length arithmetic + * (both in the sentinel fill and in {@code readField}) uses the + * single-byte constant, not the multi-byte {@code dlen}.
    • + *
+ * A correct parse of the row confirms the branch actually ran. + */ + @Test + public void testSingleByteDelimHitsSpecializedHotPath() throws Exception { + Properties props = new Properties(); + props.setProperty(serdeConstants.FIELD_DELIM, "|"); + props.setProperty(serdeConstants.SERIALIZATION_FORMAT, "|"); + LazySerDeParameters params = new LazySerDeParameters(new HiveConf(), props, + LazySimpleSerDe.class.getName()); + // Intentionally do NOT call setFieldDelimMulti — this is a LazySimple table. + + TypeInfo[] typeInfos = new TypeInfo[] { + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo + }; + LazySimpleDeserializeRead r = new LazySimpleDeserializeRead(typeInfos, null, true, params); + + byte[] row = "alpha|beta|gamma".getBytes(StandardCharsets.UTF_8); + r.set(row, 0, row.length); + assertArrayEquals("alpha".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("beta".getBytes(StandardCharsets.UTF_8), readStringField(r)); + assertArrayEquals("gamma".getBytes(StandardCharsets.UTF_8), readStringField(r)); + + // Path-selection proof: the reader's fieldDelimMulti stayed null and its + // topLevelSeparatorLen is 1 — the only code path in topLevelParse() that + // could have produced the correct parse above. + assertNull("fieldDelimMulti must be null on the LazySimple fast path", + readPrivateField(r, "fieldDelimMulti")); + assertEquals("topLevelSeparatorLen must be 1 on the LazySimple fast path", + 1, ((Integer) readPrivateField(r, "topLevelSeparatorLen")).intValue()); + } + + private static Object readPrivateField(Object target, String name) throws Exception { + Field f = LazySimpleDeserializeRead.class.getDeclaredField(name); + f.setAccessible(true); + return f.get(target); + } }