diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorPercentRank.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorPercentRank.java
new file mode 100644
index 000000000000..7eeaac3b2d58
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorPercentRank.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.ptf;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
+
+/**
+ * Evaluates {@code percent_rank()} as a group-aggregated streaming
+ * evaluator.
+ *
+ *
+ * The partition is buffered so {@link #setPartitionSize(int)} is known before
+ * output is written.
+ * Peer-group rank is tracked incrementally (like
+ * {@link VectorPTFEvaluatorRank}) during batch
+ * forward; {@link #addStreamingGroupResults} is a no-op because no pre-pass is
+ * required.
+ */
+public class VectorPTFEvaluatorPercentRank extends VectorPTFEvaluatorBase {
+
+ private int rank;
+ private int groupCount;
+
+ public VectorPTFEvaluatorPercentRank(WindowFrameDef windowFrameDef, int outputColumnNum) {
+ super(windowFrameDef, outputColumnNum);
+ resetEvaluator();
+ }
+
+ @Override
+ public boolean isGroupAggregatedStreamingEvaluator() {
+ return true;
+ }
+
+ @Override
+ public void addStreamingGroupResults(List groupRowCounts) {
+ // Rank is advanced during batch forward; partition size alone is needed up
+ // front.
+ }
+
+ @Override
+ public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException {
+ if (partitionSize <= 0) {
+ throw new HiveException("Partition size must be set before computing percent_rank");
+ }
+ final double divisor = partitionSize > 1 ? partitionSize - 1 : 1;
+ DoubleColumnVector outputColVector = (DoubleColumnVector) batch.cols[outputColumnNum];
+ outputColVector.isRepeating = true;
+ outputColVector.noNulls = true;
+ outputColVector.isNull[0] = false;
+ outputColVector.vector[0] = (rank - 1) / divisor;
+ groupCount += batch.size;
+ }
+
+ @Override
+ public void doLastBatchWork() {
+ rank += groupCount;
+ groupCount = 0;
+ }
+
+ @Override
+ public boolean streamsResult() {
+ return true;
+ }
+
+ @Override
+ public Type getResultColumnVectorType() {
+ return Type.DOUBLE;
+ }
+
+ @Override
+ public void resetEvaluator() {
+ rank = 1;
+ partitionSize = -1;
+ groupCount = 0;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 00cdcc50ad18..e8a528babf6f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -3005,10 +3005,7 @@ private boolean validatePTFOperator(PTFOperator op, VectorizationContext vContex
throw new RuntimeException("Unexpected window type " + windowFrameDef.getWindowType());
}
- // RANK/DENSE_RANK/CUME_DIST don't care about columns.
- if (supportedFunctionType != SupportedFunctionType.RANK &&
- supportedFunctionType != SupportedFunctionType.DENSE_RANK &&
- supportedFunctionType != SupportedFunctionType.CUME_DIST) {
+ if (!VectorPTFDesc.COLUMN_AGNOSTIC_FUNCTIONS.contains(supportedFunctionType)) {
if (exprNodeDescList != null) {
// LEAD and LAG now supports multiple arguments in vectorized mode
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
index aab04bee56cf..d6038dab5126 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
@@ -20,8 +20,10 @@
package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
+import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.lang3.ArrayUtils;
@@ -58,6 +60,7 @@
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongMax;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongMin;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorLongSum;
+import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorPercentRank;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorRank;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorRowNumber;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFEvaluatorStreamingDecimalAvg;
@@ -93,6 +96,7 @@ public enum SupportedFunctionType {
ROW_NUMBER,
RANK,
DENSE_RANK,
+ PERCENT_RANK,
CUME_DIST,
MIN,
MAX,
@@ -133,6 +137,11 @@ public boolean isSupportDistinct() {
supportedFunctionNames.addAll(treeSet);
}
+ // functions that don't care about input columns.
+ public static final Set COLUMN_AGNOSTIC_FUNCTIONS =
+ EnumSet.of(SupportedFunctionType.RANK, SupportedFunctionType.DENSE_RANK,
+ SupportedFunctionType.PERCENT_RANK, SupportedFunctionType.CUME_DIST);
+
private TypeInfo[] reducerBatchTypeInfos;
private DataTypePhysicalVariation[] reducerBatchDataTypePhysicalVariations;
@@ -204,6 +213,9 @@ public static VectorPTFEvaluatorBase getEvaluator(SupportedFunctionType function
case DENSE_RANK:
evaluator = new VectorPTFEvaluatorDenseRank(windowFrameDef, outputColumnNum);
break;
+ case PERCENT_RANK:
+ evaluator = new VectorPTFEvaluatorPercentRank(windowFrameDef, outputColumnNum);
+ break;
case CUME_DIST:
evaluator = new VectorPTFEvaluatorCumeDist(windowFrameDef, outputColumnNum);
break;
diff --git a/ql/src/test/queries/clientpositive/cbo_windowing.q b/ql/src/test/queries/clientpositive/cbo_windowing.q
index c39f6848a294..1fa27fd96632 100644
--- a/ql/src/test/queries/clientpositive/cbo_windowing.q
+++ b/ql/src/test/queries/clientpositive/cbo_windowing.q
@@ -10,6 +10,11 @@ set hive.auto.convert.join=false;
-- 9. Test Windowing Functions
-- SORT_QUERY_RESULTS
+-- Vector PTF does not buffer PARTITION BY columns (constant within a partition). When a partition
+-- column is also a window-function argument (e.g. sum(c_float) OVER (PARTITION BY c_float)),
+-- input-column remapping is wrong and can cause ClassCastException. Use non-vector PTF until fixed.
+set hive.vectorized.execution.ptf.enabled=false;
+
select count(c_int) over() from cbo_t1;
select count(c_int) over(partition by c_float order by key), sum(c_float) over(partition by c_float order by key), max(c_int) over(partition by c_float order by key), min(c_int) over(partition by c_float order by key), row_number() over(partition by c_float order by key) as rn, rank() over(partition by c_float order by key), dense_rank() over(partition by c_float order by key), round(percent_rank() over(partition by c_float order by key), 2), lead(c_int, 2, c_int) over(partition by c_float order by key), lag(c_float, 2, c_float) over(partition by c_float order by key) from cbo_t1 order by rn;
select * from (select count(c_int) over(partition by c_float order by key), sum(c_float) over(partition by c_float order by key), max(c_int) over(partition by c_float order by key), min(c_int) over(partition by c_float order by key), row_number() over(partition by c_float order by key) as rn, rank() over(partition by c_float order by key), dense_rank() over(partition by c_float order by key), round(percent_rank() over(partition by c_float order by key),2), lead(c_int, 2, c_int) over(partition by c_float order by key ), lag(c_float, 2, c_float) over(partition by c_float order by key) from cbo_t1 order by rn) cbo_t1;
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_percent_rank.q b/ql/src/test/queries/clientpositive/vector_ptf_percent_rank.q
new file mode 100644
index 000000000000..f9316793ea86
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_percent_rank.q
@@ -0,0 +1,113 @@
+set hive.vectorized.testing.reducer.batch.size=2;
+
+DROP TABLE IF EXISTS vector_ptf_percent_rank_int;
+
+CREATE TABLE vector_ptf_percent_rank_int(name string, rowindex int, mynumber int) stored as orc;
+
+INSERT INTO vector_ptf_percent_rank_int values
+('five', 1, 10),
+('five', 2, 20),
+('five', 3, 30),
+('five', 4, 40),
+('five', 5, 50),
+('six', 1, 10),
+('six', 2, 20),
+('six', 3, 30),
+('six', 4, 40),
+('six', 5, 50),
+('six', 6, 60),
+-- single-row partition: percent_rank 0.0
+('lonely', 99, 42),
+-- two-row null partition
+(NULL, 1, 100),
+(NULL, 2, 100);
+
+-- NON-VECTORIZED
+set hive.vectorized.execution.ptf.enabled=false;
+
+select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int;
+
+-- VECTORIZED
+set hive.vectorized.execution.ptf.enabled=true;
+
+explain vectorization detail select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int;
+
+explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int;
+
+select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int;
diff --git a/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out b/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out
index 335cc4a4b995..470021889f76 100644
--- a/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out
@@ -4323,7 +4323,7 @@ STAGE PLANS:
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: string), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: float)
diff --git a/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_quotes_1.q.out b/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_quotes_1.q.out
index a2d955298324..309031ac86b1 100644
--- a/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_quotes_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_quotes_1.q.out
@@ -4473,7 +4473,7 @@ STAGE PLANS:
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: string), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: float)
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_percent_rank.q.out b/ql/src/test/results/clientpositive/llap/vector_ptf_percent_rank.q.out
new file mode 100644
index 000000000000..cd3ccd4f5df7
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_percent_rank.q.out
@@ -0,0 +1,1176 @@
+PREHOOK: query: DROP TABLE IF EXISTS vector_ptf_percent_rank_int
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS vector_ptf_percent_rank_int
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE vector_ptf_percent_rank_int(name string, rowindex int, mynumber int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vector_ptf_percent_rank_int
+POSTHOOK: query: CREATE TABLE vector_ptf_percent_rank_int(name string, rowindex int, mynumber int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vector_ptf_percent_rank_int
+PREHOOK: query: INSERT INTO vector_ptf_percent_rank_int values
+('five', 1, 10),
+('five', 2, 20),
+('five', 3, 30),
+('five', 4, 40),
+('five', 5, 50),
+('six', 1, 10),
+('six', 2, 20),
+('six', 3, 30),
+('six', 4, 40),
+('six', 5, 50),
+('six', 6, 60),
+
+('lonely', 99, 42),
+
+(NULL, 1, 100),
+(NULL, 2, 100)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@vector_ptf_percent_rank_int
+POSTHOOK: query: INSERT INTO vector_ptf_percent_rank_int values
+('five', 1, 10),
+('five', 2, 20),
+('five', 3, 30),
+('five', 4, 40),
+('five', 5, 50),
+('six', 1, 10),
+('six', 2, 20),
+('six', 3, 30),
+('six', 4, 40),
+('six', 5, 50),
+('six', 6, 60),
+
+('lonely', 99, 42),
+
+(NULL, 1, 100),
+(NULL, 2, 100)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@vector_ptf_percent_rank_int
+POSTHOOK: Lineage: vector_ptf_percent_rank_int.mynumber SCRIPT []
+POSTHOOK: Lineage: vector_ptf_percent_rank_int.name SCRIPT []
+POSTHOOK: Lineage: vector_ptf_percent_rank_int.rowindex SCRIPT []
+PREHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 0.0
+NULL 1 100 0.0
+five 1 10 0.0
+five 2 20 0.25
+five 3 30 0.5
+five 4 40 0.75
+five 5 50 1.0
+lonely 99 42 0.0
+six 1 10 0.0
+six 2 20 0.2
+six 3 30 0.4
+six 4 40 0.6
+six 5 50 0.8
+six 6 60 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+five 1 10 1 1 0.0
+five 2 20 2 2 0.25
+five 3 30 3 3 0.5
+five 4 40 4 4 0.75
+five 5 50 5 5 1.0
+lonely 99 42 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 2 2 0.2
+six 3 30 3 3 0.4
+six 4 40 4 4 0.6
+six 5 50 5 5 0.8
+six 6 60 6 6 1.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+six 1 10 1 1 0.0
+five 1 10 1 1 0.0
+six 2 20 3 2 0.15384615384615385
+five 2 20 3 2 0.15384615384615385
+six 3 30 5 3 0.3076923076923077
+five 3 30 5 3 0.3076923076923077
+five 4 40 7 4 0.46153846153846156
+six 4 40 7 4 0.46153846153846156
+lonely 99 42 9 5 0.6153846153846154
+five 5 50 10 6 0.6923076923076923
+six 5 50 10 6 0.6923076923076923
+six 6 60 12 7 0.8461538461538461
+NULL 1 100 13 8 0.9230769230769231
+NULL 2 100 13 8 0.9230769230769231
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+six 1 10 1 1 0.0
+six 3 30 1 1 0.0
+six 2 20 1 1 0.0
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 1 1 0.0
+six 3 30 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 1 100 1 1 0.0
+NULL 2 100 1 1 0.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string, 1:rowindex:int, 2:mynumber:int, 3:ROW__ID:struct, 4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: percent_rank_window_0
+ arguments: _col2
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:int]
+ functionNames: [percent_rank]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 0, 2, 1]
+ outputTypes: [double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), percent_rank_window_0 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3]
+ Statistics: Num rows: 14 Data size: 1181 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1181 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 0.0
+NULL 1 100 0.0
+five 1 10 0.0
+five 2 20 0.25
+five 3 30 0.5
+five 4 40 0.75
+five 5 50 1.0
+lonely 99 42 0.0
+six 1 10 0.0
+six 2 20 0.2
+six 3 30 0.4
+six 4 40 0.6
+six 5 50 0.8
+six 6 60 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string, 1:rowindex:int, 2:mynumber:int, 3:ROW__ID:struct, 4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:string, 2:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 0:string
+ valueColumns: 1:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, KEY.reducesinkkey1:int, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col2
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col2
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: _col2
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank, VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:int, col 1:int, col 1:int]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:int]
+ outputColumns: [3, 4, 5, 0, 2, 1]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name order by mynumber) as r,
+dense_rank() over (partition by name order by mynumber) as dr,
+percent_rank() over (partition by name order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+five 1 10 1 1 0.0
+five 2 20 2 2 0.25
+five 3 30 3 3 0.5
+five 4 40 4 4 0.75
+five 5 50 5 5 1.0
+lonely 99 42 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 2 2 0.2
+six 3 30 3 3 0.4
+six 4 40 4 4 0.6
+six 5 50 5 5 0.8
+six 6 60 6 6 1.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string, 1:rowindex:int, 2:mynumber:int, 3:ROW__ID:struct, 4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), mynumber (type: int)
+ null sort order: az
+ sort order: ++
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 5:int, 2:int
+ keyExpressions: ConstantVectorExpression(val 0) -> 5:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 6:int
+ valueColumns: 0:string, 1:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int, KEY.reducesinkkey1:int, VALUE._col0:string, VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: int), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col2 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col2
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col2
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: _col2
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank, VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:int, col 1:int, col 1:int]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: [1]
+ native: true
+ nonKeyInputColumns: [2, 3]
+ orderExpressions: [col 1:int]
+ outputColumns: [4, 5, 6, 2, 3, 1]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0) -> 7:int]
+ streamingColumns: [4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 1, 4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (order by mynumber) as r,
+dense_rank() over (order by mynumber) as dr,
+percent_rank() over (order by mynumber) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+six 1 10 1 1 0.0
+five 1 10 1 1 0.0
+six 2 20 3 2 0.15384615384615385
+five 2 20 3 2 0.15384615384615385
+six 3 30 5 3 0.3076923076923077
+five 3 30 5 3 0.3076923076923077
+five 4 40 7 4 0.46153846153846156
+six 4 40 7 4 0.46153846153846156
+lonely 99 42 9 5 0.6153846153846154
+five 5 50 10 6 0.6923076923076923
+six 5 50 10 6 0.6923076923076923
+six 6 60 12 7 0.8461538461538461
+NULL 1 100 13 8 0.9230769230769231
+NULL 2 100 13 8 0.9230769230769231
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string, 1:rowindex:int, 2:mynumber:int, 3:ROW__ID:struct, 4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: name (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: name (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int, 2:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: rowindex (type: int), mynumber (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, VALUE._col0:int, VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: _col0
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: _col0
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank, VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 0:string, col 0:string, col 0:string]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: [0]
+ native: true
+ nonKeyInputColumns: [1, 2]
+ orderExpressions: [col 0:string]
+ outputColumns: [3, 4, 5, 0, 1, 2]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [col 0:string]
+ streamingColumns: [3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over (partition by name) as r,
+dense_rank() over (partition by name) as dr,
+percent_rank() over (partition by name) as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 2 100 1 1 0.0
+NULL 1 100 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+six 1 10 1 1 0.0
+six 3 30 1 1 0.0
+six 2 20 1 1 0.0
+PREHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization detail select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_percent_rank_int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string, 1:rowindex:int, 2:mynumber:int, 3:ROW__ID:struct, 4:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ keyColumns: 5:int
+ keyExpressions: ConstantVectorExpression(val 0) -> 5:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 0:string, 1:int, 2:int
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: name (type: string), rowindex (type: int), mynumber (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, rowindex:int, mynumber:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int, VALUE._col0:string, VALUE._col1:int, VALUE._col2:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, double, bigint, bigint, bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: int, _col2: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: 0 ASC NULLS FIRST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: 0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_1
+ arguments: 0
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: percent_rank_window_2
+ arguments: 0
+ name: percent_rank
+ window function: GenericUDAFPercentRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRank, VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [ConstantVectorExpression(val 0) -> 9:int, ConstantVectorExpression(val 0) -> 10:int, ConstantVectorExpression(val 0) -> 11:int]
+ functionNames: [rank, dense_rank, percent_rank]
+ keyInputColumns: []
+ native: true
+ nonKeyInputColumns: [1, 2, 3]
+ orderExpressions: [ConstantVectorExpression(val 0) -> 8:int]
+ outputColumns: [4, 5, 6, 1, 2, 3]
+ outputTypes: [int, int, double, string, int, int]
+ partitionExpressions: [ConstantVectorExpression(val 0) -> 7:int]
+ streamingColumns: [4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1243 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), rank_window_0 (type: int), dense_rank_window_1 (type: int), percent_rank_window_2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 3, 4, 5, 6]
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 14 Data size: 1293 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+POSTHOOK: query: select name, rowindex, mynumber,
+rank() over () as r,
+dense_rank() over () as dr,
+percent_rank() over () as pr
+from vector_ptf_percent_rank_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_percent_rank_int
+#### A masked pattern was here ####
+five 1 10 1 1 0.0
+five 2 20 1 1 0.0
+five 3 30 1 1 0.0
+five 4 40 1 1 0.0
+five 5 50 1 1 0.0
+six 1 10 1 1 0.0
+six 2 20 1 1 0.0
+six 3 30 1 1 0.0
+six 4 40 1 1 0.0
+six 5 50 1 1 0.0
+six 6 60 1 1 0.0
+lonely 99 42 1 1 0.0
+NULL 1 100 1 1 0.0
+NULL 2 100 1 1 0.0
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
index 5ba2c7af513e..f0e0c408f607 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
@@ -3250,7 +3250,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in supported functions [avg, count, cume_dist, dense_rank, first_value, lag, last_value, lead, max, min, rank, row_number, sum]
+ notVectorizedReason: PTF operator: ntile not in supported functions [avg, count, cume_dist, dense_rank, first_value, lag, last_value, lead, max, min, percent_rank, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
index c2faf5d03a6d..f27c31ec9e28 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing_gby2.q.out
@@ -792,16 +792,28 @@ STAGE PLANS:
Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: dense_rank_window_1 (type: int), _col0 (type: int)
Reducer 5
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in supported functions [avg, count, cume_dist, dense_rank, first_value, lag, last_value, lead, max, min, rank, row_number, sum]
- vectorized: false
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:int, KEY.reducesinkkey1:double, VALUE._col0:int, VALUE._col1:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double)
outputColumnNames: _col0, _col1, _col6, _col7
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3, 0, 1]
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
PTF Operator
Function definitions:
@@ -823,13 +835,34 @@ STAGE PLANS:
window function: GenericUDAFPercentRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:double]
+ functionNames: [percent_rank]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2, 3]
+ orderExpressions: [col 1:double]
+ outputColumns: [4, 2, 3, 0, 1]
+ outputTypes: [double, int, int, int, double]
+ partitionExpressions: [col 0:int]
+ streamingColumns: [4]
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col1 (type: int), _col0 (type: int), percent_rank_window_2 (type: double)
outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3, 2, 4]
Statistics: Num rows: 10 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 10 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
index 45ab7990da01..692661b8946d 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing_rank.q.out
@@ -904,16 +904,28 @@ STAGE PLANS:
partitionColumnCount: 0
scratchColumnTypeNames: []
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
- notVectorizedReason: PTF operator: percent_rank not in supported functions [avg, count, cume_dist, dense_rank, first_value, lag, last_value, lead, max, min, rank, row_number, sum]
- vectorized: false
+ reduceColumnNullOrder: az
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:decimal(4,2), KEY.reducesinkkey1:float, VALUE._col6:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double]
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: float), VALUE._col6 (type: string), KEY.reducesinkkey0 (type: decimal(4,2))
outputColumnNames: _col4, _col7, _col9
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 2, 0]
Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
@@ -935,16 +947,40 @@ STAGE PLANS:
window function: GenericUDAFPercentRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorPercentRank]
+ functionInputExpressions: [col 1:float]
+ functionNames: [percent_rank]
+ keyInputColumns: [1, 0]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1:float]
+ outputColumns: [3, 1, 2, 0]
+ outputTypes: [double, float, string, decimal(4,2)]
+ partitionExpressions: [col 0:decimal(4,2)]
+ streamingColumns: [3]
Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col7 (type: string), percent_rank_window_0 (type: double)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 3]
Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
diff --git a/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out b/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out
index c7a29126299b..1155317e66ba 100644
--- a/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out
+++ b/ql/src/test/results/clientpositive/llap/windowing_gby2.q.out
@@ -416,7 +416,7 @@ STAGE PLANS:
Statistics: Num rows: 10 Data size: 1005 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: dense_rank_window_1 (type: int), _col0 (type: int)
Reducer 5
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double)