From 171d05096a38169b8f1323736c3bc0ca92ccbd11 Mon Sep 17 00:00:00 2001 From: zzwqqq Date: Sun, 30 Aug 2026 14:49:00 +0800 Subject: [PATCH] [CALCITE-7755] Support IEJoin for inequality joins --- .../adapter/enumerable/EnumerableIEJoin.java | 272 +++++++++++++ .../enumerable/EnumerableIEJoinRule.java | 111 ++++++ .../adapter/enumerable/EnumerableRules.java | 7 + .../org/apache/calcite/tools/Programs.java | 1 + .../apache/calcite/util/BuiltInMethod.java | 5 + .../test/enumerable/EnumerableIEJoinTest.java | 373 ++++++++++++++++++ .../calcite/test/CombineRelOptRulesTest.xml | 4 +- .../calcite/linq4j/EnumerableDefaults.java | 28 ++ .../calcite/linq4j/IEJoinEnumerator.java | 228 +++++++++++ .../calcite/linq4j/InequalityOperator.java | 33 ++ .../calcite/linq4j/test/IEJoinTest.java | 196 +++++++++ 11 files changed, 1256 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoin.java create mode 100644 core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoinRule.java create mode 100644 core/src/test/java/org/apache/calcite/test/enumerable/EnumerableIEJoinTest.java create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/IEJoinEnumerator.java create mode 100644 linq4j/src/main/java/org/apache/calcite/linq4j/InequalityOperator.java create mode 100644 linq4j/src/test/java/org/apache/calcite/linq4j/test/IEJoinTest.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoin.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoin.java new file mode 100644 index 000000000000..692a04df5fcc --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoin.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.linq4j.InequalityOperator; +import org.apache.calcite.linq4j.function.Function1; +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.ParameterExpression; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelNodes; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.RelMdUtil; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.Util; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** Implementation of an inner IEJoin with two inequality predicates in + * {@link EnumerableConvention enumerable calling convention}. */ +public class EnumerableIEJoin extends Join implements EnumerableRel { + private final ImmutableList conditions; + + /** Creates an EnumerableIEJoin. + * + *

Use {@link #create} unless you know what you're doing. */ + protected EnumerableIEJoin(RelOptCluster cluster, RelTraitSet traitSet, + RelNode left, RelNode right, RexNode condition) { + super(cluster, traitSet, ImmutableList.of(), left, right, condition, + ImmutableSet.of(), JoinRelType.INNER); + final List conjunctions = RelOptUtil.conjunctions(condition); + if (conjunctions.size() != 2) { + throw new IllegalArgumentException( + "condition must contain exactly two supported cross-input inequalities"); + } + final int leftFieldCount = left.getRowType().getFieldCount(); + final Condition first = + analyzeConjunction(conjunctions.get(0), leftFieldCount); + final Condition second = + analyzeConjunction(conjunctions.get(1), leftFieldCount); + if (first == null || second == null) { + throw new IllegalArgumentException( + "condition must contain supported cross-input inequalities"); + } + final ImmutableList conditions = ImmutableList.of(first, second); + for (Condition inequality : conditions) { + if (!supportsKeyTypes(left, right, inequality)) { + throw new IllegalArgumentException("unsupported IEJoin key types: left " + + left.getRowType().getFieldList().get(inequality.leftKey).getType() + + ", right " + + right.getRowType().getFieldList().get(inequality.rightKey).getType()); + } + } + this.conditions = conditions; + } + + /** Creates an EnumerableIEJoin. */ + public static EnumerableIEJoin create(RelNode left, RelNode right, + RexNode condition) { + return new EnumerableIEJoin(left.getCluster(), + left.getCluster().traitSetOf(EnumerableConvention.INSTANCE), + left, right, condition); + } + + @Override public EnumerableIEJoin copy(RelTraitSet traitSet, + RexNode condition, RelNode left, RelNode right, JoinRelType joinType, + boolean semiJoinDone) { + if (joinType != JoinRelType.INNER) { + throw new IllegalArgumentException("EnumerableIEJoin only supports inner joins"); + } + return new EnumerableIEJoin(getCluster(), traitSet, left, right, + condition); + } + + @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner, + RelMetadataQuery mq) { + final double leftRows = mq.getRowCount(left); + final double rightRows = mq.getRowCount(right); + double outputRows = mq.getRowCount(this); + if (RelNodes.COMPARATOR.compare(left, right) > 0) { + outputRows = RelMdUtil.addEpsilon(outputRows); + } + final double inputRows = leftRows + rightRows; + // IEJoin sorts the union twice, scans it once, then emits the result. + final double cost = + 2D * Util.nLogN(inputRows) + inputRows + outputRows; + return planner.getCostFactory().makeCost(cost, 0, 0); + } + + @Override public Result implement(EnumerableRelImplementor implementor, + Prefer pref) { + final BlockBuilder builder = new BlockBuilder(); + final Result leftResult = + implementor.visitChild(this, 0, (EnumerableRel) left, pref); + final Expression leftExpression = + builder.append("left", leftResult.block); + final Result rightResult = + implementor.visitChild(this, 1, (EnumerableRel) right, pref); + final Expression rightExpression = + builder.append("right", rightResult.block); + final ParameterExpression leftParameter = + Expressions.parameter(leftResult.physType.getJavaRowType(), "leftRow"); + final ParameterExpression rightParameter = + Expressions.parameter(rightResult.physType.getJavaRowType(), "rightRow"); + final JavaTypeFactory typeFactory = implementor.getTypeFactory(); + final List keySelectors = new ArrayList<>(); + final List comparators = new ArrayList<>(); + + for (Condition condition : conditions) { + final RelDataType leftType = + left.getRowType().getFieldList().get(condition.leftKey).getType(); + final RelDataType rightType = + right.getRowType().getFieldList().get(condition.rightKey).getType(); + final RelDataType keyType = + requireNonNull(typeFactory.leastRestrictive(ImmutableList.of(leftType, rightType))); + final Type keyClass = typeFactory.getJavaClass(keyType); + keySelectors.add( + Expressions.lambda( + Function1.class, + EnumUtils.convert( + leftResult.physType.fieldReference( + leftParameter, condition.leftKey), keyClass), leftParameter)); + keySelectors.add( + Expressions.lambda( + Function1.class, + EnumUtils.convert( + rightResult.physType.fieldReference( + rightParameter, condition.rightKey), keyClass), rightParameter)); + // PhysType generates comparators for row fields, so wrap the key in a + // scalar row type. + final RelDataType keyRowType = + typeFactory.builder().add("key", keyType).build(); + final PhysType keyPhysType = + PhysTypeImpl.of(typeFactory, keyRowType, JavaRowFormat.SCALAR); + comparators.add( + keyPhysType.generateComparator( + RelCollations.of( + new RelFieldCollation(0, + RelFieldCollation.Direction.ASCENDING, + RelFieldCollation.NullDirection.LAST)))); + } + + final PhysType physType = + PhysTypeImpl.of(typeFactory, getRowType(), pref.preferArray()); + final List arguments = new ArrayList<>(); + arguments.add(leftExpression); + arguments.add(rightExpression); + arguments.addAll(keySelectors); + arguments.addAll(comparators); + arguments.add(Expressions.constant(conditions.get(0).operator)); + arguments.add(Expressions.constant(conditions.get(1).operator)); + arguments.add( + EnumUtils.joinSelector(joinType, physType, + ImmutableList.of(leftResult.physType, rightResult.physType))); + + return implementor.result(physType, + builder.append( + Expressions.call(BuiltInMethod.IE_JOIN.method, + arguments)).toBlock()); + } + + static @Nullable Condition analyzeConjunction(RexNode node, + int leftFieldCount) { + if (!(node instanceof RexCall) || ((RexCall) node).operands.size() != 2) { + return null; + } + final RexCall call = (RexCall) node; + if (!(call.operands.get(0) instanceof RexInputRef) + || !(call.operands.get(1) instanceof RexInputRef)) { + return null; + } + final int first = ((RexInputRef) call.operands.get(0)).getIndex(); + final int second = ((RexInputRef) call.operands.get(1)).getIndex(); + final boolean firstIsLeft = first < leftFieldCount; + final boolean secondIsLeft = second < leftFieldCount; + if (firstIsLeft == secondIsLeft) { + return null; + } + final InequalityOperator operator; + switch (firstIsLeft ? call.getKind() : call.getKind().reverse()) { + case LESS_THAN: + operator = InequalityOperator.LESS_THAN; + break; + case LESS_THAN_OR_EQUAL: + operator = InequalityOperator.LESS_THAN_OR_EQUAL; + break; + case GREATER_THAN: + operator = InequalityOperator.GREATER_THAN; + break; + case GREATER_THAN_OR_EQUAL: + operator = InequalityOperator.GREATER_THAN_OR_EQUAL; + break; + default: + return null; + } + return firstIsLeft + ? new Condition(first, second - leftFieldCount, operator) + : new Condition(second, first - leftFieldCount, operator); + } + + static boolean supportsKeyTypes(RelNode left, RelNode right, + Condition condition) { + final RelDataType leftType = + left.getRowType().getFieldList().get(condition.leftKey).getType(); + final RelDataType rightType = + right.getRowType().getFieldList().get(condition.rightKey).getType(); + final SqlTypeName typeName = leftType.getSqlTypeName(); + return SqlTypeUtil.equalSansNullability( + left.getCluster().getTypeFactory(), leftType, rightType) + && (SqlTypeUtil.isBoolean(leftType) + || (SqlTypeUtil.isExactNumeric(leftType) + && !SqlTypeName.UNSIGNED_TYPES.contains(typeName)) + || SqlTypeUtil.isCharacter(leftType) + || SqlTypeUtil.isBinary(leftType) + || SqlTypeUtil.isDatetime(leftType) + || SqlTypeUtil.isInterval(leftType)); + } + + /** A normalized IEJoin condition. */ + static final class Condition { + final int leftKey; + final int rightKey; + final InequalityOperator operator; + + private Condition(int leftKey, int rightKey, + InequalityOperator operator) { + this.leftKey = leftKey; + this.rightKey = rightKey; + this.operator = operator; + } + } +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoinRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoinRule.java new file mode 100644 index 000000000000..5023fc5e5763 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableIEJoinRule.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexUtil; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** Planner rule that converts an inner {@link LogicalJoin} whose condition + * consists of at least two cross-input field inequalities to an + * {@link EnumerableIEJoin}. + * + *

Based on Khayyat et al., + * "Lightning Fast and Space + * Efficient Inequality Joins," PVLDB 8(13), 2015. The first two + * inequalities drive the join and additional inequalities are evaluated by an + * {@link EnumerableCalc}. + * + * @see EnumerableRules#ENUMERABLE_IE_JOIN_RULE + */ +class EnumerableIEJoinRule extends ConverterRule { + /** Default configuration. */ + static final Config DEFAULT_CONFIG = Config.INSTANCE + .withConversion(LogicalJoin.class, Convention.NONE, + EnumerableConvention.INSTANCE, "EnumerableIEJoinRule") + .withRuleFactory(EnumerableIEJoinRule::new); + + /** Called from the Config. */ + protected EnumerableIEJoinRule(Config config) { + super(config); + } + + @Override public @Nullable RelNode convert(RelNode rel) { + final Join join = (Join) rel; + if (join.getJoinType() != JoinRelType.INNER + || !join.getVariablesSet().isEmpty() + || !join.getSystemFieldList().isEmpty()) { + return null; + } + + final int leftFieldCount = join.getLeft().getRowType().getFieldCount(); + final List conjunctions = + RelOptUtil.conjunctions(join.getCondition()); + if (conjunctions.size() < 2) { + return null; + } + for (int i = 0; i < conjunctions.size(); i++) { + final EnumerableIEJoin.Condition condition = + EnumerableIEJoin.analyzeConjunction(conjunctions.get(i), leftFieldCount); + if (condition == null) { + return null; + } + if (i < 2 + && !EnumerableIEJoin.supportsKeyTypes( + join.getLeft(), join.getRight(), condition)) { + return null; + } + } + + final RelNode left = convert(join.getLeft(), join.getLeft().getTraitSet() + .replace(EnumerableConvention.INSTANCE)); + final RelNode right = convert(join.getRight(), join.getRight().getTraitSet() + .replace(EnumerableConvention.INSTANCE)); + final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); + final RexNode ieCondition = + requireNonNull(RexUtil.composeConjunction(rexBuilder, conjunctions.subList(0, 2))); + final EnumerableIEJoin ieJoin = + EnumerableIEJoin.create(left, right, ieCondition); + if (conjunctions.size() == 2) { + return ieJoin; + } + + final RexNode residual = + requireNonNull( + RexUtil.composeConjunction(rexBuilder, + conjunctions.subList(2, conjunctions.size()))); + final RexProgram program = + RexProgram.create(ieJoin.getRowType(), + rexBuilder.identityProjects(ieJoin.getRowType()), residual, + ieJoin.getRowType(), rexBuilder); + return EnumerableCalc.create(ieJoin, program); + } +} diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java index a470d3b7cee0..5b20e665f4ce 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRules.java @@ -61,6 +61,12 @@ private EnumerableRules() { EnumerableMergeJoinRule.DEFAULT_CONFIG .toRule(EnumerableMergeJoinRule.class); + /** Rule that converts a compatible + * {@link org.apache.calcite.rel.logical.LogicalJoin} to an + * {@link EnumerableIEJoin}. */ + public static final RelOptRule ENUMERABLE_IE_JOIN_RULE = + EnumerableIEJoinRule.DEFAULT_CONFIG.toRule(EnumerableIEJoinRule.class); + public static final RelOptRule ENUMERABLE_CORRELATE_RULE = EnumerableCorrelateRule.DEFAULT_CONFIG .toRule(EnumerableCorrelateRule.class); @@ -225,6 +231,7 @@ private EnumerableRules() { ImmutableList.of(EnumerableRules.ENUMERABLE_JOIN_RULE, EnumerableRules.ENUMERABLE_ASOFJOIN_RULE, EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE, + EnumerableRules.ENUMERABLE_IE_JOIN_RULE, EnumerableRules.ENUMERABLE_CORRELATE_RULE, EnumerableRules.ENUMERABLE_CONDITIONAL_CORRELATE_RULE, EnumerableRules.ENUMERABLE_PROJECT_RULE, diff --git a/core/src/main/java/org/apache/calcite/tools/Programs.java b/core/src/main/java/org/apache/calcite/tools/Programs.java index 1307e70100cb..27f6850be615 100644 --- a/core/src/main/java/org/apache/calcite/tools/Programs.java +++ b/core/src/main/java/org/apache/calcite/tools/Programs.java @@ -81,6 +81,7 @@ public class Programs { EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE, EnumerableRules.ENUMERABLE_JOIN_RULE, EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE, + EnumerableRules.ENUMERABLE_IE_JOIN_RULE, EnumerableRules.ENUMERABLE_CORRELATE_RULE, EnumerableRules.ENUMERABLE_CONDITIONAL_CORRELATE_RULE, EnumerableRules.ENUMERABLE_PROJECT_RULE, diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 948b62f28816..9fba4c2c7638 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -38,6 +38,7 @@ import org.apache.calcite.linq4j.EnumerableDefaults; import org.apache.calcite.linq4j.Enumerator; import org.apache.calcite.linq4j.ExtendedEnumerable; +import org.apache.calcite.linq4j.InequalityOperator; import org.apache.calcite.linq4j.JoinType; import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.linq4j.MemoryFactory; @@ -255,6 +256,10 @@ public enum BuiltInMethod { MERGE_JOIN(EnumerableDefaults.class, "mergeJoin", Enumerable.class, Enumerable.class, Function1.class, Function1.class, Predicate2.class, Function2.class, JoinType.class, Comparator.class, EqualityComparer.class), + IE_JOIN(EnumerableDefaults.class, "ieJoin", Enumerable.class, + Enumerable.class, Function1.class, Function1.class, Function1.class, + Function1.class, Comparator.class, Comparator.class, + InequalityOperator.class, InequalityOperator.class, Function2.class), SLICE0(Enumerables.class, "slice0", Enumerable.class), SEMI_JOIN(EnumerableDefaults.class, "semiJoin", Enumerable.class, Enumerable.class, Function1.class, Function1.class, diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableIEJoinTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableIEJoinTest.java new file mode 100644 index 000000000000..136cbf9b8702 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableIEJoinTest.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test.enumerable; + +import org.apache.calcite.adapter.enumerable.EnumerableIEJoin; +import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.adapter.java.ReflectiveSchema; +import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.config.Lex; +import org.apache.calcite.jdbc.JavaCollation; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.runtime.Hook; +import org.apache.calcite.sql.SqlCollation; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.test.schemata.hr.HrSchema; +import org.apache.calcite.test.schemata.hr.HrSchemaBig; +import org.apache.calcite.util.Holder; +import org.apache.calcite.util.Util; + +import com.google.common.collect.ImmutableList; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.jupiter.api.Test; + +import java.text.Collator; +import java.util.Locale; +import java.util.function.Consumer; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import static java.util.Objects.requireNonNull; + +/** Unit tests for {@link EnumerableIEJoin}. */ +class EnumerableIEJoinTest { + private static final SqlCollation PRIMARY_COLLATION = + new JavaCollation(SqlCollation.Coercibility.IMPLICIT, Locale.US, + Util.getDefaultCharset(), Collator.PRIMARY); + + @Test void ieJoin() { + final Holder<@Nullable RelRoot> root = Holder.empty(); + tester(new HrSchema()) + .withRel(builder -> { + builder + .values(new String[]{"lx", "ly"}, + 2, 8, + 5, 4, + null, 3) + .values(new String[]{"rx", "ry"}, + 4, 3, + 7, 6, + 5, 4) + .join(JoinRelType.INNER, + builder.and( + builder.lessThan( + builder.field(2, 1, "rx"), + builder.field(2, 0, "lx")), + builder.greaterThan( + builder.field(2, 0, "ly"), + builder.field(2, 1, "ry")))); + return builder.build(); + }) + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE)) + .withHook(Hook.PLAN_BEFORE_IMPLEMENTATION, + (Consumer) root::set) + .explainHookMatches( + "EnumerableIEJoin(condition=[AND(<($2, $0), >($1, $3))], joinType=[inner])\n" + + " EnumerableValues(tuples=[[{ 2, 8 }, { 5, 4 }, { null, 3 }]])\n" + + " EnumerableValues(tuples=[[{ 4, 3 }, { 7, 6 }, { 5, 4 }]])\n") + .returnsUnordered("lx=5; ly=4; rx=4; ry=3"); + + final EnumerableIEJoin join = + (EnumerableIEJoin) requireNonNull(root.get()).rel; + assertThrows(IllegalArgumentException.class, + () -> join.copy(join.getTraitSet(), join.getCondition(), + join.getLeft(), join.getRight(), JoinRelType.LEFT, false)); + assertThrows(IllegalArgumentException.class, + () -> join.copy(join.getTraitSet(), + join.getCluster().getRexBuilder().makeLiteral(true), + join.getLeft(), join.getRight(), JoinRelType.INNER, false)); + } + + @Test void ieJoinDoesNotSupportApproximateNumbers() { + final Holder<@Nullable RelRoot> root = Holder.empty(); + tester(new TestSchema()) + .query("select l.name as left_name, r.name as right_name " + + "from lefts l join rights r " + + "on l.x >= r.x and l.y > r.y") + .withHook(Hook.PLAN_BEFORE_IMPLEMENTATION, + (Consumer) root::set) + .explainHookMatches( + allOf( + containsString("EnumerableNestedLoopJoin"), + not(containsString("EnumerableIEJoin")))) + .returnsUnordered( + "left_name=minusZero; right_name=zero", + "left_name=plusZero; right_name=zero"); + final Join join = findJoin(requireNonNull(root.get()).rel); + assertThat( + assertThrows(IllegalArgumentException.class, + () -> EnumerableIEJoin.create( + join.getLeft(), join.getRight(), join.getCondition())) + .getMessage(), + allOf(containsString("left JavaType(double)"), + containsString("right JavaType(double)"))); + } + + @Test void ieJoinDoesNotSupportAny() { + final Holder<@Nullable RelNode> root = Holder.empty(); + tester(new TestSchema()) + .query("select l.name as left_name, r.name as right_name " + + "from anyLefts l join anyRights r " + + "on l.x < r.x and l.y > r.y") + .withHook(Hook.TRIMMED, (Consumer) root::set) + .explainContains("EnumerableNestedLoopJoin"); + final Join join = findJoin(requireNonNull(root.get())); + assertThrows(IllegalArgumentException.class, + () -> EnumerableIEJoin.create( + join.getLeft(), join.getRight(), join.getCondition())); + } + + @Test void ieJoinCostAndDefaultSelection() { + final Holder<@Nullable RelRoot> root = Holder.empty(); + tester(new HrSchemaBig()) + .query("select count(*) from emps l join emps r " + + "on l.empid < r.empid and l.deptno >= r.deptno") + .withHook(Hook.PLAN_BEFORE_IMPLEMENTATION, + (Consumer) root::set) + .explainHookMatches(containsString("EnumerableIEJoin")) + .returns("EXPR$0=600\n"); + + final Join join = findJoin(requireNonNull(root.get()).rel); + final EnumerableIEJoin selfJoin = + EnumerableIEJoin.create(join.getLeft(), join.getLeft(), + join.getCondition()); + final RelMetadataQuery mq = new RelMetadataQuery() { + @Override public Double getRowCount(RelNode rel) { + return rel == selfJoin ? 3D : 10D; + } + }; + assertThat( + requireNonNull( + selfJoin.computeSelfCost(selfJoin.getCluster().getPlanner(), mq)) + .getRows(), + is(2D * Util.nLogN(20D) + 23D)); + } + + @Test void ieJoinWithBooleanAndDecimalKeys() { + tester(new HrSchema()) + .query("select l.id as left_id, r.id as right_id " + + "from (values (1, true, cast(2 as decimal(5, 2))), " + + "(2, false, cast(4 as decimal(5, 2)))) as l(id, b, d) " + + "join (values (3, false, cast(3 as decimal(5, 2))), " + + "(4, true, cast(1 as decimal(5, 2)))) as r(id, b, d) " + + "on l.b > r.b and l.d < r.d") + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE)) + .explainHookMatches(containsString("EnumerableIEJoin")) + .returnsUnordered("left_id=1; right_id=3"); + } + + @Test void ieJoinWithBinaryAndNullableDateKeys() { + tester(new HrSchema()) + .query("select l.id as left_id, r.id as right_id " + + "from (values (1, cast(X'01' as varbinary(2)), date '2020-01-02'), " + + "(2, cast(X'01' as varbinary(2)), cast(null as date))) as l(id, b, d) " + + "join (values (3, cast(X'02' as varbinary(2)), date '2020-01-01'), " + + "(4, cast(X'00' as varbinary(2)), date '2020-01-03')) " + + "as r(id, b, d) on l.b < r.b and l.d > r.d") + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE)) + .explainHookMatches(containsString("EnumerableIEJoin")) + .returnsUnordered("left_id=1; right_id=3"); + } + + @Test void ieJoinWithIntervalKeys() { + tester(new HrSchema()) + .query("select l.id as left_id, r.id as right_id " + + "from (values (1, interval '2' day, interval '4' hour), " + + "(2, interval '4' day, interval '1' hour)) " + + "as l(id, d, h) " + + "join (values (3, interval '3' day, interval '2' hour), " + + "(4, interval '1' day, interval '5' hour)) " + + "as r(id, d, h) on l.d < r.d and l.h > r.h") + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE)) + .explainHookMatches(containsString("EnumerableIEJoin")) + .returnsUnordered("left_id=1; right_id=3"); + } + + @Test void ieJoinWithCollatedVarcharKeys() { + tester(new HrSchema()) + .withRel(builder -> { + final RelDataType stringType = + builder.getTypeFactory().createTypeWithCharsetAndCollation( + builder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), + builder.getTypeFactory().getDefaultCharset(), PRIMARY_COLLATION); + final RelDataType rowType = builder.getTypeFactory().builder() + .add("s", stringType) + .add("n", SqlTypeName.INTEGER) + .build(); + return builder + .values(rowType, "abc", 2, "z", 1).as("l") + .values(rowType, "ÀBC", 1, "b", 3).as("r") + .join(JoinRelType.INNER, + builder.and( + builder.greaterThanOrEqual( + builder.field(2, "l", "s"), + builder.field(2, "r", "s")), + builder.greaterThan( + builder.field(2, "l", "n"), + builder.field(2, "r", "n")))) + .project(builder.field("l", "s"), builder.field("r", "s")) + .build(); + }) + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE)) + .explainHookMatches(containsString("EnumerableIEJoin")) + .returnsUnordered("s=abc; s0=ÀBC"); + } + + @Test void ieJoinWithApproximateRemainingPredicate() { + tester(new TestSchema()) + .query("select l.name as left_name, r.name as right_name " + + "from residualLefts l join residualRights r " + + "on l.x < r.x and l.y > r.y and l.z >= r.z") + .withHook(Hook.PLANNER, (Consumer) planner -> + planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE)) + .explainHookMatches( + allOf( + containsString("EnumerableIEJoin"), + containsString(">=($t3, $t7)"))) + .returnsUnordered( + "left_name=minusZero; right_name=zero", + "left_name=plusZero; right_name=zero"); + } + + @Test void ieJoinUnsupportedConditionsUseExistingRules() { + for (String condition : ImmutableList.of( + "e.empid < d.deptno", + "e.empid < d.deptno and e.deptno > d.deptno " + + "and e.empid = d.deptno", + "e.empid + 1 < d.deptno and e.deptno > d.deptno", + "e.empid < d.deptno or e.deptno > d.deptno", + "e.empid < e.deptno and e.deptno > d.deptno")) { + tester(new HrSchema()) + .query("select * from emps e join depts d on " + condition) + .explainHookMatches(not(containsString("EnumerableIEJoin"))) + .runs(); + } + tester(new HrSchema()) + .query("select * from emps e left join depts d " + + "on e.empid < d.deptno and e.deptno > d.deptno") + .explainHookMatches(not(containsString("EnumerableIEJoin"))) + .runs(); + } + + private CalciteAssert.AssertThat tester(Object schema) { + return CalciteAssert.that() + .with(CalciteConnectionProperty.LEX, Lex.JAVA) + .with(CalciteConnectionProperty.FORCE_DECORRELATE, false) + .withSchema("s", new ReflectiveSchema(schema)); + } + + private static Join findJoin(RelNode rel) { + final Holder<@Nullable Join> join = Holder.empty(); + new RelVisitor() { + @Override public void visit(RelNode node, int ordinal, + @Nullable RelNode parent) { + if (node instanceof Join) { + join.set((Join) node); + } else { + super.visit(node, ordinal, parent); + } + } + }.go(rel); + return requireNonNull(join.get()); + } + + /** Test schema for unsupported and residual key types. */ + public static class TestSchema { + public final ApproximatePoint[] lefts = { + new ApproximatePoint("nan", Double.NaN, 1), + new ApproximatePoint("minusZero", -0D, 1), + new ApproximatePoint("plusZero", 0D, 1) + }; + public final ApproximatePoint[] rights = { + new ApproximatePoint("zero", 0D, 0) + }; + public final AnyPoint[] anyLefts = { + new AnyPoint("one", 1, 2) + }; + public final AnyPoint[] anyRights = { + new AnyPoint("two", 2, 1) + }; + public final ResidualPoint[] residualLefts = { + new ResidualPoint("nan", 2, 8, Double.NaN), + new ResidualPoint("minusZero", 2, 8, -0D), + new ResidualPoint("plusZero", 2, 8, 0D) + }; + public final ResidualPoint[] residualRights = { + new ResidualPoint("zero", 4, 3, 0D) + }; + } + + /** Row with an approximate key. */ + public static class ApproximatePoint { + public final String name; + public final double x; + public final int y; + + ApproximatePoint(String name, double x, int y) { + this.name = name; + this.x = x; + this.y = y; + } + } + + /** Row with dynamically typed keys. */ + public static class AnyPoint { + public final String name; + public final Object x; + public final Object y; + + AnyPoint(String name, Object x, Object y) { + this.name = name; + this.x = x; + this.y = y; + } + } + + /** Row with an approximate residual key. */ + public static class ResidualPoint { + public final String name; + public final int x; + public final int y; + public final double z; + + ResidualPoint(String name, int x, int y, double z) { + this.name = name; + this.x = x; + this.y = y; + this.z = z; + } + } +} diff --git a/core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml index 1ae0496621cc..fc1ab908f867 100644 --- a/core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/CombineRelOptRulesTest.xml @@ -110,7 +110,7 @@ EnumerableCombine EnumerableTableScan(table=[[scott, DEPT]]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableProject(ENAME=[$1], GRADE=[$8]) - EnumerableNestedLoopJoin(condition=[AND(>=($5, $9), <=($5, $10))], joinType=[inner]) + EnumerableIEJoin(condition=[AND(>=($5, $9), <=($5, $10))], joinType=[inner]) EnumerableTableScan(table=[[scott, EMP]]) EnumerableTableScan(table=[[scott, SALGRADE]]) ]]> @@ -180,7 +180,7 @@ Combine EnumerableCombine EnumerableAggregate(group=[{9, 11}], EMP_COUNT=[COUNT()]) EnumerableTableSpool(readType=[LAZY], writeType=[LAZY], table=[[TEMP, spool_0]]) - EnumerableNestedLoopJoin(condition=[AND(>=($5, $12), <=($5, $13))], joinType=[inner]) + EnumerableIEJoin(condition=[AND(>=($5, $12), <=($5, $13))], joinType=[inner]) EnumerableProject(EMPNO=[$3], ENAME=[$4], JOB=[$5], MGR=[$6], HIREDATE=[$7], SAL=[$8], COMM=[$9], DEPTNO=[$10], DEPTNO0=[$0], DNAME=[$1], LOC=[$2]) EnumerableHashJoin(condition=[=($0, $10)], joinType=[inner]) EnumerableTableScan(table=[[scott, DEPT]]) diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 0eca4b6d5283..863c0524642b 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -2533,6 +2533,34 @@ private static Enumerable semiEquiJoin_( }; } + /** + * Joins pairs that satisfy two inequality predicates. + * + *

Each operator compares a left key with a right key using the + * corresponding comparator. Null keys do not match. Both inputs are + * materialized and sorted before the first result is returned. + */ + public static + Enumerable ieJoin(Enumerable left, + Enumerable right, + Function1 leftKeySelector1, + Function1 rightKeySelector1, + Function1 leftKeySelector2, + Function1 rightKeySelector2, + Comparator comparator1, + Comparator comparator2, + InequalityOperator operator1, InequalityOperator operator2, + Function2 resultSelector) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + return new IEJoinEnumerator<>(left, right, + leftKeySelector1, rightKeySelector1, + leftKeySelector2, rightKeySelector2, + comparator1, comparator2, operator1, operator2, resultSelector); + } + }; + } + /** * Correlates the elements of two sequences based on a predicate. */ diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/IEJoinEnumerator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/IEJoinEnumerator.java new file mode 100644 index 000000000000..532f5b829d4c --- /dev/null +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/IEJoinEnumerator.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.linq4j; + +import org.apache.calcite.linq4j.function.Function1; +import org.apache.calcite.linq4j.function.Function2; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Comparator; +import java.util.List; +import java.util.NoSuchElementException; + +import static org.apache.calcite.linq4j.Nullness.castNonNull; + +/** Enumerator that implements IEJoin for two inequality predicates. + * + *

Implements the union-array algorithm described by Khayyat et al. in + * "Lightning Fast and Space + * Efficient Inequality Joins," PVLDB 8(13), 2015. + * + *

In Section 4.2, {@code firstOrder}, temporary {@code secondOrder}, + * {@code permutation}, and {@code activeRights} correspond to merged + * {@code L1/L1'}, merged {@code L2/L2'}, merged {@code P/P'}, and extended + * {@code B'}, respectively. {@code Entry.isLeft} records the source input. + * + * @param Left row type + * @param Right row type + * @param First key type + * @param Second key type + * @param Result row type + */ +final class IEJoinEnumerator + implements Enumerator { + private final List leftRows = new ArrayList<>(); + private final List rightRows = new ArrayList<>(); + private final List> firstOrder; + private int[] permutation; + private final BitSet activeRights = new BitSet(); + private final Function2 resultSelector; + + private int secondPosition; + private int nextBit; + private @Nullable Entry currentLeft; + private @Nullable TResult current; + private boolean hasCurrent; + + IEJoinEnumerator(Enumerable left, Enumerable right, + Function1 leftKeySelector1, + Function1 rightKeySelector1, + Function1 leftKeySelector2, + Function1 rightKeySelector2, + Comparator comparator1, + Comparator comparator2, + InequalityOperator operator1, InequalityOperator operator2, + Function2 resultSelector) { + this.resultSelector = resultSelector; + + final List> entries = new ArrayList<>(); + try (Enumerator enumerator = left.enumerator()) { + while (enumerator.moveNext()) { + final TLeft row = enumerator.current(); + final @Nullable TKey1 key1 = leftKeySelector1.apply(row); + final @Nullable TKey2 key2 = leftKeySelector2.apply(row); + if (key1 != null && key2 != null) { + final int rowIndex = leftRows.size(); + leftRows.add(row); + entries.add(new Entry<>(true, rowIndex, key1, key2)); + } + } + } + try (Enumerator enumerator = right.enumerator()) { + while (enumerator.moveNext()) { + final TRight row = enumerator.current(); + final @Nullable TKey1 key1 = rightKeySelector1.apply(row); + final @Nullable TKey2 key2 = rightKeySelector2.apply(row); + if (key1 != null && key2 != null) { + final int rowIndex = rightRows.size(); + rightRows.add(row); + entries.add(new Entry<>(false, rowIndex, key1, key2)); + } + } + } + + firstOrder = new ArrayList<>(entries); + firstOrder.sort( + entryComparator(comparator1, operator1, true)); + for (int i = 0; i < firstOrder.size(); i++) { + firstOrder.get(i).firstPosition = i; + } + + final List> secondOrder = new ArrayList<>(entries); + secondOrder.sort( + entryComparator(comparator2, operator2, false)); + permutation = new int[secondOrder.size()]; + for (int i = 0; i < secondOrder.size(); i++) { + permutation[i] = secondOrder.get(i).firstPosition; + } + } + + @SuppressWarnings("unchecked") + private static Comparator> + entryComparator(Comparator comparator, + InequalityOperator operator, boolean isFirstOrder) { + final boolean greaterThan = + operator == InequalityOperator.GREATER_THAN + || operator == InequalityOperator.GREATER_THAN_OR_EQUAL; + final boolean descending = + isFirstOrder ? greaterThan : !greaterThan; + // Equal right keys follow a left entry in firstOrder and precede it in + // secondOrder only for non-strict operators. + final boolean strict = + operator == InequalityOperator.LESS_THAN + || operator == InequalityOperator.GREATER_THAN; + final boolean leftSideFirst = isFirstOrder != strict; + return (entry1, entry2) -> { + final TKey key1 = isFirstOrder + ? castNonNull((TKey) entry1.key1) + : castNonNull((TKey) entry1.key2); + final TKey key2 = isFirstOrder + ? castNonNull((TKey) entry2.key1) + : castNonNull((TKey) entry2.key2); + final int c = descending + ? comparator.compare(key2, key1) + : comparator.compare(key1, key2); + if (c != 0 || entry1.isLeft == entry2.isLeft) { + return c; + } + + return entry1.isLeft == leftSideFirst ? -1 : 1; + }; + } + + @Override public TResult current() { + if (!hasCurrent) { + throw new NoSuchElementException(); + } + return castNonNull(current); + } + + @Override public boolean moveNext() { + hasCurrent = false; + while (true) { + // Rights already seen in secondOrder satisfy predicate 2; active bits + // after currentLeft's position in firstOrder also satisfy predicate 1. + if (currentLeft != null) { + final int bit = activeRights.nextSetBit(nextBit); + if (bit >= 0) { + nextBit = bit + 1; + final Entry right = firstOrder.get(bit); + current = + resultSelector.apply(leftRows.get(currentLeft.rowIndex), + rightRows.get(right.rowIndex)); + hasCurrent = true; + return true; + } + currentLeft = null; + } + + if (secondPosition >= permutation.length) { + current = null; + return false; + } + + final int firstPosition = permutation[secondPosition++]; + final Entry entry = firstOrder.get(firstPosition); + if (entry.isLeft) { + currentLeft = entry; + nextBit = firstPosition + 1; + } else { + activeRights.set(firstPosition); + } + } + } + + @Override public void reset() { + activeRights.clear(); + secondPosition = 0; + nextBit = 0; + currentLeft = null; + current = null; + hasCurrent = false; + } + + @Override public void close() { + reset(); + leftRows.clear(); + rightRows.clear(); + firstOrder.clear(); + permutation = new int[0]; + } + + /** Row entry shared by the two sorted orders. + * + * @param First key type + * @param Second key type + */ + private static final class Entry { + final boolean isLeft; + final int rowIndex; + final TKey1 key1; + final TKey2 key2; + int firstPosition; + + private Entry(boolean isLeft, int rowIndex, TKey1 key1, TKey2 key2) { + this.isLeft = isLeft; + this.rowIndex = rowIndex; + this.key1 = key1; + this.key2 = key2; + } + } +} diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/InequalityOperator.java b/linq4j/src/main/java/org/apache/calcite/linq4j/InequalityOperator.java new file mode 100644 index 000000000000..df82c0b66734 --- /dev/null +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/InequalityOperator.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.linq4j; + +/** Comparison operator applied to the left and right keys of an inequality + * join. */ +public enum InequalityOperator { + /** Left key is less than right key. */ + LESS_THAN, + + /** Left key is less than or equal to right key. */ + LESS_THAN_OR_EQUAL, + + /** Left key is greater than right key. */ + GREATER_THAN, + + /** Left key is greater than or equal to right key. */ + GREATER_THAN_OR_EQUAL +} diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/IEJoinTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/IEJoinTest.java new file mode 100644 index 000000000000..5c4c4fc4b582 --- /dev/null +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/IEJoinTest.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.linq4j.test; + +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.EnumerableDefaults; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.InequalityOperator; +import org.apache.calcite.linq4j.Linq4j; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Random; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** Tests for IEJoin. */ +class IEJoinTest { + private static final List LEFT = + Arrays.asList(new Point("l0", 2, 8), + new Point("l1", 5, 4), + new Point("l2", 5, 4), + new Point("lnx", null, 3), + new Point("lny", 3, null)); + + private static final List RIGHT = + Arrays.asList(new Point("r0", 4, 3), + new Point("r1", 7, 6), + new Point("r2", 5, 4), + new Point("rnx", null, 2), + new Point("rny", 6, null)); + + @Test void testAllOperatorCombinationsAgainstNestedLoop() { + for (InequalityOperator operator1 : InequalityOperator.values()) { + for (InequalityOperator operator2 : InequalityOperator.values()) { + assertMatches("fixed input", LEFT, RIGHT, operator1, operator2); + } + } + } + + @Test void testRandomInputsAgainstNestedLoop() { + final Random random = new Random(0); + for (int trial = 0; trial < 200; trial++) { + final List left = new ArrayList<>(); + final List right = new ArrayList<>(); + final int leftCount = random.nextInt(9); + final int rightCount = random.nextInt(9); + for (int i = 0; i < leftCount; i++) { + left.add( + new Point("l" + i, + random.nextInt(5) == 0 ? null : random.nextInt(7) - 3, + random.nextInt(5) == 0 ? null : random.nextInt(7) - 3)); + } + for (int i = 0; i < rightCount; i++) { + right.add( + new Point("r" + i, + random.nextInt(5) == 0 ? null : random.nextInt(7) - 3, + random.nextInt(5) == 0 ? null : random.nextInt(7) - 3)); + } + for (InequalityOperator operator1 : InequalityOperator.values()) { + for (InequalityOperator operator2 : InequalityOperator.values()) { + assertMatches("trial " + trial, left, right, operator1, operator2); + } + } + } + } + + @Test void testEmptyAndSameInputs() { + assertMatches("empty left", Collections.emptyList(), RIGHT, + InequalityOperator.LESS_THAN, InequalityOperator.GREATER_THAN); + assertMatches("empty right", LEFT, Collections.emptyList(), + InequalityOperator.LESS_THAN, InequalityOperator.GREATER_THAN); + assertMatches("same input", LEFT, LEFT, + InequalityOperator.LESS_THAN_OR_EQUAL, + InequalityOperator.GREATER_THAN_OR_EQUAL); + } + + @Test void testReset() { + final Enumerable join = + ieJoin(LEFT, RIGHT, InequalityOperator.LESS_THAN, + InequalityOperator.GREATER_THAN); + final List first = new ArrayList<>(); + final List second = new ArrayList<>(); + try (Enumerator enumerator = join.enumerator()) { + assertThrows(NoSuchElementException.class, enumerator::current); + while (enumerator.moveNext()) { + first.add(enumerator.current()); + } + assertThrows(NoSuchElementException.class, enumerator::current); + enumerator.reset(); + assertThrows(NoSuchElementException.class, enumerator::current); + while (enumerator.moveNext()) { + second.add(enumerator.current()); + } + } + assertThat(second, is(first)); + assertThat(join.toList(), is(first)); + } + + private static void assertMatches(String context, List left, + List right, InequalityOperator operator1, + InequalityOperator operator2) { + final List expected = nestedLoop(left, right, operator1, operator2); + final List actual = ieJoin(left, right, operator1, operator2).toList(); + Collections.sort(actual); + assertThat(context + ", " + operator1 + "/" + operator2 + + ", left=" + left + ", right=" + right, + actual, is(expected)); + } + + private static Enumerable ieJoin(List left, List right, + InequalityOperator operator1, InequalityOperator operator2) { + return EnumerableDefaults.ieJoin( + Linq4j.asEnumerable(left), Linq4j.asEnumerable(right), + point -> point.x, point -> point.x, + point -> point.y, point -> point.y, + Comparator.naturalOrder(), Comparator.naturalOrder(), + operator1, operator2, + (leftPoint, rightPoint) -> leftPoint.name + ":" + rightPoint.name); + } + + private static List nestedLoop(List leftRows, + List rightRows, InequalityOperator operator1, + InequalityOperator operator2) { + final List result = new ArrayList<>(); + for (Point left : leftRows) { + for (Point right : rightRows) { + if (test(left.x, right.x, operator1) + && test(left.y, right.y, operator2)) { + result.add(left.name + ":" + right.name); + } + } + } + Collections.sort(result); + return result; + } + + private static boolean test(@Nullable Integer left, @Nullable Integer right, + InequalityOperator operator) { + if (left == null || right == null) { + return false; + } + switch (operator) { + case LESS_THAN: + return left < right; + case LESS_THAN_OR_EQUAL: + return left <= right; + case GREATER_THAN: + return left > right; + case GREATER_THAN_OR_EQUAL: + return left >= right; + default: + throw new AssertionError(operator); + } + } + + /** Test row. */ + private static final class Point { + final String name; + final @Nullable Integer x; + final @Nullable Integer y; + + private Point(String name, @Nullable Integer x, @Nullable Integer y) { + this.name = name; + this.x = x; + this.y = y; + } + + @Override public String toString() { + return name + "(" + x + ", " + y + ")"; + } + } +}