From d619e5e673ddba6fca05491f0e2dcae557c83656 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Oct 2026 15:58:44 +0100 Subject: [PATCH 1/2] test: move the float semantics sweep to SQL file tests CometFloatSemanticsSuite expanded to 412 runtime tests from one test(...) call, and its cases only checked the answer. Move the cases that match Spark into sql-tests/float_semantics/*.sql, where a plain query also asserts that the plan runs natively. Closes #6636. --- .../contributor-guide/floating_point.md | 14 +- .../sql-tests/float_semantics/aggregates.sql | 194 ++++ .../aggregates_collect_set_4_2.sql | 77 ++ .../sql-tests/float_semantics/comparison.sql | 539 ++++++++++ .../float_semantics/comparison_joins.sql | 338 ++++++ .../float_semantics/comparison_nested.sql | 307 ++++++ .../sql-tests/float_semantics/expressions.sql | 414 ++++++++ .../sql-tests/float_semantics/keys.sql | 263 +++++ .../float_semantics/literal_comparison.sql | 978 ++++++++++++++++++ .../comet/CometFloatSemanticsSuite.scala | 305 +----- 10 files changed, 3154 insertions(+), 275 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/float_semantics/aggregates.sql create mode 100644 spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql create mode 100644 spark/src/test/resources/sql-tests/float_semantics/comparison.sql create mode 100644 spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql create mode 100644 spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql create mode 100644 spark/src/test/resources/sql-tests/float_semantics/expressions.sql create mode 100644 spark/src/test/resources/sql-tests/float_semantics/keys.sql create mode 100644 spark/src/test/resources/sql-tests/float_semantics/literal_comparison.sql diff --git a/docs/source/contributor-guide/floating_point.md b/docs/source/contributor-guide/floating_point.md index 758350db8f1..a402342ffc6 100644 --- a/docs/source/contributor-guide/floating_point.md +++ b/docs/source/contributor-guide/floating_point.md @@ -201,11 +201,15 @@ and only for a type that contains a `FLOAT` or `DOUBLE`. `MaxSparkVersion`, and add the `run-all-spark-profiles` label to the pull request. CI pins one patch release per Spark line, so when a patch release changed the rule, also test the routing in Scala. -- Add the expression to `CometFloatSemanticsSuite`, which crosses the edge values with operator - contexts and with every expression and aggregate that accepts floating-point input. New - expressions go in its `expressions` or `aggregates` list. If one does not match Spark yet, add a - `KnownGap` that links the issue. Known gaps are strict: a case that starts matching Spark fails, - so the fix also removes the entry. +- Add the expression to the sweep in `float_semantics/`, which crosses the edge values with + operator contexts and with every expression and aggregate that accepts floating-point input. + `comparison.sql`, `comparison_joins.sql` and `comparison_nested.sql` cover comparisons, + `literal_comparison.sql` compares a column with literals, and `keys.sql` covers grouping, join, + sort and window keys. A new expression goes in `expressions.sql`, or in `aggregates.sql` for an + aggregate, with one query for `DOUBLE` and one for `FLOAT`. If one does not match Spark yet, + mark its queries `ignore()`. A SQL file test cannot expect a difference, so a gap that + should fail once it is fixed goes in `CometFloatSemanticsSuite`, whose test for `collect_set` + before Spark 4.2 fails when the answers start to match. Fixtures to start from include `expressions/conditional/float_comparisons.sql`, `expressions/math/nan_divisor.sql`, `expressions/math/greatest_least_floating_point.sql`, diff --git a/spark/src/test/resources/sql-tests/float_semantics/aggregates.sql b/spark/src/test/resources/sql-tests/float_semantics/aggregates.sql new file mode 100644 index 00000000000..17a7509298a --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/aggregates.sql @@ -0,0 +1,194 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Aggregate functions over float input. Each pair of edge values is expanded from one +-- input row, so its two values reach the same partial aggregate in array order. Window +-- tests also order by pos to preserve signed-zero ties after a shuffle. mode gets each pair +-- as (l, r, r), so the mode is unique under every Spark equality rule. +-- +-- The e and p CTEs negate the flagged stored values to make the sign-bit NaN at query time +-- (see comparison.sql). The sweep tracks #6385. +-- +-- collect_set is in aggregates_collect_set_4_2.sql. + +-- ConstantFolding: enabled + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_double USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_double a CROSS JOIN fs_edge_double b + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_float USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_float a CROSS JOIN fs_edge_float b + +-- max, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- max, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- min, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, min(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- min, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, min(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- max_by, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max_by(pos, x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- max_by, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max_by(pos, x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- min_by, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, min_by(pos, x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- min_by, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, min_by(pos, x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- sum, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, sum(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- sum, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, sum(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- avg, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, avg(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- avg, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, avg(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- count(DISTINCT), DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(DISTINCT x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- count(DISTINCT), FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(DISTINCT x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- approx_count_distinct, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, approx_count_distinct(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- approx_count_distinct, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, approx_count_distinct(x) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- mode, DOUBLE +-- mode falls back to Spark unless Mode.allowIncompatible is set, so only the answer is checked +-- here. expressions/aggregate/mode.sql runs it natively. +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, mode(x) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r, r)) t AS x) GROUP BY id + +-- mode, FLOAT +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, mode(x) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r, r)) t AS x) GROUP BY id + +-- global max and min, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT max(c), min(c) FROM e + +-- global max and min, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT max(c), min(c) FROM e + +-- global max_by and min_by, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT max_by(id, c), min_by(id, c) FROM e + +-- global max_by and min_by, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT max_by(id, c), min_by(id, c) FROM e + +-- window max and min, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, pos, max(x) OVER w, min(x) OVER w FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) +WINDOW w AS (PARTITION BY id ORDER BY pos +ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) + +-- window max and min, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, pos, max(x) OVER w, min(x) OVER w FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) +WINDOW w AS (PARTITION BY id ORDER BY pos +ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) diff --git a/spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql b/spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql new file mode 100644 index 00000000000..16fe8e9b46f --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql @@ -0,0 +1,77 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- collect_set over float input, from Spark 4.2 on. Spark 4.2 normalizes the input of +-- collect_set: -0.0 and 0.0 are one value, and so are all NaNs. Before 4.2 +-- Spark's collect_set treats -0.0 and 0.0 as one value and NaNs as distinct, which Comet does +-- not match (#5312). CometFloatSemanticsSuite keeps those versions as a known gap, so that +-- it starts failing when a fix lands. +-- +-- Each pair of edge values is expanded from one input row (see aggregates.sql). + +-- ConstantFolding: enabled +-- MinSparkVersion: 4.2 + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_double USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_double a CROSS JOIN fs_edge_double b + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_float USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_float a CROSS JOIN fs_edge_float b + +-- DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, size(collect_set(x)) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id + +-- FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, size(collect_set(x)) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id diff --git a/spark/src/test/resources/sql-tests/float_semantics/comparison.sql b/spark/src/test/resources/sql-tests/float_semantics/comparison.sql new file mode 100644 index 00000000000..e8b93739e8a --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/comparison.sql @@ -0,0 +1,539 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Spark compares floats with SQL ordering: -0.0 equals 0.0, every NaN equals every other NaN, +-- and NaN sorts above every other value. Each query applies one comparison operator to every +-- ordered pair of the edge values, in one operator context. The sweep tracks #6385. +-- +-- The edge values are 0.0, -0.0, a canonical NaN and a NaN with the sign bit set, plus 1.0, +-- -1.0, both infinities and NULL. Spark's Parquet writer canonicalizes NaN, so the tables hold +-- the canonical NaN and a flag, and the p CTE negates the flagged values to make the sign-bit +-- NaN at query time. That works on every platform; on x86-64, every NaN that arithmetic +-- produces also has the sign bit set. +-- +-- comparison_joins.sql covers join conditions and comparison_nested.sql covers comparisons of +-- arrays and structs. literal_comparison.sql compares a column with a literal. +-- +-- The fixtures run with Spark's ConstantFolding on, which the sweep always did. The literal +-- queries need it for a literal to arrive as a sign-bit NaN, and nothing here is an +-- all-literal expression that the default exclusion would keep from being folded. + +-- ConstantFolding: enabled + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_double USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_double a CROSS JOIN fs_edge_double b + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_float USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_float a CROSS JOIN fs_edge_float b + +-- Project, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l = r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l <=> r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l != r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l < r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l <= r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l > r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l >= r FROM p + +-- Project, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l = r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l <=> r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l != r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l < r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l <= r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l > r FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l >= r FROM p + +-- Filter, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p WHERE l = r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p WHERE l <=> r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p WHERE l != r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p WHERE l < r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p WHERE l <= r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p WHERE l > r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p WHERE l >= r + +-- Filter, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p WHERE l = r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p WHERE l <=> r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p WHERE l != r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p WHERE l < r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p WHERE l <= r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p WHERE l > r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p WHERE l >= r + +-- aggregate argument, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l = r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l <=> r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l != r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l < r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l <= r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l > r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l >= r, 1, 0)) FROM p GROUP BY id + +-- aggregate argument, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l = r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l <=> r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l != r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l < r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l <= r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l > r, 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l >= r, 1, 0)) FROM p GROUP BY id + +-- aggregate FILTER, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) FILTER (WHERE l = r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) FILTER (WHERE l <=> r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) FILTER (WHERE l != r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) FILTER (WHERE l < r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) FILTER (WHERE l <= r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) FILTER (WHERE l > r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) FILTER (WHERE l >= r) FROM p GROUP BY id + +-- aggregate FILTER, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) FILTER (WHERE l = r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) FILTER (WHERE l <=> r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) FILTER (WHERE l != r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) FILTER (WHERE l < r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) FILTER (WHERE l <= r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) FILTER (WHERE l > r) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) FILTER (WHERE l >= r) FROM p GROUP BY id + +-- sort key, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p ORDER BY l = r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p ORDER BY l <=> r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p ORDER BY l != r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p ORDER BY l < r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p ORDER BY l <= r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p ORDER BY l > r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id FROM p ORDER BY l >= r, id + +-- sort key, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p ORDER BY l = r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p ORDER BY l <=> r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p ORDER BY l != r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p ORDER BY l < r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p ORDER BY l <= r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p ORDER BY l > r, id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id FROM p ORDER BY l >= r, id + +-- Generate, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, x FROM p LATERAL VIEW explode(array(l = r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, x FROM p LATERAL VIEW explode(array(l <=> r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, x FROM p LATERAL VIEW explode(array(l != r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, x FROM p LATERAL VIEW explode(array(l < r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, x FROM p LATERAL VIEW explode(array(l <= r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, x FROM p LATERAL VIEW explode(array(l > r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, x FROM p LATERAL VIEW explode(array(l >= r)) t AS x + +-- Generate, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, x FROM p LATERAL VIEW explode(array(l = r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, x FROM p LATERAL VIEW explode(array(l <=> r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, x FROM p LATERAL VIEW explode(array(l != r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, x FROM p LATERAL VIEW explode(array(l < r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, x FROM p LATERAL VIEW explode(array(l <= r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, x FROM p LATERAL VIEW explode(array(l > r)) t AS x + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, x FROM p LATERAL VIEW explode(array(l >= r)) t AS x + +-- window function argument, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l = r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l <=> r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l != r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l < r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l <= r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l > r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(l >= r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +-- window function argument, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l = r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l <=> r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l != r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l < r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l <= r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l > r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(l >= r, 1, 0)) OVER (PARTITION BY id % 7) FROM p + +-- window partition key, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) OVER (PARTITION BY l = r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) OVER (PARTITION BY l <=> r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) OVER (PARTITION BY l != r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) OVER (PARTITION BY l < r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) OVER (PARTITION BY l <= r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) OVER (PARTITION BY l > r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, count(*) OVER (PARTITION BY l >= r) FROM p + +-- window partition key, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) OVER (PARTITION BY l = r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) OVER (PARTITION BY l <=> r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) OVER (PARTITION BY l != r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) OVER (PARTITION BY l < r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) OVER (PARTITION BY l <= r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) OVER (PARTITION BY l > r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, count(*) OVER (PARTITION BY l >= r) FROM p diff --git a/spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql b/spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql new file mode 100644 index 00000000000..99e55327ef8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql @@ -0,0 +1,338 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Comparison operators in the condition of each join type, over every ordered pair of the +-- edge values (see comparison.sql). The equality on id is the key of the three equi-joins. The +-- nested loop join has no key, so the comparison is its whole condition, over the edge values +-- alone; its = and <=> make it an equi-join, which Spark plans as a hash join, so they repeat +-- the join keys of keys.sql. + +-- ConstantFolding: enabled + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_double USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_double a CROSS JOIN fs_edge_double b + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_float USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_float a CROSS JOIN fs_edge_float b + +-- broadcast hash join condition, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l = b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <=> b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l != b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l < b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <= b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l > b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l >= b.r + +-- broadcast hash join condition, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l = b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <=> b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l != b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l < b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <= b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l > b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ BROADCAST(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l >= b.r + +-- shuffled hash join condition, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l = b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <=> b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l != b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l < b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <= b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l > b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l >= b.r + +-- shuffled hash join condition, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l = b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <=> b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l != b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l < b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <= b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l > b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l >= b.r + +-- sort merge join condition, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l = b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <=> b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l != b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l < b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <= b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l > b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l >= b.r + +-- sort merge join condition, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l = b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <=> b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l != b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l < b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l <= b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l > b.r + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT /*+ MERGE(b) */ a.id FROM p a JOIN p b +ON a.id = b.id AND a.l >= b.r + +-- nested loop join condition, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c <=> b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c != b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c < b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c <= b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c > b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c >= b.c + +-- nested loop join condition, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c <=> b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c != b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c < b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c <= b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c > b.c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c >= b.c diff --git a/spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql b/spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql new file mode 100644 index 00000000000..5c3f96b6fca --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql @@ -0,0 +1,307 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Comparisons of arrays and structs that hold one float, over every ordered pair of the edge +-- values (see comparison.sql). Spark compares the elements and fields with SQL ordering. + +-- ConstantFolding: enabled + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_double USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_double a CROSS JOIN fs_edge_double b + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_float USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_float a CROSS JOIN fs_edge_float b + +-- array operands, Project, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array(l) = array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array(l) <=> array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array(l) != array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array(l) < array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array(l) <= array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array(l) > array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array(l) >= array(r) FROM p + +-- array operands, Project, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array(l) = array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array(l) <=> array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array(l) != array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array(l) < array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array(l) <= array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array(l) > array(r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array(l) >= array(r) FROM p + +-- array operands, aggregate argument, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(array(l) = array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(array(l) <=> array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(array(l) != array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(array(l) < array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(array(l) <= array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(array(l) > array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(array(l) >= array(r), 1, 0)) FROM p GROUP BY id + +-- array operands, aggregate argument, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(array(l) = array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(array(l) <=> array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(array(l) != array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(array(l) < array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(array(l) <= array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(array(l) > array(r), 1, 0)) FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(array(l) >= array(r), 1, 0)) FROM p GROUP BY id + +-- struct operands, Project, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, named_struct('x', l) = named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, named_struct('x', l) <=> named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, named_struct('x', l) != named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, named_struct('x', l) < named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, named_struct('x', l) <= named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, named_struct('x', l) > named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, named_struct('x', l) >= named_struct('x', r) FROM p + +-- struct operands, Project, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, named_struct('x', l) = named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, named_struct('x', l) <=> named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, named_struct('x', l) != named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, named_struct('x', l) < named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, named_struct('x', l) <= named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, named_struct('x', l) > named_struct('x', r) FROM p + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, named_struct('x', l) >= named_struct('x', r) FROM p + +-- struct operands, aggregate argument, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(named_struct('x', l) = named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(named_struct('x', l) <=> named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(named_struct('x', l) != named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(named_struct('x', l) < named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(named_struct('x', l) <= named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(named_struct('x', l) > named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, max(IF(named_struct('x', l) >= named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +-- struct operands, aggregate argument, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(named_struct('x', l) = named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(named_struct('x', l) <=> named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(named_struct('x', l) != named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(named_struct('x', l) < named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(named_struct('x', l) <= named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(named_struct('x', l) > named_struct('x', r), 1, 0)) +FROM p GROUP BY id + +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, max(IF(named_struct('x', l) >= named_struct('x', r), 1, 0)) +FROM p GROUP BY id diff --git a/spark/src/test/resources/sql-tests/float_semantics/expressions.sql b/spark/src/test/resources/sql-tests/float_semantics/expressions.sql new file mode 100644 index 00000000000..ac51d6e308e --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/expressions.sql @@ -0,0 +1,414 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Expressions with float input, over the edge values (one column) or every ordered pair of +-- them (two columns). Each follows one of Spark's float equality rules, which the contributor +-- guide lists (contributor-guide/floating_point.md). The sweep tracks #6385. +-- +-- The e and p CTEs negate the flagged stored values to make the sign-bit NaN at query time +-- (see comparison.sql). +-- +-- hash and xxhash64 of a float, an array and a struct are in expressions/hash/hash.sql. +-- collect_set is in aggregates_collect_set_4_2.sql. + +-- ConstantFolding: enabled + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_double USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_double a CROSS JOIN fs_edge_double b + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_pair_float USING parquet AS +SELECT a.id * 10 + b.id AS id, a.c AS l, b.c AS r, a.neg AS lneg, b.neg AS rneg +FROM fs_edge_float a CROSS JOIN fs_edge_float b + +-- abs, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, abs(c) FROM e + +-- abs, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, abs(c) FROM e + +-- negative, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, -c FROM e + +-- negative, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, -c FROM e + +-- signum, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, signum(c) FROM e + +-- signum, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, signum(c) FROM e + +-- ceil, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, ceil(c) FROM e + +-- ceil, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, ceil(c) FROM e + +-- floor, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, floor(c) FROM e + +-- floor, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, floor(c) FROM e + +-- round, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, round(c, 1), bround(c, 1) FROM e + +-- round, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, round(c, 1), bround(c, 1) FROM e + +-- sqrt, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, sqrt(c) FROM e + +-- sqrt, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, sqrt(c) FROM e + +-- isnan, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, isnan(c) FROM e + +-- isnan, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, isnan(c) FROM e + +-- cast to string, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, CAST(c AS STRING) FROM e + +-- cast to string, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, CAST(c AS STRING) FROM e + +-- cast to integral, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, try_cast(c AS INT), try_cast(c AS BIGINT) FROM e + +-- cast to integral, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, try_cast(c AS INT), try_cast(c AS BIGINT) FROM e + +-- cast to decimal, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, CAST(c AS DECIMAL(10, 2)) FROM e + +-- cast to decimal, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, CAST(c AS DECIMAL(10, 2)) FROM e + +-- cast to boolean, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, CAST(c AS BOOLEAN) FROM e + +-- cast to boolean, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, CAST(c AS BOOLEAN) FROM e + +-- cast between float types, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, CAST(c AS DOUBLE), CAST(c AS FLOAT) FROM e + +-- cast between float types, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, CAST(c AS DOUBLE), CAST(c AS FLOAT) FROM e + +-- cast array to string, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, CAST(array(c) AS STRING) FROM e + +-- cast array to string, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, CAST(array(c) AS STRING) FROM e + +-- greatest, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, greatest(l, r) FROM p + +-- greatest, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, greatest(l, r) FROM p + +-- least, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, least(l, r) FROM p + +-- least, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, least(l, r) FROM p + +-- nanvl, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, nanvl(l, r) FROM p + +-- nanvl, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, nanvl(l, r) FROM p + +-- IN with column list, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, l IN (r, CAST('2.0' AS DOUBLE)) FROM p + +-- IN with column list, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, l IN (r, CAST('2.0' AS FLOAT)) FROM p + +-- CASE value WHEN, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, CASE l WHEN r THEN 1 ELSE 0 END FROM p + +-- CASE value WHEN, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, CASE l WHEN r THEN 1 ELSE 0 END FROM p + +-- array_contains, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_contains(array(l, 2), r) FROM p + +-- array_contains, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_contains(array(l, 2), r) FROM p + +-- array_position, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_position(array(2, l), r) FROM p + +-- array_position, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_position(array(2, l), r) FROM p + +-- array_remove, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_remove(array(l, 2), r) FROM p + +-- array_remove, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_remove(array(l, 2), r) FROM p + +-- arrays_overlap, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, arrays_overlap(array(l), array(r)) FROM p + +-- arrays_overlap, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, arrays_overlap(array(l), array(r)) FROM p + +-- array_distinct, DOUBLE +-- Whether float elements run natively depends on the Spark release (array_distinct.sql), so only +-- the answer is checked. +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_distinct(array(l, r)) FROM p + +-- array_distinct, FLOAT +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_distinct(array(l, r)) FROM p + +-- array_union, DOUBLE +-- Whether float elements run natively depends on the Spark release (array_union.sql), so only the +-- answer is checked. +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_union(array(l), array(r)) FROM p + +-- array_union, FLOAT +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_union(array(l), array(r)) FROM p + +-- array_intersect, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_intersect(array(l), array(r)) FROM p + +-- array_intersect, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_intersect(array(l), array(r)) FROM p + +-- array_except, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_except(array(l), array(r)) FROM p + +-- array_except, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_except(array(l), array(r)) FROM p + +-- sort_array, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, sort_array(array(l, r, 2)) FROM p + +-- sort_array, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, sort_array(array(l, r, 2)) FROM p + +-- sort_array descending, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, sort_array(array(l, r, 2), false) FROM p + +-- sort_array descending, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, sort_array(array(l, r, 2), false) FROM p + +-- array_sort, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_sort(array(l, r, 2)) FROM p + +-- array_sort, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_sort(array(l, r, 2)) FROM p + +-- array_max, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_max(array(l, r)) FROM p + +-- array_max, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_max(array(l, r)) FROM p + +-- array_min, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, array_min(array(l, r)) FROM p + +-- array_min, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, array_min(array(l, r)) FROM p + +-- map lookup, DOUBLE +-- A map with a float key falls back to Spark (map_builders_float_key_gate.sql), so only the answer +-- is checked. +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, map(l, 1)[r], element_at(map(l, 1), r) FROM p WHERE l IS NOT NULL + +-- map lookup, FLOAT +query spark_answer_only +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, map(l, 1)[r], element_at(map(l, 1), r) FROM p WHERE l IS NOT NULL + +-- map_contains_key, DOUBLE +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) +SELECT id, map_contains_key(map(l, 1), r) FROM p WHERE l IS NOT NULL + +-- map_contains_key, FLOAT +query +WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) +SELECT id, map_contains_key(map(l, 1), r) FROM p WHERE l IS NOT NULL diff --git a/spark/src/test/resources/sql-tests/float_semantics/keys.sql b/spark/src/test/resources/sql-tests/float_semantics/keys.sql new file mode 100644 index 00000000000..8984f2e6358 --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/keys.sql @@ -0,0 +1,263 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Contexts where Spark normalizes a float key (grouping, join and window partition keys +-- get NormalizeNaNAndZero) or compares it with SQL ordering (IN, ORDER BY, rank): -0.0 and +-- 0.0 are one key, and so are all NaNs. The sweep tracks #6385. +-- +-- The edge values are 0.0, -0.0, a canonical NaN and a NaN with the sign bit set, plus 1.0, +-- -1.0, both infinities and NULL. The e CTE negates the flagged stored values to make the +-- sign-bit NaN at query time (see comparison.sql). +-- +-- Nested ORDER BY and window order keys are in windows/nested_float_order_keys.sql, which +-- uses the same edge values. + +-- ConstantFolding: enabled + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +-- GROUP BY, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT c, count(*) FROM e GROUP BY c + +-- GROUP BY, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT c, count(*) FROM e GROUP BY c + +-- DISTINCT, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT DISTINCT c FROM e + +-- DISTINCT, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT DISTINCT c FROM e + +-- count(DISTINCT), DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT count(DISTINCT c) FROM e + +-- count(DISTINCT), FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT count(DISTINCT c) FROM e + +-- broadcast hash join key, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +-- broadcast hash join key, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ BROADCAST(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +-- shuffled hash join key, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ SHUFFLE_HASH(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +-- shuffled hash join key, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ SHUFFLE_HASH(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +-- sort merge join key, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT /*+ MERGE(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +-- sort merge join key, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT /*+ MERGE(b) */ a.id, b.id FROM e a JOIN e b ON a.c = b.c + +-- null-safe join key, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT a.id, b.id FROM e a JOIN e b ON a.c <=> b.c + +-- null-safe join key, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT a.id, b.id FROM e a JOIN e b ON a.c <=> b.c + +-- left semi join key, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT a.id FROM e a LEFT SEMI JOIN (SELECT * FROM e WHERE id IN (2, 4)) b +ON a.c = b.c + +-- left semi join key, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT a.id FROM e a LEFT SEMI JOIN (SELECT * FROM e WHERE id IN (2, 4)) b +ON a.c = b.c + +-- left anti join key, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT a.id FROM e a LEFT ANTI JOIN (SELECT * FROM e WHERE id IN (2, 4)) b +ON a.c = b.c + +-- left anti join key, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT a.id FROM e a LEFT ANTI JOIN (SELECT * FROM e WHERE id IN (2, 4)) b +ON a.c = b.c + +-- INTERSECT, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT c FROM e INTERSECT SELECT c FROM e WHERE id IN (2, 4) + +-- INTERSECT, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT c FROM e INTERSECT SELECT c FROM e WHERE id IN (2, 4) + +-- EXCEPT, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT c FROM e EXCEPT SELECT c FROM e WHERE id IN (2, 4) + +-- EXCEPT, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT c FROM e EXCEPT SELECT c FROM e WHERE id IN (2, 4) + +-- IN list, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id FROM e WHERE c IN (CAST('0.0' AS DOUBLE), CAST('NaN' AS DOUBLE)) + +-- IN list, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id FROM e WHERE c IN (CAST('0.0' AS FLOAT), CAST('NaN' AS FLOAT)) + +-- IN subquery, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id FROM e WHERE c IN (SELECT c FROM e WHERE id IN (2, 4)) + +-- IN subquery, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id FROM e WHERE c IN (SELECT c FROM e WHERE id IN (2, 4)) + +-- ORDER BY, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id FROM e ORDER BY c, id + +-- ORDER BY, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id FROM e ORDER BY c, id + +-- ORDER BY DESC, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id FROM e ORDER BY c DESC NULLS LAST, id + +-- ORDER BY DESC, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id FROM e ORDER BY c DESC NULLS LAST, id + +-- window partition key, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, count(*) OVER (PARTITION BY c) FROM e + +-- window partition key, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, count(*) OVER (PARTITION BY c) FROM e + +-- window order key, rank, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, rank() OVER (ORDER BY c) FROM e + +-- window order key, rank, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, rank() OVER (ORDER BY c) FROM e + +-- window order key, dense_rank, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, dense_rank() OVER (ORDER BY c) FROM e + +-- window order key, dense_rank, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, dense_rank() OVER (ORDER BY c) FROM e + +-- hash repartition, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, spark_partition_id() FROM (SELECT /*+ REPARTITION(4, c) */ * FROM e) + +-- hash repartition, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, spark_partition_id() FROM (SELECT /*+ REPARTITION(4, c) */ * FROM e) + +-- nested GROUP BY, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT array(c), count(*) FROM e GROUP BY array(c) + +-- nested GROUP BY, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT array(c), count(*) FROM e GROUP BY array(c) diff --git a/spark/src/test/resources/sql-tests/float_semantics/literal_comparison.sql b/spark/src/test/resources/sql-tests/float_semantics/literal_comparison.sql new file mode 100644 index 00000000000..490daacb4e0 --- /dev/null +++ b/spark/src/test/resources/sql-tests/float_semantics/literal_comparison.sql @@ -0,0 +1,978 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- A column compared with each edge literal, on either side. The literals are 0.0, -0.0, a +-- canonical NaN, a NaN with the sign bit set (-CAST('NaN' AS ...)) and 1.0. +-- +-- This file needs ConstantFolding: Spark folds each literal into a Literal, so the +-- sign-bit NaN reaches Comet as a constant. The Parquet scan filter is only pushed down +-- when its literal has been folded. +-- +-- Each predicate is its own UNION ALL branch. With all of them in one projection, Spark +-- 4.1.3's whole-stage codegen returns false for 0.0 = 0.0 (it is right with subexpression +-- elimination turned off), which would hide a Comet difference behind a Spark bug. +-- +-- A stored column compared with a literal is pushed into the Parquet reader, so the third +-- context reads the table directly, without the CTE. + +-- ConstantFolding: enabled + +statement +CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_double VALUES + (1, 0.0D, false), + (2, double('-0.0'), false), + (3, double('NaN'), false), + (4, double('NaN'), true), + (5, 1.0D, false), + (6, -1.0D, false), + (7, double('Infinity'), false), + (8, double('-Infinity'), false), + (9, NULL, false) + +statement +CREATE TABLE fs_edge_float(id INT, c FLOAT, neg BOOLEAN) USING parquet + +statement +INSERT INTO fs_edge_float VALUES + (1, float(0.0), false), + (2, float('-0.0'), false), + (3, float('NaN'), false), + (4, float('NaN'), true), + (5, float(1.0), false), + (6, float(-1.0), false), + (7, float('Infinity'), false), + (8, float('-Infinity'), false), + (9, NULL, false) + +-- Project, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0, c = CAST('0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS DOUBLE) = c FROM e +UNION ALL +SELECT id, 2, c = CAST('-0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS DOUBLE) = c FROM e +UNION ALL +SELECT id, 4, c = CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS DOUBLE) = c FROM e +UNION ALL +SELECT id, 6, c = -CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS DOUBLE) = c FROM e +UNION ALL +SELECT id, 8, c = CAST('1.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS DOUBLE) = c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0, c <=> CAST('0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS DOUBLE) <=> c FROM e +UNION ALL +SELECT id, 2, c <=> CAST('-0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS DOUBLE) <=> c FROM e +UNION ALL +SELECT id, 4, c <=> CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS DOUBLE) <=> c FROM e +UNION ALL +SELECT id, 6, c <=> -CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS DOUBLE) <=> c FROM e +UNION ALL +SELECT id, 8, c <=> CAST('1.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS DOUBLE) <=> c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0, c != CAST('0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS DOUBLE) != c FROM e +UNION ALL +SELECT id, 2, c != CAST('-0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS DOUBLE) != c FROM e +UNION ALL +SELECT id, 4, c != CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS DOUBLE) != c FROM e +UNION ALL +SELECT id, 6, c != -CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS DOUBLE) != c FROM e +UNION ALL +SELECT id, 8, c != CAST('1.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS DOUBLE) != c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0, c < CAST('0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS DOUBLE) < c FROM e +UNION ALL +SELECT id, 2, c < CAST('-0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS DOUBLE) < c FROM e +UNION ALL +SELECT id, 4, c < CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS DOUBLE) < c FROM e +UNION ALL +SELECT id, 6, c < -CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS DOUBLE) < c FROM e +UNION ALL +SELECT id, 8, c < CAST('1.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS DOUBLE) < c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0, c <= CAST('0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS DOUBLE) <= c FROM e +UNION ALL +SELECT id, 2, c <= CAST('-0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS DOUBLE) <= c FROM e +UNION ALL +SELECT id, 4, c <= CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS DOUBLE) <= c FROM e +UNION ALL +SELECT id, 6, c <= -CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS DOUBLE) <= c FROM e +UNION ALL +SELECT id, 8, c <= CAST('1.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS DOUBLE) <= c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0, c > CAST('0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS DOUBLE) > c FROM e +UNION ALL +SELECT id, 2, c > CAST('-0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS DOUBLE) > c FROM e +UNION ALL +SELECT id, 4, c > CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS DOUBLE) > c FROM e +UNION ALL +SELECT id, 6, c > -CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS DOUBLE) > c FROM e +UNION ALL +SELECT id, 8, c > CAST('1.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS DOUBLE) > c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0, c >= CAST('0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS DOUBLE) >= c FROM e +UNION ALL +SELECT id, 2, c >= CAST('-0.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS DOUBLE) >= c FROM e +UNION ALL +SELECT id, 4, c >= CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS DOUBLE) >= c FROM e +UNION ALL +SELECT id, 6, c >= -CAST('NaN' AS DOUBLE) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS DOUBLE) >= c FROM e +UNION ALL +SELECT id, 8, c >= CAST('1.0' AS DOUBLE) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS DOUBLE) >= c FROM e + +-- Project, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0, c = CAST('0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS FLOAT) = c FROM e +UNION ALL +SELECT id, 2, c = CAST('-0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS FLOAT) = c FROM e +UNION ALL +SELECT id, 4, c = CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS FLOAT) = c FROM e +UNION ALL +SELECT id, 6, c = -CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS FLOAT) = c FROM e +UNION ALL +SELECT id, 8, c = CAST('1.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS FLOAT) = c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0, c <=> CAST('0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS FLOAT) <=> c FROM e +UNION ALL +SELECT id, 2, c <=> CAST('-0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS FLOAT) <=> c FROM e +UNION ALL +SELECT id, 4, c <=> CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS FLOAT) <=> c FROM e +UNION ALL +SELECT id, 6, c <=> -CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS FLOAT) <=> c FROM e +UNION ALL +SELECT id, 8, c <=> CAST('1.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS FLOAT) <=> c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0, c != CAST('0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS FLOAT) != c FROM e +UNION ALL +SELECT id, 2, c != CAST('-0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS FLOAT) != c FROM e +UNION ALL +SELECT id, 4, c != CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS FLOAT) != c FROM e +UNION ALL +SELECT id, 6, c != -CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS FLOAT) != c FROM e +UNION ALL +SELECT id, 8, c != CAST('1.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS FLOAT) != c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0, c < CAST('0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS FLOAT) < c FROM e +UNION ALL +SELECT id, 2, c < CAST('-0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS FLOAT) < c FROM e +UNION ALL +SELECT id, 4, c < CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS FLOAT) < c FROM e +UNION ALL +SELECT id, 6, c < -CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS FLOAT) < c FROM e +UNION ALL +SELECT id, 8, c < CAST('1.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS FLOAT) < c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0, c <= CAST('0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS FLOAT) <= c FROM e +UNION ALL +SELECT id, 2, c <= CAST('-0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS FLOAT) <= c FROM e +UNION ALL +SELECT id, 4, c <= CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS FLOAT) <= c FROM e +UNION ALL +SELECT id, 6, c <= -CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS FLOAT) <= c FROM e +UNION ALL +SELECT id, 8, c <= CAST('1.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS FLOAT) <= c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0, c > CAST('0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS FLOAT) > c FROM e +UNION ALL +SELECT id, 2, c > CAST('-0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS FLOAT) > c FROM e +UNION ALL +SELECT id, 4, c > CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS FLOAT) > c FROM e +UNION ALL +SELECT id, 6, c > -CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS FLOAT) > c FROM e +UNION ALL +SELECT id, 8, c > CAST('1.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS FLOAT) > c FROM e + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0, c >= CAST('0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 1, CAST('0.0' AS FLOAT) >= c FROM e +UNION ALL +SELECT id, 2, c >= CAST('-0.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 3, CAST('-0.0' AS FLOAT) >= c FROM e +UNION ALL +SELECT id, 4, c >= CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 5, CAST('NaN' AS FLOAT) >= c FROM e +UNION ALL +SELECT id, 6, c >= -CAST('NaN' AS FLOAT) FROM e +UNION ALL +SELECT id, 7, -CAST('NaN' AS FLOAT) >= c FROM e +UNION ALL +SELECT id, 8, c >= CAST('1.0' AS FLOAT) FROM e +UNION ALL +SELECT id, 9, CAST('1.0' AS FLOAT) >= c FROM e + +-- Filter, DOUBLE +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0 FROM e WHERE c = CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS DOUBLE) = c +UNION ALL +SELECT id, 2 FROM e WHERE c = CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS DOUBLE) = c +UNION ALL +SELECT id, 4 FROM e WHERE c = CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS DOUBLE) = c +UNION ALL +SELECT id, 6 FROM e WHERE c = -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS DOUBLE) = c +UNION ALL +SELECT id, 8 FROM e WHERE c = CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS DOUBLE) = c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0 FROM e WHERE c <=> CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS DOUBLE) <=> c +UNION ALL +SELECT id, 2 FROM e WHERE c <=> CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS DOUBLE) <=> c +UNION ALL +SELECT id, 4 FROM e WHERE c <=> CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS DOUBLE) <=> c +UNION ALL +SELECT id, 6 FROM e WHERE c <=> -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS DOUBLE) <=> c +UNION ALL +SELECT id, 8 FROM e WHERE c <=> CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS DOUBLE) <=> c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0 FROM e WHERE c != CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS DOUBLE) != c +UNION ALL +SELECT id, 2 FROM e WHERE c != CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS DOUBLE) != c +UNION ALL +SELECT id, 4 FROM e WHERE c != CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS DOUBLE) != c +UNION ALL +SELECT id, 6 FROM e WHERE c != -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS DOUBLE) != c +UNION ALL +SELECT id, 8 FROM e WHERE c != CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS DOUBLE) != c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0 FROM e WHERE c < CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS DOUBLE) < c +UNION ALL +SELECT id, 2 FROM e WHERE c < CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS DOUBLE) < c +UNION ALL +SELECT id, 4 FROM e WHERE c < CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS DOUBLE) < c +UNION ALL +SELECT id, 6 FROM e WHERE c < -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS DOUBLE) < c +UNION ALL +SELECT id, 8 FROM e WHERE c < CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS DOUBLE) < c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0 FROM e WHERE c <= CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS DOUBLE) <= c +UNION ALL +SELECT id, 2 FROM e WHERE c <= CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS DOUBLE) <= c +UNION ALL +SELECT id, 4 FROM e WHERE c <= CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS DOUBLE) <= c +UNION ALL +SELECT id, 6 FROM e WHERE c <= -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS DOUBLE) <= c +UNION ALL +SELECT id, 8 FROM e WHERE c <= CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS DOUBLE) <= c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0 FROM e WHERE c > CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS DOUBLE) > c +UNION ALL +SELECT id, 2 FROM e WHERE c > CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS DOUBLE) > c +UNION ALL +SELECT id, 4 FROM e WHERE c > CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS DOUBLE) > c +UNION ALL +SELECT id, 6 FROM e WHERE c > -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS DOUBLE) > c +UNION ALL +SELECT id, 8 FROM e WHERE c > CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS DOUBLE) > c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_double) +SELECT id, 0 FROM e WHERE c >= CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS DOUBLE) >= c +UNION ALL +SELECT id, 2 FROM e WHERE c >= CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS DOUBLE) >= c +UNION ALL +SELECT id, 4 FROM e WHERE c >= CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS DOUBLE) >= c +UNION ALL +SELECT id, 6 FROM e WHERE c >= -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS DOUBLE) >= c +UNION ALL +SELECT id, 8 FROM e WHERE c >= CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS DOUBLE) >= c + +-- Filter, FLOAT +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0 FROM e WHERE c = CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS FLOAT) = c +UNION ALL +SELECT id, 2 FROM e WHERE c = CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS FLOAT) = c +UNION ALL +SELECT id, 4 FROM e WHERE c = CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS FLOAT) = c +UNION ALL +SELECT id, 6 FROM e WHERE c = -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS FLOAT) = c +UNION ALL +SELECT id, 8 FROM e WHERE c = CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS FLOAT) = c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0 FROM e WHERE c <=> CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS FLOAT) <=> c +UNION ALL +SELECT id, 2 FROM e WHERE c <=> CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS FLOAT) <=> c +UNION ALL +SELECT id, 4 FROM e WHERE c <=> CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS FLOAT) <=> c +UNION ALL +SELECT id, 6 FROM e WHERE c <=> -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS FLOAT) <=> c +UNION ALL +SELECT id, 8 FROM e WHERE c <=> CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS FLOAT) <=> c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0 FROM e WHERE c != CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS FLOAT) != c +UNION ALL +SELECT id, 2 FROM e WHERE c != CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS FLOAT) != c +UNION ALL +SELECT id, 4 FROM e WHERE c != CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS FLOAT) != c +UNION ALL +SELECT id, 6 FROM e WHERE c != -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS FLOAT) != c +UNION ALL +SELECT id, 8 FROM e WHERE c != CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS FLOAT) != c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0 FROM e WHERE c < CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS FLOAT) < c +UNION ALL +SELECT id, 2 FROM e WHERE c < CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS FLOAT) < c +UNION ALL +SELECT id, 4 FROM e WHERE c < CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS FLOAT) < c +UNION ALL +SELECT id, 6 FROM e WHERE c < -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS FLOAT) < c +UNION ALL +SELECT id, 8 FROM e WHERE c < CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS FLOAT) < c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0 FROM e WHERE c <= CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS FLOAT) <= c +UNION ALL +SELECT id, 2 FROM e WHERE c <= CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS FLOAT) <= c +UNION ALL +SELECT id, 4 FROM e WHERE c <= CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS FLOAT) <= c +UNION ALL +SELECT id, 6 FROM e WHERE c <= -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS FLOAT) <= c +UNION ALL +SELECT id, 8 FROM e WHERE c <= CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS FLOAT) <= c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0 FROM e WHERE c > CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS FLOAT) > c +UNION ALL +SELECT id, 2 FROM e WHERE c > CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS FLOAT) > c +UNION ALL +SELECT id, 4 FROM e WHERE c > CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS FLOAT) > c +UNION ALL +SELECT id, 6 FROM e WHERE c > -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS FLOAT) > c +UNION ALL +SELECT id, 8 FROM e WHERE c > CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS FLOAT) > c + +query +WITH e AS (SELECT id, IF(neg, -c, c) AS c FROM fs_edge_float) +SELECT id, 0 FROM e WHERE c >= CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM e WHERE CAST('0.0' AS FLOAT) >= c +UNION ALL +SELECT id, 2 FROM e WHERE c >= CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM e WHERE CAST('-0.0' AS FLOAT) >= c +UNION ALL +SELECT id, 4 FROM e WHERE c >= CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM e WHERE CAST('NaN' AS FLOAT) >= c +UNION ALL +SELECT id, 6 FROM e WHERE c >= -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM e WHERE -CAST('NaN' AS FLOAT) >= c +UNION ALL +SELECT id, 8 FROM e WHERE c >= CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM e WHERE CAST('1.0' AS FLOAT) >= c + +-- scan filter, DOUBLE +query +SELECT id, 0 FROM fs_edge_double WHERE c = CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM fs_edge_double WHERE CAST('0.0' AS DOUBLE) = c +UNION ALL +SELECT id, 2 FROM fs_edge_double WHERE c = CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM fs_edge_double WHERE CAST('-0.0' AS DOUBLE) = c +UNION ALL +SELECT id, 4 FROM fs_edge_double WHERE c = CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM fs_edge_double WHERE CAST('NaN' AS DOUBLE) = c +UNION ALL +SELECT id, 6 FROM fs_edge_double WHERE c = -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM fs_edge_double WHERE -CAST('NaN' AS DOUBLE) = c +UNION ALL +SELECT id, 8 FROM fs_edge_double WHERE c = CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM fs_edge_double WHERE CAST('1.0' AS DOUBLE) = c + +query +SELECT id, 0 FROM fs_edge_double WHERE c <=> CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM fs_edge_double WHERE CAST('0.0' AS DOUBLE) <=> c +UNION ALL +SELECT id, 2 FROM fs_edge_double WHERE c <=> CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM fs_edge_double WHERE CAST('-0.0' AS DOUBLE) <=> c +UNION ALL +SELECT id, 4 FROM fs_edge_double WHERE c <=> CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM fs_edge_double WHERE CAST('NaN' AS DOUBLE) <=> c +UNION ALL +SELECT id, 6 FROM fs_edge_double WHERE c <=> -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM fs_edge_double WHERE -CAST('NaN' AS DOUBLE) <=> c +UNION ALL +SELECT id, 8 FROM fs_edge_double WHERE c <=> CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM fs_edge_double WHERE CAST('1.0' AS DOUBLE) <=> c + +query +SELECT id, 0 FROM fs_edge_double WHERE c != CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM fs_edge_double WHERE CAST('0.0' AS DOUBLE) != c +UNION ALL +SELECT id, 2 FROM fs_edge_double WHERE c != CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM fs_edge_double WHERE CAST('-0.0' AS DOUBLE) != c +UNION ALL +SELECT id, 4 FROM fs_edge_double WHERE c != CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM fs_edge_double WHERE CAST('NaN' AS DOUBLE) != c +UNION ALL +SELECT id, 6 FROM fs_edge_double WHERE c != -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM fs_edge_double WHERE -CAST('NaN' AS DOUBLE) != c +UNION ALL +SELECT id, 8 FROM fs_edge_double WHERE c != CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM fs_edge_double WHERE CAST('1.0' AS DOUBLE) != c + +query +SELECT id, 0 FROM fs_edge_double WHERE c < CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM fs_edge_double WHERE CAST('0.0' AS DOUBLE) < c +UNION ALL +SELECT id, 2 FROM fs_edge_double WHERE c < CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM fs_edge_double WHERE CAST('-0.0' AS DOUBLE) < c +UNION ALL +SELECT id, 4 FROM fs_edge_double WHERE c < CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM fs_edge_double WHERE CAST('NaN' AS DOUBLE) < c +UNION ALL +SELECT id, 6 FROM fs_edge_double WHERE c < -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM fs_edge_double WHERE -CAST('NaN' AS DOUBLE) < c +UNION ALL +SELECT id, 8 FROM fs_edge_double WHERE c < CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM fs_edge_double WHERE CAST('1.0' AS DOUBLE) < c + +query +SELECT id, 0 FROM fs_edge_double WHERE c <= CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM fs_edge_double WHERE CAST('0.0' AS DOUBLE) <= c +UNION ALL +SELECT id, 2 FROM fs_edge_double WHERE c <= CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM fs_edge_double WHERE CAST('-0.0' AS DOUBLE) <= c +UNION ALL +SELECT id, 4 FROM fs_edge_double WHERE c <= CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM fs_edge_double WHERE CAST('NaN' AS DOUBLE) <= c +UNION ALL +SELECT id, 6 FROM fs_edge_double WHERE c <= -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM fs_edge_double WHERE -CAST('NaN' AS DOUBLE) <= c +UNION ALL +SELECT id, 8 FROM fs_edge_double WHERE c <= CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM fs_edge_double WHERE CAST('1.0' AS DOUBLE) <= c + +query +SELECT id, 0 FROM fs_edge_double WHERE c > CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM fs_edge_double WHERE CAST('0.0' AS DOUBLE) > c +UNION ALL +SELECT id, 2 FROM fs_edge_double WHERE c > CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM fs_edge_double WHERE CAST('-0.0' AS DOUBLE) > c +UNION ALL +SELECT id, 4 FROM fs_edge_double WHERE c > CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM fs_edge_double WHERE CAST('NaN' AS DOUBLE) > c +UNION ALL +SELECT id, 6 FROM fs_edge_double WHERE c > -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM fs_edge_double WHERE -CAST('NaN' AS DOUBLE) > c +UNION ALL +SELECT id, 8 FROM fs_edge_double WHERE c > CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM fs_edge_double WHERE CAST('1.0' AS DOUBLE) > c + +query +SELECT id, 0 FROM fs_edge_double WHERE c >= CAST('0.0' AS DOUBLE) +UNION ALL +SELECT id, 1 FROM fs_edge_double WHERE CAST('0.0' AS DOUBLE) >= c +UNION ALL +SELECT id, 2 FROM fs_edge_double WHERE c >= CAST('-0.0' AS DOUBLE) +UNION ALL +SELECT id, 3 FROM fs_edge_double WHERE CAST('-0.0' AS DOUBLE) >= c +UNION ALL +SELECT id, 4 FROM fs_edge_double WHERE c >= CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 5 FROM fs_edge_double WHERE CAST('NaN' AS DOUBLE) >= c +UNION ALL +SELECT id, 6 FROM fs_edge_double WHERE c >= -CAST('NaN' AS DOUBLE) +UNION ALL +SELECT id, 7 FROM fs_edge_double WHERE -CAST('NaN' AS DOUBLE) >= c +UNION ALL +SELECT id, 8 FROM fs_edge_double WHERE c >= CAST('1.0' AS DOUBLE) +UNION ALL +SELECT id, 9 FROM fs_edge_double WHERE CAST('1.0' AS DOUBLE) >= c + +-- scan filter, FLOAT +query +SELECT id, 0 FROM fs_edge_float WHERE c = CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM fs_edge_float WHERE CAST('0.0' AS FLOAT) = c +UNION ALL +SELECT id, 2 FROM fs_edge_float WHERE c = CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM fs_edge_float WHERE CAST('-0.0' AS FLOAT) = c +UNION ALL +SELECT id, 4 FROM fs_edge_float WHERE c = CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM fs_edge_float WHERE CAST('NaN' AS FLOAT) = c +UNION ALL +SELECT id, 6 FROM fs_edge_float WHERE c = -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM fs_edge_float WHERE -CAST('NaN' AS FLOAT) = c +UNION ALL +SELECT id, 8 FROM fs_edge_float WHERE c = CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM fs_edge_float WHERE CAST('1.0' AS FLOAT) = c + +query +SELECT id, 0 FROM fs_edge_float WHERE c <=> CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM fs_edge_float WHERE CAST('0.0' AS FLOAT) <=> c +UNION ALL +SELECT id, 2 FROM fs_edge_float WHERE c <=> CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM fs_edge_float WHERE CAST('-0.0' AS FLOAT) <=> c +UNION ALL +SELECT id, 4 FROM fs_edge_float WHERE c <=> CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM fs_edge_float WHERE CAST('NaN' AS FLOAT) <=> c +UNION ALL +SELECT id, 6 FROM fs_edge_float WHERE c <=> -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM fs_edge_float WHERE -CAST('NaN' AS FLOAT) <=> c +UNION ALL +SELECT id, 8 FROM fs_edge_float WHERE c <=> CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM fs_edge_float WHERE CAST('1.0' AS FLOAT) <=> c + +query +SELECT id, 0 FROM fs_edge_float WHERE c != CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM fs_edge_float WHERE CAST('0.0' AS FLOAT) != c +UNION ALL +SELECT id, 2 FROM fs_edge_float WHERE c != CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM fs_edge_float WHERE CAST('-0.0' AS FLOAT) != c +UNION ALL +SELECT id, 4 FROM fs_edge_float WHERE c != CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM fs_edge_float WHERE CAST('NaN' AS FLOAT) != c +UNION ALL +SELECT id, 6 FROM fs_edge_float WHERE c != -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM fs_edge_float WHERE -CAST('NaN' AS FLOAT) != c +UNION ALL +SELECT id, 8 FROM fs_edge_float WHERE c != CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM fs_edge_float WHERE CAST('1.0' AS FLOAT) != c + +query +SELECT id, 0 FROM fs_edge_float WHERE c < CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM fs_edge_float WHERE CAST('0.0' AS FLOAT) < c +UNION ALL +SELECT id, 2 FROM fs_edge_float WHERE c < CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM fs_edge_float WHERE CAST('-0.0' AS FLOAT) < c +UNION ALL +SELECT id, 4 FROM fs_edge_float WHERE c < CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM fs_edge_float WHERE CAST('NaN' AS FLOAT) < c +UNION ALL +SELECT id, 6 FROM fs_edge_float WHERE c < -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM fs_edge_float WHERE -CAST('NaN' AS FLOAT) < c +UNION ALL +SELECT id, 8 FROM fs_edge_float WHERE c < CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM fs_edge_float WHERE CAST('1.0' AS FLOAT) < c + +query +SELECT id, 0 FROM fs_edge_float WHERE c <= CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM fs_edge_float WHERE CAST('0.0' AS FLOAT) <= c +UNION ALL +SELECT id, 2 FROM fs_edge_float WHERE c <= CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM fs_edge_float WHERE CAST('-0.0' AS FLOAT) <= c +UNION ALL +SELECT id, 4 FROM fs_edge_float WHERE c <= CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM fs_edge_float WHERE CAST('NaN' AS FLOAT) <= c +UNION ALL +SELECT id, 6 FROM fs_edge_float WHERE c <= -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM fs_edge_float WHERE -CAST('NaN' AS FLOAT) <= c +UNION ALL +SELECT id, 8 FROM fs_edge_float WHERE c <= CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM fs_edge_float WHERE CAST('1.0' AS FLOAT) <= c + +query +SELECT id, 0 FROM fs_edge_float WHERE c > CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM fs_edge_float WHERE CAST('0.0' AS FLOAT) > c +UNION ALL +SELECT id, 2 FROM fs_edge_float WHERE c > CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM fs_edge_float WHERE CAST('-0.0' AS FLOAT) > c +UNION ALL +SELECT id, 4 FROM fs_edge_float WHERE c > CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM fs_edge_float WHERE CAST('NaN' AS FLOAT) > c +UNION ALL +SELECT id, 6 FROM fs_edge_float WHERE c > -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM fs_edge_float WHERE -CAST('NaN' AS FLOAT) > c +UNION ALL +SELECT id, 8 FROM fs_edge_float WHERE c > CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM fs_edge_float WHERE CAST('1.0' AS FLOAT) > c + +query +SELECT id, 0 FROM fs_edge_float WHERE c >= CAST('0.0' AS FLOAT) +UNION ALL +SELECT id, 1 FROM fs_edge_float WHERE CAST('0.0' AS FLOAT) >= c +UNION ALL +SELECT id, 2 FROM fs_edge_float WHERE c >= CAST('-0.0' AS FLOAT) +UNION ALL +SELECT id, 3 FROM fs_edge_float WHERE CAST('-0.0' AS FLOAT) >= c +UNION ALL +SELECT id, 4 FROM fs_edge_float WHERE c >= CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 5 FROM fs_edge_float WHERE CAST('NaN' AS FLOAT) >= c +UNION ALL +SELECT id, 6 FROM fs_edge_float WHERE c >= -CAST('NaN' AS FLOAT) +UNION ALL +SELECT id, 7 FROM fs_edge_float WHERE -CAST('NaN' AS FLOAT) >= c +UNION ALL +SELECT id, 8 FROM fs_edge_float WHERE c >= CAST('1.0' AS FLOAT) +UNION ALL +SELECT id, 9 FROM fs_edge_float WHERE CAST('1.0' AS FLOAT) >= c diff --git a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala index 442cb7c41a7..267eb98141d 100644 --- a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala @@ -24,17 +24,20 @@ import org.scalatest.exceptions.TestFailedException import org.apache.spark.sql.CometTestBase /** - * Sweeps Spark's `-0.0` and NaN semantics across operator contexts and the expressions that - * accept `FLOAT` or `DOUBLE` input (#6385). + * The part of the `-0.0` and NaN sweep (#6385) that the SQL file tests cannot express. + * + * The sweep itself lives in the `float_semantics` directory of `sql-tests`, where every case must + * match Spark. + * + * A SQL file test has no mode that expects a difference, so it cannot hold a case that is a known + * gap. This suite holds the one that remains, `collect_set` before Spark 4.2 (#5312), and expects + * it to return a different answer from Spark. The test fails once the answers match, so that a + * fix, or a DataFusion upgrade that changes float behavior, removes the test instead of leaving + * it stale. `aggregates_collect_set_4_2.sql` covers Spark 4.2, where the answers match. * * The edge values are `0.0`, `-0.0`, a canonical NaN and a NaN with the sign bit set, plus `1.0`, * `-1.0`, both infinities and `NULL`. A Parquet round trip through Spark's writer canonicalizes - * NaN, so the sign-bit NaN is made at query time by negating a stored NaN. That works on every - * platform; on x86-64, every NaN that arithmetic produces also has the sign bit set. - * - * A case listed in [[CometFloatSemanticsSuite.knownGaps]] is expected to return a different - * answer from Spark. The case fails once it matches, so that a fix, or a DataFusion upgrade that - * changes float behavior, removes its entry from the list instead of leaving it stale. + * NaN, so the sign-bit NaN is made at query time by negating a stored NaN. */ class CometFloatSemanticsSuite extends CometTestBase { @@ -47,20 +50,10 @@ class CometFloatSemanticsSuite extends CometTestBase { sql("""CREATE TABLE fs_pair_raw USING parquet AS |SELECT a.id * 10 + b.id AS id, a.d, a.f, a.neg AS dneg, b.d AS e, b.f AS h, b.neg AS eneg |FROM fs_edge_raw a CROSS JOIN fs_edge_raw b""".stripMargin) - // One row per edge value, in columns d (DOUBLE) and f (FLOAT). - sql("""CREATE TEMPORARY VIEW fs_e AS - |SELECT id, IF(neg, -d, d) AS d, IF(neg, -f, f) AS f FROM fs_edge_raw""".stripMargin) - // Every ordered pair of edge values, in columns (d, e) and (f, h). - sql("""CREATE TEMPORARY VIEW fs_p AS - |SELECT id, IF(dneg, -d, d) AS d, IF(eneg, -e, e) AS e, - | IF(dneg, -f, f) AS f, IF(eneg, -h, h) AS h - |FROM fs_pair_raw""".stripMargin) } override def afterAll(): Unit = { try { - sql("DROP VIEW IF EXISTS fs_e") - sql("DROP VIEW IF EXISTS fs_p") sql("DROP TABLE IF EXISTS fs_edge_raw") sql("DROP TABLE IF EXISTS fs_pair_raw") } finally { @@ -68,22 +61,30 @@ class CometFloatSemanticsSuite extends CometTestBase { } } - for (c <- cases) { - test(c.name) { - knownGaps.find(_.covers(c)) match { - case None => checkSparkAnswer(c.sql) - case Some(gap) => - val matched = - try { - checkSparkAnswer(c.sql) - true - } catch { - case e: TestFailedException if e.getMessage.startsWith("Results do not match") => - false - } - assert( - !matched, - s"This case now matches Spark. Remove it from the known gaps for ${gap.issue}.") + // Spark 4.2 normalizes the input of collect_set, so Comet matches it there. + if (!CometSparkSessionExtensions.isSpark42Plus) { + for ((name, l, r) <- Seq(("double", "d", "e"), ("float", "f", "h"))) { + test(s"known gap #5312: collect_set before Spark 4.2 [$name]") { + // Each pair is expanded from one input row, so its two values reach the same partial + // aggregate in array order. + val query = + s"""WITH p AS ( + | SELECT id, IF(dneg, -$l, $l) AS l, IF(eneg, -$r, $r) AS r FROM fs_pair_raw) + |SELECT id, size(collect_set(x)) + |FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) + |GROUP BY id""".stripMargin + val matched = + try { + checkSparkAnswer(query) + true + } catch { + case e: TestFailedException if e.getMessage.startsWith("Results do not match") => + false + } + assert( + !matched, + "This case now matches Spark. Remove this test, and the MinSparkVersion line in " + + "aggregates_collect_set_4_2.sql, for https://github.com/apache/datafusion-comet/issues/5312.") } } } @@ -101,240 +102,4 @@ object CometFloatSemanticsSuite { "(7, double('Infinity'), float('Infinity'), false)", "(8, double('-Infinity'), float('-Infinity'), false)", "(9, NULL, NULL, false)") - - /** - * Column names for one float type. `c` is the column of `fs_e`, and `l` and `r` are the two - * columns of `fs_p`. - */ - case class FloatType(name: String, sqlType: String, c: String, l: String, r: String) { - def lit(v: String): String = s"CAST('$v' AS $sqlType)" - - /** Literals for the edge values, including a constant-folded sign-bit NaN. */ - def literals: Seq[String] = - Seq(lit("0.0"), lit("-0.0"), lit("NaN"), s"-${lit("NaN")}", lit("1.0")) - } - - val types: Seq[FloatType] = Seq( - FloatType("double", "DOUBLE", c = "d", l = "d", r = "e"), - FloatType("float", "FLOAT", c = "f", l = "f", r = "h")) - - case class Case(group: String, context: String, variant: String, tpe: FloatType, sql: String) { - def name: String = - s"$group: $context${if (variant.isEmpty) "" else s", $variant"} [${tpe.name}]" - } - - case class KnownGap(issue: String, reason: String, covers: Case => Boolean) - - private def render(template: String, t: FloatType): String = - template - .replace("{c}", t.c) - .replace("{l}", t.l) - .replace("{r}", t.r) - .replace("{T}", t.sqlType) - - val comparisonOps: Seq[String] = Seq("=", "<=>", "!=", "<", "<=", ">", ">=") - - /** - * Operator contexts that host a comparison of two columns, `{l} {op} {r}` over `fs_p`, or of - * one column with itself over `fs_e` for the nested loop join. - */ - val comparisonContexts: Seq[(String, String)] = Seq( - "Project" -> "SELECT id, {l} {op} {r} FROM fs_p", - "Filter" -> "SELECT id FROM fs_p WHERE {l} {op} {r}", - "aggregate argument" -> "SELECT id, max(IF({l} {op} {r}, 1, 0)) FROM fs_p GROUP BY id", - "aggregate FILTER" -> - "SELECT id, count(*) FILTER (WHERE {l} {op} {r}) FROM fs_p GROUP BY id", - "broadcast hash join condition" -> - "SELECT /*+ BROADCAST(b) */ a.id FROM fs_p a JOIN fs_p b ON a.id = b.id AND a.{l} {op} b.{r}", - "shuffled hash join condition" -> - ("SELECT /*+ SHUFFLE_HASH(b) */ a.id FROM fs_p a JOIN fs_p b " + - "ON a.id = b.id AND a.{l} {op} b.{r}"), - "sort merge join condition" -> - "SELECT /*+ MERGE(b) */ a.id FROM fs_p a JOIN fs_p b ON a.id = b.id AND a.{l} {op} b.{r}", - "nested loop join condition" -> - "SELECT /*+ BROADCAST(b) */ a.id, b.id FROM fs_e a JOIN fs_e b ON a.{c} {op} b.{c}", - "sort key" -> "SELECT id FROM fs_p ORDER BY {l} {op} {r}, id", - "Generate" -> "SELECT id, x FROM fs_p LATERAL VIEW explode(array({l} {op} {r})) t AS x", - "window function argument" -> - "SELECT id, max(IF({l} {op} {r}, 1, 0)) OVER (PARTITION BY id % 7) FROM fs_p", - "window partition key" -> "SELECT id, count(*) OVER (PARTITION BY {l} {op} {r}) FROM fs_p", - "array operands, Project" -> "SELECT id, array({l}) {op} array({r}) FROM fs_p", - "array operands, aggregate argument" -> - "SELECT id, max(IF(array({l}) {op} array({r}), 1, 0)) FROM fs_p GROUP BY id", - "struct operands, Project" -> - "SELECT id, named_struct('x', {l}) {op} named_struct('x', {r}) FROM fs_p", - "struct operands, aggregate argument" -> - ("SELECT id, max(IF(named_struct('x', {l}) {op} named_struct('x', {r}), 1, 0)) " + - "FROM fs_p GROUP BY id")) - - private def comparisonCases: Seq[Case] = - for { - t <- types - (context, template) <- comparisonContexts - op <- comparisonOps - } yield Case("comparison", context, op, t, render(template.replace("{op}", op), t)) - - /** A column compared with each edge literal, on either side. */ - private def literalCases: Seq[Case] = - for { - t <- types - op <- comparisonOps - (context, from, where) <- Seq( - ("Project", "fs_e", false), - ("Filter", "fs_e", true), - // A stored column compared with a literal is pushed into the Parquet reader. - ("scan filter", "fs_edge_raw", true)) - } yield { - // One UNION ALL branch per predicate. With all ten `=` predicates in one projection, - // Spark 4.1.3's whole-stage codegen returns false for `0.0 = 0.0`, and is right with - // subexpression elimination turned off. - val preds = t.literals.flatMap(lit => Seq(s"{c} $op $lit", s"$lit $op {c}")) - val sql = preds.zipWithIndex - .map { case (p, i) => - if (where) s"SELECT id, $i FROM $from WHERE $p" else s"SELECT id, $i, $p FROM $from" - } - .mkString(" UNION ALL ") - Case("literal comparison", context, op, t, render(sql, t)) - } - - /** Contexts where Spark normalizes a float key, or compares it with SQL ordering. */ - val keyContexts: Seq[(String, String)] = Seq( - "GROUP BY" -> "SELECT {c}, count(*) FROM fs_e GROUP BY {c}", - "DISTINCT" -> "SELECT DISTINCT {c} FROM fs_e", - "count(DISTINCT)" -> "SELECT count(DISTINCT {c}) FROM fs_e", - "broadcast hash join key" -> - "SELECT /*+ BROADCAST(b) */ a.id, b.id FROM fs_e a JOIN fs_e b ON a.{c} = b.{c}", - "shuffled hash join key" -> - "SELECT /*+ SHUFFLE_HASH(b) */ a.id, b.id FROM fs_e a JOIN fs_e b ON a.{c} = b.{c}", - "sort merge join key" -> - "SELECT /*+ MERGE(b) */ a.id, b.id FROM fs_e a JOIN fs_e b ON a.{c} = b.{c}", - "null-safe join key" -> "SELECT a.id, b.id FROM fs_e a JOIN fs_e b ON a.{c} <=> b.{c}", - "left semi join key" -> - ("SELECT a.id FROM fs_e a LEFT SEMI JOIN (SELECT * FROM fs_e WHERE id IN (2, 4)) b " + - "ON a.{c} = b.{c}"), - "left anti join key" -> - ("SELECT a.id FROM fs_e a LEFT ANTI JOIN (SELECT * FROM fs_e WHERE id IN (2, 4)) b " + - "ON a.{c} = b.{c}"), - "INTERSECT" -> "SELECT {c} FROM fs_e INTERSECT SELECT {c} FROM fs_e WHERE id IN (2, 4)", - "EXCEPT" -> "SELECT {c} FROM fs_e EXCEPT SELECT {c} FROM fs_e WHERE id IN (2, 4)", - "IN list" -> - "SELECT id FROM fs_e WHERE {c} IN (CAST('0.0' AS {T}), CAST('NaN' AS {T}))", - "IN subquery" -> "SELECT id FROM fs_e WHERE {c} IN (SELECT {c} FROM fs_e WHERE id IN (2, 4))", - "ORDER BY" -> "SELECT id FROM fs_e ORDER BY {c}, id", - "ORDER BY DESC" -> "SELECT id FROM fs_e ORDER BY {c} DESC NULLS LAST, id", - "window partition key" -> "SELECT id, count(*) OVER (PARTITION BY {c}) FROM fs_e", - "window order key, rank" -> "SELECT id, rank() OVER (ORDER BY {c}) FROM fs_e", - "window order key, dense_rank" -> "SELECT id, dense_rank() OVER (ORDER BY {c}) FROM fs_e", - "hash repartition" -> - "SELECT id, spark_partition_id() FROM (SELECT /*+ REPARTITION(4, {c}) */ * FROM fs_e)", - "nested GROUP BY" -> "SELECT array({c}), count(*) FROM fs_e GROUP BY array({c})", - "nested ORDER BY" -> "SELECT id FROM fs_e ORDER BY array({c}), id", - "nested window order key" -> - "SELECT id, rank() OVER (ORDER BY named_struct('x', {c})) FROM fs_e") - - private def keyCases: Seq[Case] = - for { - t <- types - (context, template) <- keyContexts - } yield Case("key", context, "", t, render(template, t)) - - /** Expressions with float input, over one column of `fs_e` or two columns of `fs_p`. */ - val expressions: Seq[(String, String)] = Seq( - "abs" -> "SELECT id, abs({c}) FROM fs_e", - "negative" -> "SELECT id, -{c} FROM fs_e", - "signum" -> "SELECT id, signum({c}) FROM fs_e", - "ceil" -> "SELECT id, ceil({c}) FROM fs_e", - "floor" -> "SELECT id, floor({c}) FROM fs_e", - "round" -> "SELECT id, round({c}, 1), bround({c}, 1) FROM fs_e", - "sqrt" -> "SELECT id, sqrt({c}) FROM fs_e", - "isnan" -> "SELECT id, isnan({c}) FROM fs_e", - "cast to string" -> "SELECT id, CAST({c} AS STRING) FROM fs_e", - "cast to integral" -> "SELECT id, try_cast({c} AS INT), try_cast({c} AS BIGINT) FROM fs_e", - "cast to decimal" -> "SELECT id, CAST({c} AS DECIMAL(10, 2)) FROM fs_e", - "cast to boolean" -> "SELECT id, CAST({c} AS BOOLEAN) FROM fs_e", - "cast between float types" -> "SELECT id, CAST({c} AS DOUBLE), CAST({c} AS FLOAT) FROM fs_e", - "cast array to string" -> "SELECT id, CAST(array({c}) AS STRING) FROM fs_e", - "hash" -> "SELECT id, hash({c}) FROM fs_e", - "xxhash64" -> "SELECT id, xxhash64({c}) FROM fs_e", - "hash of array" -> "SELECT id, hash(array({c})), xxhash64(array({c})) FROM fs_e", - "hash of struct" -> - "SELECT id, hash(named_struct('x', {c})), xxhash64(named_struct('x', {c})) FROM fs_e", - "greatest" -> "SELECT id, greatest({l}, {r}) FROM fs_p", - "least" -> "SELECT id, least({l}, {r}) FROM fs_p", - "nanvl" -> "SELECT id, nanvl({l}, {r}) FROM fs_p", - "IN with column list" -> "SELECT id, {l} IN ({r}, CAST('2.0' AS {T})) FROM fs_p", - "CASE value WHEN" -> "SELECT id, CASE {l} WHEN {r} THEN 1 ELSE 0 END FROM fs_p", - "array_contains" -> "SELECT id, array_contains(array({l}, 2), {r}) FROM fs_p", - "array_position" -> "SELECT id, array_position(array(2, {l}), {r}) FROM fs_p", - "array_remove" -> "SELECT id, array_remove(array({l}, 2), {r}) FROM fs_p", - "arrays_overlap" -> "SELECT id, arrays_overlap(array({l}), array({r})) FROM fs_p", - "array_distinct" -> "SELECT id, array_distinct(array({l}, {r})) FROM fs_p", - "array_union" -> "SELECT id, array_union(array({l}), array({r})) FROM fs_p", - "array_intersect" -> "SELECT id, array_intersect(array({l}), array({r})) FROM fs_p", - "array_except" -> "SELECT id, array_except(array({l}), array({r})) FROM fs_p", - "sort_array" -> "SELECT id, sort_array(array({l}, {r}, 2)) FROM fs_p", - "sort_array descending" -> "SELECT id, sort_array(array({l}, {r}, 2), false) FROM fs_p", - "array_sort" -> "SELECT id, array_sort(array({l}, {r}, 2)) FROM fs_p", - "array_max" -> "SELECT id, array_max(array({l}, {r})) FROM fs_p", - "array_min" -> "SELECT id, array_min(array({l}, {r})) FROM fs_p", - "map lookup" -> - "SELECT id, map({l}, 1)[{r}], element_at(map({l}, 1), {r}) FROM fs_p WHERE {l} IS NOT NULL", - "map_contains_key" -> - "SELECT id, map_contains_key(map({l}, 1), {r}) FROM fs_p WHERE {l} IS NOT NULL") - - private def expressionCases: Seq[Case] = - for { - t <- types - (name, template) <- expressions - } yield Case("expression", name, "", t, render(template, t)) - - /** - * Each pair is expanded from one input row, so its two values reach the same partial aggregate - * in array order. Window tests also order by pos to preserve signed-zero ties after a shuffle. - */ - private val pairGroups = - "(SELECT id, pos, x FROM fs_p LATERAL VIEW posexplode(array({l}, {r})) t AS pos, x)" - - /** Each pair as (l, r, r), so the mode is unique under every Spark equality rule. */ - private val tripleGroups = - "(SELECT id, x FROM fs_p LATERAL VIEW explode(array({l}, {r}, {r})) t AS x)" - - val aggregates: Seq[(String, String)] = Seq( - "max" -> s"SELECT id, max(x) FROM $pairGroups GROUP BY id", - "min" -> s"SELECT id, min(x) FROM $pairGroups GROUP BY id", - "max_by" -> s"SELECT id, max_by(pos, x) FROM $pairGroups GROUP BY id", - "min_by" -> s"SELECT id, min_by(pos, x) FROM $pairGroups GROUP BY id", - "sum" -> s"SELECT id, sum(x) FROM $pairGroups GROUP BY id", - "avg" -> s"SELECT id, avg(x) FROM $pairGroups GROUP BY id", - "count(DISTINCT)" -> s"SELECT id, count(DISTINCT x) FROM $pairGroups GROUP BY id", - "approx_count_distinct" -> - s"SELECT id, approx_count_distinct(x) FROM $pairGroups GROUP BY id", - "collect_set" -> s"SELECT id, size(collect_set(x)) FROM $pairGroups GROUP BY id", - "mode" -> s"SELECT id, mode(x) FROM $tripleGroups GROUP BY id", - "global max and min" -> "SELECT max({c}), min({c}) FROM fs_e", - "global max_by and min_by" -> "SELECT max_by(id, {c}), min_by(id, {c}) FROM fs_e", - "window max and min" -> - (s"SELECT id, pos, max(x) OVER w, min(x) OVER w FROM $pairGroups " + - "WINDOW w AS (PARTITION BY id ORDER BY pos " + - "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)")) - - private def aggregateCases: Seq[Case] = - for { - t <- types - (name, template) <- aggregates - } yield Case("aggregate", name, "", t, render(template, t)) - - val cases: Seq[Case] = - comparisonCases ++ literalCases ++ keyCases ++ expressionCases ++ aggregateCases - - private def issue(n: Int): String = s"https://github.com/apache/datafusion-comet/issues/$n" - - private def in(group: String, contexts: String*)(c: Case): Boolean = - c.group == group && contexts.contains(c.context) - - val knownGaps: Seq[KnownGap] = Seq( - KnownGap( - issue(5312), - "collect_set before Spark 4.2 treats -0.0 and 0.0 as one value and NaNs as distinct.", - c => in("aggregate", "collect_set")(c) && !CometSparkSessionExtensions.isSpark42Plus)) } From 47ced43067cfeb2df45e2c9135acd3aa4a65b228 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 19 Oct 2026 16:16:48 +0100 Subject: [PATCH 2/2] test: drop no-op ConstantFolding directives and unused setup from the float sweep Keep ConstantFolding on only in the three fixtures whose queries hold a foldable literal. Drop the unused pos column from the collect_set queries, build the Scala suite's pair table from an inline VALUES CTE, and trim prose that repeated the fixture headers. --- docs/source/contributor-guide/floating_point.md | 12 +++++------- .../sql-tests/float_semantics/aggregates.sql | 1 - .../aggregates_collect_set_4_2.sql | 5 ++--- .../sql-tests/float_semantics/comparison.sql | 5 ----- .../float_semantics/comparison_joins.sql | 1 - .../float_semantics/comparison_nested.sql | 1 - .../sql-tests/float_semantics/expressions.sql | 3 +++ .../resources/sql-tests/float_semantics/keys.sql | 3 +++ .../apache/comet/CometFloatSemanticsSuite.scala | 16 +++++----------- 9 files changed, 18 insertions(+), 29 deletions(-) diff --git a/docs/source/contributor-guide/floating_point.md b/docs/source/contributor-guide/floating_point.md index a402342ffc6..635a12b4abf 100644 --- a/docs/source/contributor-guide/floating_point.md +++ b/docs/source/contributor-guide/floating_point.md @@ -203,13 +203,11 @@ and only for a type that contains a `FLOAT` or `DOUBLE`. Scala. - Add the expression to the sweep in `float_semantics/`, which crosses the edge values with operator contexts and with every expression and aggregate that accepts floating-point input. - `comparison.sql`, `comparison_joins.sql` and `comparison_nested.sql` cover comparisons, - `literal_comparison.sql` compares a column with literals, and `keys.sql` covers grouping, join, - sort and window keys. A new expression goes in `expressions.sql`, or in `aggregates.sql` for an - aggregate, with one query for `DOUBLE` and one for `FLOAT`. If one does not match Spark yet, - mark its queries `ignore()`. A SQL file test cannot expect a difference, so a gap that - should fail once it is fixed goes in `CometFloatSemanticsSuite`, whose test for `collect_set` - before Spark 4.2 fails when the answers start to match. + A new expression goes in `expressions.sql`, or in `aggregates.sql` for an aggregate, with one + query for `DOUBLE` and one for `FLOAT`. If one does not match Spark yet, mark its queries + `ignore()`. A SQL file test cannot expect a difference, so a gap that should fail once it + is fixed goes in `CometFloatSemanticsSuite`, whose test for `collect_set` before Spark 4.2 fails + when the answers start to match. Fixtures to start from include `expressions/conditional/float_comparisons.sql`, `expressions/math/nan_divisor.sql`, `expressions/math/greatest_least_floating_point.sql`, diff --git a/spark/src/test/resources/sql-tests/float_semantics/aggregates.sql b/spark/src/test/resources/sql-tests/float_semantics/aggregates.sql index 17a7509298a..26241233d0e 100644 --- a/spark/src/test/resources/sql-tests/float_semantics/aggregates.sql +++ b/spark/src/test/resources/sql-tests/float_semantics/aggregates.sql @@ -25,7 +25,6 @@ -- -- collect_set is in aggregates_collect_set_4_2.sql. --- ConstantFolding: enabled statement CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet diff --git a/spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql b/spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql index 16fe8e9b46f..095e58a06f7 100644 --- a/spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql +++ b/spark/src/test/resources/sql-tests/float_semantics/aggregates_collect_set_4_2.sql @@ -23,7 +23,6 @@ -- -- Each pair of edge values is expanded from one input row (see aggregates.sql). --- ConstantFolding: enabled -- MinSparkVersion: 4.2 statement @@ -69,9 +68,9 @@ FROM fs_edge_float a CROSS JOIN fs_edge_float b -- DOUBLE query WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_double) -SELECT id, size(collect_set(x)) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id +SELECT id, size(collect_set(x)) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r)) t AS x) GROUP BY id -- FLOAT query WITH p AS (SELECT id, IF(lneg, -l, l) AS l, IF(rneg, -r, r) AS r FROM fs_pair_float) -SELECT id, size(collect_set(x)) FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) GROUP BY id +SELECT id, size(collect_set(x)) FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r)) t AS x) GROUP BY id diff --git a/spark/src/test/resources/sql-tests/float_semantics/comparison.sql b/spark/src/test/resources/sql-tests/float_semantics/comparison.sql index e8b93739e8a..68750b2231d 100644 --- a/spark/src/test/resources/sql-tests/float_semantics/comparison.sql +++ b/spark/src/test/resources/sql-tests/float_semantics/comparison.sql @@ -27,12 +27,7 @@ -- -- comparison_joins.sql covers join conditions and comparison_nested.sql covers comparisons of -- arrays and structs. literal_comparison.sql compares a column with a literal. --- --- The fixtures run with Spark's ConstantFolding on, which the sweep always did. The literal --- queries need it for a literal to arrive as a sign-bit NaN, and nothing here is an --- all-literal expression that the default exclusion would keep from being folded. --- ConstantFolding: enabled statement CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet diff --git a/spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql b/spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql index 99e55327ef8..364c36595cf 100644 --- a/spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql +++ b/spark/src/test/resources/sql-tests/float_semantics/comparison_joins.sql @@ -21,7 +21,6 @@ -- alone; its = and <=> make it an equi-join, which Spark plans as a hash join, so they repeat -- the join keys of keys.sql. --- ConstantFolding: enabled statement CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet diff --git a/spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql b/spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql index 5c3f96b6fca..82d85a29db5 100644 --- a/spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql +++ b/spark/src/test/resources/sql-tests/float_semantics/comparison_nested.sql @@ -18,7 +18,6 @@ -- Comparisons of arrays and structs that hold one float, over every ordered pair of the edge -- values (see comparison.sql). Spark compares the elements and fields with SQL ordering. --- ConstantFolding: enabled statement CREATE TABLE fs_edge_double(id INT, c DOUBLE, neg BOOLEAN) USING parquet diff --git a/spark/src/test/resources/sql-tests/float_semantics/expressions.sql b/spark/src/test/resources/sql-tests/float_semantics/expressions.sql index ac51d6e308e..05441e579ce 100644 --- a/spark/src/test/resources/sql-tests/float_semantics/expressions.sql +++ b/spark/src/test/resources/sql-tests/float_semantics/expressions.sql @@ -24,6 +24,9 @@ -- -- hash and xxhash64 of a float, an array and a struct are in expressions/hash/hash.sql. -- collect_set is in aggregates_collect_set_4_2.sql. +-- +-- ConstantFolding is on so that the IN query reaches Comet with a folded literal, as Spark +-- plans it. -- ConstantFolding: enabled diff --git a/spark/src/test/resources/sql-tests/float_semantics/keys.sql b/spark/src/test/resources/sql-tests/float_semantics/keys.sql index 8984f2e6358..aee50087157 100644 --- a/spark/src/test/resources/sql-tests/float_semantics/keys.sql +++ b/spark/src/test/resources/sql-tests/float_semantics/keys.sql @@ -25,6 +25,9 @@ -- -- Nested ORDER BY and window order keys are in windows/nested_float_order_keys.sql, which -- uses the same edge values. +-- +-- ConstantFolding is on so that the IN list reaches Comet with folded literals, as Spark +-- plans it. -- ConstantFolding: enabled diff --git a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala index 267eb98141d..3645766587a 100644 --- a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala @@ -35,9 +35,7 @@ import org.apache.spark.sql.CometTestBase * fix, or a DataFusion upgrade that changes float behavior, removes the test instead of leaving * it stale. `aggregates_collect_set_4_2.sql` covers Spark 4.2, where the answers match. * - * The edge values are `0.0`, `-0.0`, a canonical NaN and a NaN with the sign bit set, plus `1.0`, - * `-1.0`, both infinities and `NULL`. A Parquet round trip through Spark's writer canonicalizes - * NaN, so the sign-bit NaN is made at query time by negating a stored NaN. + * The edge values and how the sign-bit NaN is made are described in `comparison.sql`. */ class CometFloatSemanticsSuite extends CometTestBase { @@ -45,16 +43,14 @@ class CometFloatSemanticsSuite extends CometTestBase { override def beforeAll(): Unit = { super.beforeAll() - sql("CREATE TABLE fs_edge_raw(id INT, d DOUBLE, f FLOAT, neg BOOLEAN) USING parquet") - sql(s"INSERT INTO fs_edge_raw VALUES ${edgeRows.mkString(", ")}") - sql("""CREATE TABLE fs_pair_raw USING parquet AS + sql(s"""CREATE TABLE fs_pair_raw USING parquet AS + |WITH edge(id, d, f, neg) AS (VALUES ${edgeRows.mkString(", ")}) |SELECT a.id * 10 + b.id AS id, a.d, a.f, a.neg AS dneg, b.d AS e, b.f AS h, b.neg AS eneg - |FROM fs_edge_raw a CROSS JOIN fs_edge_raw b""".stripMargin) + |FROM edge a CROSS JOIN edge b""".stripMargin) } override def afterAll(): Unit = { try { - sql("DROP TABLE IF EXISTS fs_edge_raw") sql("DROP TABLE IF EXISTS fs_pair_raw") } finally { super.afterAll() @@ -65,13 +61,11 @@ class CometFloatSemanticsSuite extends CometTestBase { if (!CometSparkSessionExtensions.isSpark42Plus) { for ((name, l, r) <- Seq(("double", "d", "e"), ("float", "f", "h"))) { test(s"known gap #5312: collect_set before Spark 4.2 [$name]") { - // Each pair is expanded from one input row, so its two values reach the same partial - // aggregate in array order. val query = s"""WITH p AS ( | SELECT id, IF(dneg, -$l, $l) AS l, IF(eneg, -$r, $r) AS r FROM fs_pair_raw) |SELECT id, size(collect_set(x)) - |FROM (SELECT id, pos, x FROM p LATERAL VIEW posexplode(array(l, r)) t AS pos, x) + |FROM (SELECT id, x FROM p LATERAL VIEW explode(array(l, r)) t AS x) |GROUP BY id""".stripMargin val matched = try {