diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index fc15c908c63..e0f4b53e3f6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -994,7 +994,13 @@ public enum Index { "timewrap_test", "timewrap_test", "{\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"host\":{\"type\":\"keyword\"},\"requests\":{\"type\":\"integer\"},\"errors\":{\"type\":\"integer\"}}}}", - "src/test/resources/timewrap_test.json"); + "src/test/resources/timewrap_test.json"), + DATE_HISTOGRAM_TEST( + "date_histogram_test", + "date_histogram_test", + "{\"mappings\":{\"properties\":{\"ts\":{\"type\":\"date\",\"format\":\"yyyy-MM-dd" + + " HH:mm:ss\"},\"category\":{\"type\":\"keyword\"},\"value\":{\"type\":\"integer\"}}}}", + "src/test/resources/date_histogram_test.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java new file mode 100644 index 00000000000..76e154cf2e7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java @@ -0,0 +1,191 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.util.Capability.LEGACY_ENGINE_FALLBACK; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; + +/** + * Execution coverage for {@code date_histogram} and {@code histogram}. The expander unit tests + * assert the AST that gets built; these assert what comes back after analysis, planning and + * pushdown, against 72 documents on fixed timestamps: + * + * 
+ *   00:00 x5 alpha   00:30 x7 beta    01:00 x11 alpha
+ *   01:45 x13 gamma  02:00 x17 beta   03:00 x19 alpha
+ *
+ * + * so hourly grouping must yield 12/24/17/19 and half-hourly 5/7/11/13/17/19. + */ +public class DateHistogramBucketFunctionIT extends SQLIntegTestCase { + + private static final String IDX = "date_histogram_test"; + + @Override + protected void init() throws Exception { + super.init(); + loadIndex(Index.DATE_HISTOGRAM_TEST); + } + + /** The planner rejects {@code GROUP BY }, so the bucket is aliased in a subquery. */ + private static String bucketed(String bucketExpr) { + return "SELECT b, COUNT(*) FROM (SELECT " + + bucketExpr + + " AS b FROM " + + IDX + + ") sub GROUP BY b ORDER BY b"; + } + + @Test + public void hourlyBucketsCarryKeysAndCounts() throws IOException { + JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1h')")); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 12), + rows("2026-01-01 01:00:00", 24), + rows("2026-01-01 02:00:00", 17), + rows("2026-01-01 03:00:00", 19)); + } + + /** A sub-hour interval must split 00:00/00:30 and 01:00/01:45 rather than merge them. */ + @Test + public void halfHourlyBucketsSplitWithinTheHour() throws IOException { + JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='30m')")); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 5), + rows("2026-01-01 00:30:00", 7), + rows("2026-01-01 01:00:00", 11), + rows("2026-01-01 01:30:00", 13), + rows("2026-01-01 02:00:00", 17), + rows("2026-01-01 03:00:00", 19)); + } + + @Test + public void dailyIntervalCollapsesEverythingIntoOneBucket() throws IOException { + JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1d')")); + + verifyDataRows(response, rows("2026-01-01 00:00:00", 72)); + } + + /** {@code fixed_interval} and {@code calendar_interval} are accepted as synonyms of interval. */ + @Test + public void intervalSynonymsProduceTheSameBuckets() throws IOException { + JSONObject viaFixed = + executeQuery(bucketed("date_histogram('field'=ts, 'fixed_interval'='1h')")); + JSONObject viaCalendar = + executeQuery(bucketed("date_histogram('field'=ts, 'calendar_interval'='1h')")); + + for (JSONObject response : new JSONObject[] {viaFixed, viaCalendar}) { + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 12), + rows("2026-01-01 01:00:00", 24), + rows("2026-01-01 02:00:00", 17), + rows("2026-01-01 03:00:00", 19)); + } + } + + /** + * The scan sits in its own derived table because the V2 engine cannot resolve the span's field + * otherwise when a second grouping key is present. + */ + @Test + public void bucketsCombineWithAnAdditionalGroupingKey() throws IOException { + JSONObject response = + executeQuery( + "SELECT b, c, COUNT(*) FROM (SELECT date_histogram('field'=ts, 'interval'='1h') AS b," + + " category AS c FROM (SELECT * FROM " + + IDX + + ") inner_scan) sub GROUP BY b, c ORDER BY b, c"); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", "alpha", 5), + rows("2026-01-01 00:00:00", "beta", 7), + rows("2026-01-01 01:00:00", "alpha", 11), + rows("2026-01-01 01:00:00", "gamma", 13), + rows("2026-01-01 02:00:00", "beta", 17), + rows("2026-01-01 03:00:00", "alpha", 19)); + } + + @Test + public void bucketsRespectAWhereClause() throws IOException { + JSONObject response = + executeQuery( + "SELECT b, COUNT(*) FROM (SELECT date_histogram('field'=ts, 'interval'='1h') AS b FROM " + + IDX + + " WHERE category = 'alpha') sub GROUP BY b ORDER BY b"); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 5), + rows("2026-01-01 01:00:00", 11), + rows("2026-01-01 03:00:00", 19)); + } + + @Test + public void numericHistogramBucketsByInterval() throws IOException { + JSONObject response = + executeQuery( + "SELECT b, COUNT(*) FROM (SELECT histogram('field'=value, 'interval'=20) AS b FROM " + + IDX + + ") sub GROUP BY b ORDER BY b"); + + // value runs 1..72, so the 20-wide buckets hold 19, 20, 20 and 13 documents. + verifyDataRowsInOrder(response, rows(0, 19), rows(20, 20), rows(40, 20), rows(60, 13)); + } + + /** + * Only the legacy V1 engine understands this spelling, and it answered before these names entered + * the V2 grammar. The expander has to keep declining with SyntaxCheckException so it still does. + */ + @Test + @RequiresCapability(LEGACY_ENGINE_FALLBACK) + public void positionalCallReturnsHourlyBuckets() throws IOException { + JSONObject response = + executeQuery( + "SELECT COUNT(*) FROM " + IDX + " GROUP BY date_histogram(field='ts','interval'='1h')"); + + verifyDataRows(response, rows(12), rows(24), rows(17), rows(19)); + } + + /** + * `alias` has no lowering here but the legacy engine implements it, so the query still has to + * answer. CsvFormatResponseIT.dateHistogramTest has asserted this shape for years. + */ + @Test + @RequiresCapability(LEGACY_ENGINE_FALLBACK) + public void callWithAliasParameterReturnsHourlyBuckets() throws IOException { + JSONObject response = + executeQuery( + "SELECT COUNT(*) FROM " + + IDX + + " GROUP BY date_histogram('field'='ts','fixed_interval'='1h','alias'='hours')"); + + verifyDataRows(response, rows(12), rows(24), rows(17), rows(19)); + } + + @Test + @RequiresCapability(LEGACY_ENGINE_FALLBACK) + public void positionalNumericHistogramReturnsBuckets() throws IOException { + JSONObject response = + executeQuery( + "SELECT COUNT(*) FROM " + IDX + " GROUP BY histogram(field='value','interval'='20')"); + + verifyDataRows(response, rows(19), rows(20), rows(20), rows(13)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index 1d4067f1414..f5a5b784e38 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -541,6 +541,17 @@ public enum Capability { * FRONTEND: legacy method-query syntax (regexp_query/wildcard_query) is not in the Calcite * grammar. */ + /** + * FRONTEND: the legacy V1 engine answers call shapes the V2 grammar declines, but only on the + * default route. Requests reach it when RestSQLQueryAction catches a SyntaxCheckException; the + * analytics-engine route enters through RestUnifiedQueryAction, which has no such fallback. + */ + LEGACY_ENGINE_FALLBACK( + "A call shape only the legacy V1 engine understands (e.g. positional" + + " date_histogram(field=, ...), or an `alias` parameter) can't be answered on the" + + " analytics-engine route: reaching that engine depends on RestSQLQueryAction's" + + " SyntaxCheckException fallback, and the analytics route does not go through it."), + LEGACY_METHOD_QUERY( "Legacy method-query syntax (regexp_query/wildcard_query/query/matchquery) is not in the" + " Calcite grammar used by the analytics-engine route."), diff --git a/integ-test/src/test/resources/date_histogram_test.json b/integ-test/src/test/resources/date_histogram_test.json new file mode 100644 index 00000000000..2d43eca9da3 --- /dev/null +++ b/integ-test/src/test/resources/date_histogram_test.json @@ -0,0 +1,144 @@ +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":1} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":2} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":3} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":4} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":5} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":6} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":7} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":8} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":9} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":10} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":11} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":12} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":13} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":14} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":15} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":16} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":17} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":18} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":19} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":20} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":21} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":22} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":23} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":24} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":25} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":26} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":27} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":28} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":29} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":30} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":31} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":32} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":33} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":34} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":35} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":36} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":37} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":38} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":39} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":40} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":41} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":42} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":43} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":44} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":45} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":46} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":47} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":48} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":49} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":50} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":51} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":52} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":53} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":54} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":55} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":56} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":57} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":58} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":59} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":60} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":61} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":62} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":63} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":64} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":65} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":66} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":67} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":68} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":69} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":70} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":71} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":72} diff --git a/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 index 5f7361160b3..5162e6d1e78 100644 --- a/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 @@ -335,6 +335,7 @@ functionCall | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | bucketFunction # bucketFunctionCall ; timestampFunction @@ -396,6 +397,14 @@ highlightFunction : HIGHLIGHT LR_BRACKET relevanceField (COMMA highlightArg)* RR_BRACKET ; +bucketFunction + : bucketFunctionName LR_BRACKET bucketArg (COMMA bucketArg)* RR_BRACKET + ; + +bucketArg + : bucketArgName EQUAL_SYMBOL bucketArgValue + ; + positionFunction : POSITION LR_BRACKET functionArg IN functionArg RR_BRACKET ; @@ -413,6 +422,11 @@ scalarFunctionName | nestedFunctionName ; +bucketFunctionName + : HISTOGRAM + | DATE_HISTOGRAM + ; + specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall @@ -756,6 +770,10 @@ highlightArgName | HIGHLIGHT_PRE_TAGS ; +bucketArgName + : stringLiteral + ; + relevanceFieldAndWeight : field = relevanceField | field = relevanceField weight = relevanceFieldWeight @@ -780,6 +798,11 @@ relevanceArgValue | constant ; +bucketArgValue + : constant + | qualifiedName + ; + highlightArgValue : stringLiteral ; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5b52b9d3387..fa0b5b91ea9 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -368,6 +368,7 @@ functionCall | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | bucketFunction # bucketFunctionCall ; timestampFunction @@ -429,6 +430,14 @@ highlightFunction : HIGHLIGHT LR_BRACKET relevanceField (COMMA highlightArg)* RR_BRACKET ; +bucketFunction + : bucketFunctionName LR_BRACKET bucketArg (COMMA bucketArg)* RR_BRACKET + ; + +bucketArg + : bucketArgName EQUAL_SYMBOL bucketArgValue + ; + positionFunction : POSITION LR_BRACKET functionArg IN functionArg RR_BRACKET ; @@ -446,6 +455,11 @@ scalarFunctionName | nestedFunctionName ; +bucketFunctionName + : HISTOGRAM + | DATE_HISTOGRAM + ; + specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall @@ -789,6 +803,10 @@ highlightArgName | HIGHLIGHT_PRE_TAGS ; +bucketArgName + : stringLiteral + ; + relevanceFieldAndWeight : field = relevanceField | field = relevanceField weight = relevanceFieldWeight @@ -813,6 +831,11 @@ relevanceArgValue | constant ; +bucketArgValue + : constant + | qualifiedName + ; + highlightArgValue : stringLiteral ; diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index e7510f31b7a..c65352335d4 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -20,6 +20,8 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BetweenPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BinaryComparisonPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BooleanContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BucketArgContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BucketFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CaseFuncAlternativeContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CaseFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnFilterContext; @@ -70,14 +72,18 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.RuleNode; @@ -89,6 +95,7 @@ import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchQueryContext; @@ -165,6 +172,130 @@ public UnresolvedExpression visitScalarFunctionCall(ScalarFunctionCallContext ct return buildFunction(ctx.scalarFunctionName().getText(), ctx.functionArgs().functionArg()); } + /** + * Lowers {@code histogram} and {@code date_histogram} to a {@link Span} over the bucketed field. + * The grammar admits only the {@code 'name'=value} form, so the positional spelling the legacy + * engine has always answered never reaches here -- it stays an unknown scalar function, and + * RestSQLQueryAction hands it back to that engine. + */ + @Override + public UnresolvedExpression visitBucketFunctionCall(BucketFunctionCallContext ctx) { + String functionName = + ctx.bucketFunction().bucketFunctionName().getText().toLowerCase(Locale.ROOT); + Map args = new LinkedHashMap<>(); + for (BucketArgContext arg : ctx.bucketFunction().bucketArg()) { + String name = StringUtils.unquoteText(arg.bucketArgName().getText()).toLowerCase(Locale.ROOT); + if (args.put(name, visit(arg.bucketArgValue())) != null) { + throw new SemanticCheckException("Duplicate parameter: " + name); + } + } + + UnresolvedExpression field = requireArg(args, "field", functionName); + UnresolvedExpression missing = args.remove("missing"); + Literal interval = intervalOf(args, functionName); + Literal format = stringArg(args, "format"); + Literal timeZone = stringArg(args, "time_zone"); + + // Anything left is a parameter with no lowering here. Some of them -- alias, min_doc_count, + // order -- are implemented by the legacy engine, so decline in the one way RestSQLQueryAction + // falls back on rather than failing the request outright. + if (!args.isEmpty()) { + throw new SyntaxCheckException( + functionName + " does not accept parameter: " + String.join(", ", args.keySet())); + } + + UnresolvedExpression bucketed = coalesceMissing(normalizeField(field), missing); + if (timeZone != null) { + bucketed = shiftByTimeZone(bucketed, timeZone); + } + Span span = AstDSL.spanFromSpanLengthLiteral(bucketed, interval); + return format == null ? span : new Function("date_format", List.of(span, format)); + } + + private static UnresolvedExpression requireArg( + Map args, String name, String functionName) { + UnresolvedExpression value = args.remove(name); + if (value == null) { + throw new SemanticCheckException(functionName + " requires " + name + " parameter"); + } + return value; + } + + /** + * {@code interval}, {@code fixed_interval} and {@code calendar_interval} are synonyms; exactly + * one must be present. The distinction between calendar and fixed intervals is not preserved. + */ + private static Literal intervalOf(Map args, String functionName) { + List supplied = + Stream.of("interval", "fixed_interval", "calendar_interval") + .map(key -> stringOrNumericArg(args, key)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (supplied.isEmpty()) { + throw new SemanticCheckException( + functionName + " requires one of: interval, fixed_interval, calendar_interval"); + } + if (supplied.size() > 1) { + throw new SemanticCheckException( + functionName + " accepts only one of: interval, fixed_interval, calendar_interval"); + } + return supplied.get(0); + } + + private static Literal stringArg(Map args, String name) { + UnresolvedExpression value = args.remove(name); + if (value == null) { + return null; + } + if (!(value instanceof Literal literal) || literal.getType() != DataType.STRING) { + throw new SemanticCheckException( + name + " must be a string literal (e.g. '1d', '15m'); got " + value); + } + return literal; + } + + private static Literal stringOrNumericArg(Map args, String name) { + UnresolvedExpression value = args.remove(name); + if (value == null) { + return null; + } + if (!(value instanceof Literal literal)) { + throw new SemanticCheckException(name + " must be a literal; got " + value); + } + return literal; + } + + /** A string literal naming a column is coerced so downstream sees a column reference. */ + private static UnresolvedExpression normalizeField(UnresolvedExpression field) { + if (field instanceof Literal literal && literal.getType() == DataType.STRING) { + return AstDSL.qualifiedName(literal.getValue().toString()); + } + return field; + } + + private static UnresolvedExpression coalesceMissing( + UnresolvedExpression field, UnresolvedExpression missing) { + return missing == null ? field : new Function("coalesce", List.of(field, missing)); + } + + /** + * Shifts the field by a {@link ZoneOffset} before bucketing. Validated here so an invalid offset + * is reported rather than surfacing as an arithmetic failure at execution. + */ + private static UnresolvedExpression shiftByTimeZone( + UnresolvedExpression field, Literal timeZone) { + String offset = timeZone.getValue().toString(); + int seconds; + try { + seconds = ZoneOffset.of(offset).getTotalSeconds(); + } catch (RuntimeException e) { + throw new SemanticCheckException( + "time_zone must be a valid offset like '+05:30' or 'Z'; got '" + offset + "'"); + } + return new Function( + "timestampadd", List.of(AstDSL.stringLiteral("SECOND"), AstDSL.intLiteral(seconds), field)); + } + @Override public UnresolvedExpression visitGetFormatFunctionCall(GetFormatFunctionCallContext ctx) { return new Function( diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index aba8023b07e..51f919001a7 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -47,12 +47,16 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.RelevanceFieldList; +import org.opensearch.sql.ast.expression.Span; +import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.WindowFrame; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLLexer; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; @@ -860,6 +864,114 @@ private static String nest(int depth, String base, UnaryOperator wrap) { return expr; } + @Test + public void canBuildDateHistogramAsSpan() { + assertEquals( + new Span(qualifiedName("ts"), intLiteral(1), SpanUnit.H), + buildExprAst("date_histogram('field'=ts, 'interval'='1h')")); + } + + @Test + public void canBuildDateHistogramWithIntervalSynonyms() { + Span expected = new Span(qualifiedName("ts"), intLiteral(1), SpanUnit.D); + assertEquals(expected, buildExprAst("date_histogram('field'=ts, 'fixed_interval'='1d')")); + assertEquals(expected, buildExprAst("date_histogram('field'=ts, 'calendar_interval'='1d')")); + } + + @Test + public void canBuildDateHistogramWithStringFieldName() { + assertEquals( + new Span(qualifiedName("ts"), intLiteral(30), SpanUnit.m), + buildExprAst("date_histogram('field'='ts', 'interval'='30m')")); + } + + /** A numeric literal field is left alone rather than coerced to a column reference. */ + @Test + public void bucketFieldGivenNonStringLiteralIsPassedThrough() { + assertEquals( + new Span(intLiteral(1), intLiteral(10), SpanUnit.NONE), + buildExprAst("histogram('field'=1, 'interval'=10)")); + } + + @Test + public void canBuildNumericHistogramAsSpan() { + assertEquals( + new Span(qualifiedName("age"), intLiteral(10), SpanUnit.NONE), + buildExprAst("histogram('field'=age, 'interval'=10)")); + } + + @Test + public void canBuildDateHistogramWithMissing() { + assertEquals( + new Span( + function("coalesce", qualifiedName("ts"), stringLiteral("1970-01-01")), + intLiteral(1), + SpanUnit.H), + buildExprAst("date_histogram('field'=ts, 'interval'='1h', 'missing'='1970-01-01')")); + } + + @Test + public void canBuildDateHistogramWithTimeZoneShift() { + assertEquals( + new Span( + function( + "timestampadd", stringLiteral("SECOND"), intLiteral(19800), qualifiedName("ts")), + intLiteral(1), + SpanUnit.H), + buildExprAst("date_histogram('field'=ts, 'interval'='1h', 'time_zone'='+05:30')")); + } + + @Test + public void canBuildDateHistogramWithFormat() { + assertEquals( + function( + "date_format", + new Span(qualifiedName("ts"), intLiteral(1), SpanUnit.D), + stringLiteral("yyyy-MM-dd")), + buildExprAst("date_histogram('field'=ts, 'interval'='1d', 'format'='yyyy-MM-dd')")); + } + + /** + * A parameter with no lowering here has to raise SyntaxCheckException -- the one type + * RestSQLQueryAction falls back on -- because the legacy engine implements alias, min_doc_count + * and order, and has answered queries using them for years. + */ + @Test + public void unsupportedBucketParameterDefersToLegacyEngine() { + assertThrows( + SyntaxCheckException.class, + () -> buildExprAst("date_histogram('field'=ts, 'interval'='1d', 'alias'='days')")); + assertThrows( + SyntaxCheckException.class, + () -> buildExprAst("histogram('field'=age, 'interval'=10, 'min_doc_count'=1)")); + } + + /** A bad argument inside a shape we own must not fall back, so the caller sees this message. */ + @Test + public void badBucketArgumentIsReportedRatherThanDeferred() { + assertThrows( + SemanticCheckException.class, () -> buildExprAst("date_histogram('interval'='1d')")); + assertThrows(SemanticCheckException.class, () -> buildExprAst("date_histogram('field'=ts)")); + assertThrows( + SemanticCheckException.class, + () -> buildExprAst("date_histogram('field'=ts, 'interval'='1d', 'fixed_interval'='2d')")); + assertThrows( + SemanticCheckException.class, + () -> buildExprAst("date_histogram('field'=ts, 'interval'='1d', 'time_zone'='nope')")); + assertThrows( + SemanticCheckException.class, + () -> buildExprAst("date_histogram('field'=ts, 'interval'='1d', 'format'=7)")); + assertThrows( + SemanticCheckException.class, + () -> buildExprAst("date_histogram('field'=ts, 'interval'='1d', 'format'=other)")); + assertThrows( + SemanticCheckException.class, + () -> buildExprAst("date_histogram('field'=ts, 'interval'=ts)")); + assertThrows( + SemanticCheckException.class, + () -> buildExprAst("date_histogram('field'=ts, 'interval'='1d', 'interval'='2d')")); + } + private Node buildExprAst(String expr) { return buildExprAst(expr, astExprBuilder); }