Skip to content

Add SQL histogram and date_histogram bucket functions #5700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RyanL1997 wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Add SQL histogram and date_histogram bucket functions #5700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6573786
Add SQL histogram and date_histogram bucket functions
RyanL1997 Aug 17, 2026
cc420ab
Defer every unlowerable bucket call to the legacy engine
RyanL1997 Aug 17, 2026
7072d03
Make the bucket-function ITs behave the same with and without analyti…
RyanL1997 Aug 18, 2026
510eaaf
Gate the legacy-only bucket tests behind a capability instead of drop…
RyanL1997 Aug 18, 2026
b2cdea1
Report a bad argument instead of deferring to the legacy engine
RyanL1997 Aug 19, 2026
b954e10
Handle bucket functions the way this parser handles its other functions
RyanL1997 Aug 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,7 +994,13 @@ public enum Index {
"timewrap_test",
"timewrap_test",
"{\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"host\":{\"type\":\"keyword\"},\"requests\":{\"type\":\"integer\"},\"errors\":{\"type\":\"integer\"}}}}",
"src/test/resources/timewrap_test.json");
"src/test/resources/timewrap_test.json"),
DATE_HISTOGRAM_TEST(
"date_histogram_test",
"date_histogram_test",
"{\"mappings\":{\"properties\":{\"ts\":{\"type\":\"date\",\"format\":\"yyyy-MM-dd"
+ " HH:mm:ss\"},\"category\":{\"type\":\"keyword\"},\"value\":{\"type\":\"integer\"}}}}",
"src/test/resources/date_histogram_test.json");

private final String name;
private final String type;
Expand Down
191 changes: 191 additions & 0 deletions integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.sql;

import static org.opensearch.sql.util.Capability.LEGACY_ENGINE_FALLBACK;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder;

import java.io.IOException;
import org.json.JSONObject;
import org.junit.Test;
import org.opensearch.sql.legacy.SQLIntegTestCase;
import org.opensearch.sql.util.RequiresCapability;

/**
* Execution coverage for {@code date_histogram} and {@code histogram}. The expander unit tests
* assert the AST that gets built; these assert what comes back after analysis, planning and
* pushdown, against 72 documents on fixed timestamps:
*
* <pre>
* 00:00 x5 alpha 00:30 x7 beta 01:00 x11 alpha
* 01:45 x13 gamma 02:00 x17 beta 03:00 x19 alpha
* </pre>
*
* so hourly grouping must yield 12/24/17/19 and half-hourly 5/7/11/13/17/19.
*/
public class DateHistogramBucketFunctionIT extends SQLIntegTestCase {

private static final String IDX = "date_histogram_test";

@Override
protected void init() throws Exception {
super.init();
loadIndex(Index.DATE_HISTOGRAM_TEST);
}

/** The planner rejects {@code GROUP BY <expression>}, so the bucket is aliased in a subquery. */
private static String bucketed(String bucketExpr) {
return "SELECT b, COUNT(*) FROM (SELECT "
+ bucketExpr
+ " AS b FROM "
+ IDX
+ ") sub GROUP BY b ORDER BY b";
}

@Test
public void hourlyBucketsCarryKeysAndCounts() throws IOException {
JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1h')"));

verifyDataRowsInOrder(
response,
rows("2026-01-01 00:00:00", 12),
rows("2026-01-01 01:00:00", 24),
rows("2026-01-01 02:00:00", 17),
rows("2026-01-01 03:00:00", 19));
}

/** A sub-hour interval must split 00:00/00:30 and 01:00/01:45 rather than merge them. */
@Test
public void halfHourlyBucketsSplitWithinTheHour() throws IOException {
JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='30m')"));

verifyDataRowsInOrder(
response,
rows("2026-01-01 00:00:00", 5),
rows("2026-01-01 00:30:00", 7),
rows("2026-01-01 01:00:00", 11),
rows("2026-01-01 01:30:00", 13),
rows("2026-01-01 02:00:00", 17),
rows("2026-01-01 03:00:00", 19));
}

@Test
public void dailyIntervalCollapsesEverythingIntoOneBucket() throws IOException {
JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1d')"));

verifyDataRows(response, rows("2026-01-01 00:00:00", 72));
}

/** {@code fixed_interval} and {@code calendar_interval} are accepted as synonyms of interval. */
@Test
public void intervalSynonymsProduceTheSameBuckets() throws IOException {
JSONObject viaFixed =
executeQuery(bucketed("date_histogram('field'=ts, 'fixed_interval'='1h')"));
JSONObject viaCalendar =
executeQuery(bucketed("date_histogram('field'=ts, 'calendar_interval'='1h')"));

for (JSONObject response : new JSONObject[] {viaFixed, viaCalendar}) {
verifyDataRowsInOrder(
response,
rows("2026-01-01 00:00:00", 12),
rows("2026-01-01 01:00:00", 24),
rows("2026-01-01 02:00:00", 17),
rows("2026-01-01 03:00:00", 19));
}
}

/**
* The scan sits in its own derived table because the V2 engine cannot resolve the span's field
* otherwise when a second grouping key is present.
*/
@Test
public void bucketsCombineWithAnAdditionalGroupingKey() throws IOException {
JSONObject response =
executeQuery(
"SELECT b, c, COUNT(*) FROM (SELECT date_histogram('field'=ts, 'interval'='1h') AS b,"
+ " category AS c FROM (SELECT * FROM "
+ IDX
+ ") inner_scan) sub GROUP BY b, c ORDER BY b, c");

verifyDataRowsInOrder(
response,
rows("2026-01-01 00:00:00", "alpha", 5),
rows("2026-01-01 00:00:00", "beta", 7),
rows("2026-01-01 01:00:00", "alpha", 11),
rows("2026-01-01 01:00:00", "gamma", 13),
rows("2026-01-01 02:00:00", "beta", 17),
rows("2026-01-01 03:00:00", "alpha", 19));
}

@Test
public void bucketsRespectAWhereClause() throws IOException {
JSONObject response =
executeQuery(
"SELECT b, COUNT(*) FROM (SELECT date_histogram('field'=ts, 'interval'='1h') AS b FROM "
+ IDX
+ " WHERE category = 'alpha') sub GROUP BY b ORDER BY b");

verifyDataRowsInOrder(
response,
rows("2026-01-01 00:00:00", 5),
rows("2026-01-01 01:00:00", 11),
rows("2026-01-01 03:00:00", 19));
}

@Test
public void numericHistogramBucketsByInterval() throws IOException {
JSONObject response =
executeQuery(
"SELECT b, COUNT(*) FROM (SELECT histogram('field'=value, 'interval'=20) AS b FROM "
+ IDX
+ ") sub GROUP BY b ORDER BY b");

// value runs 1..72, so the 20-wide buckets hold 19, 20, 20 and 13 documents.
verifyDataRowsInOrder(response, rows(0, 19), rows(20, 20), rows(40, 20), rows(60, 13));
}

/**
* Only the legacy V1 engine understands this spelling, and it answered before these names entered
* the V2 grammar. The expander has to keep declining with SyntaxCheckException so it still does.
*/
@Test
@RequiresCapability(LEGACY_ENGINE_FALLBACK)
public void positionalCallReturnsHourlyBuckets() throws IOException {
JSONObject response =
executeQuery(
"SELECT COUNT(*) FROM " + IDX + " GROUP BY date_histogram(field='ts','interval'='1h')");

@dai-chen dai-chen Aug 18, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you elaborate why this is not supported with current changes? I thought this should be handled by V2 only with grammar changes.


verifyDataRows(response, rows(12), rows(24), rows(17), rows(19));
}

/**
* `alias` has no lowering here but the legacy engine implements it, so the query still has to
* answer. CsvFormatResponseIT.dateHistogramTest has asserted this shape for years.
*/
@Test
@RequiresCapability(LEGACY_ENGINE_FALLBACK)
public void callWithAliasParameterReturnsHourlyBuckets() throws IOException {
JSONObject response =
executeQuery(
"SELECT COUNT(*) FROM "
+ IDX
+ " GROUP BY date_histogram('field'='ts','fixed_interval'='1h','alias'='hours')");

verifyDataRows(response, rows(12), rows(24), rows(17), rows(19));
}

@Test
@RequiresCapability(LEGACY_ENGINE_FALLBACK)
public void positionalNumericHistogramReturnsBuckets() throws IOException {
JSONObject response =
executeQuery(
"SELECT COUNT(*) FROM " + IDX + " GROUP BY histogram(field='value','interval'='20')");

verifyDataRows(response, rows(19), rows(20), rows(20), rows(13));
}
}
11 changes: 11 additions & 0 deletions integ-test/src/test/java/org/opensearch/sql/util/Capability.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,17 @@ public enum Capability {
* FRONTEND: legacy method-query syntax (regexp_query/wildcard_query) is not in the Calcite
* grammar.
*/
/**
* FRONTEND: the legacy V1 engine answers call shapes the V2 grammar declines, but only on the
* default route. Requests reach it when RestSQLQueryAction catches a SyntaxCheckException; the
* analytics-engine route enters through RestUnifiedQueryAction, which has no such fallback.
*/
LEGACY_ENGINE_FALLBACK(
"A call shape only the legacy V1 engine understands (e.g. positional"
+ " date_histogram(field=<col>, ...), or an `alias` parameter) can't be answered on the"
+ " analytics-engine route: reaching that engine depends on RestSQLQueryAction's"
+ " SyntaxCheckException fallback, and the analytics route does not go through it."),

LEGACY_METHOD_QUERY(
"Legacy method-query syntax (regexp_query/wildcard_query/query/matchquery) is not in the"
+ " Calcite grammar used by the analytics-engine route."),
Expand Down
144 changes: 144 additions & 0 deletions integ-test/src/test/resources/date_histogram_test.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
{"index":{}}
{"ts":"2026-01-01 00:00:00","category":"alpha","value":1}
{"index":{}}
{"ts":"2026-01-01 00:00:00","category":"alpha","value":2}
{"index":{}}
{"ts":"2026-01-01 00:00:00","category":"alpha","value":3}
{"index":{}}
{"ts":"2026-01-01 00:00:00","category":"alpha","value":4}
{"index":{}}
{"ts":"2026-01-01 00:00:00","category":"alpha","value":5}
{"index":{}}
{"ts":"2026-01-01 00:30:00","category":"beta","value":6}
{"index":{}}
{"ts":"2026-01-01 00:30:00","category":"beta","value":7}
{"index":{}}
{"ts":"2026-01-01 00:30:00","category":"beta","value":8}
{"index":{}}
{"ts":"2026-01-01 00:30:00","category":"beta","value":9}
{"index":{}}
{"ts":"2026-01-01 00:30:00","category":"beta","value":10}
{"index":{}}
{"ts":"2026-01-01 00:30:00","category":"beta","value":11}
{"index":{}}
{"ts":"2026-01-01 00:30:00","category":"beta","value":12}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":13}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":14}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":15}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":16}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":17}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":18}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":19}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":20}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":21}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":22}
{"index":{}}
{"ts":"2026-01-01 01:00:00","category":"alpha","value":23}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":24}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":25}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":26}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":27}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":28}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":29}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":30}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":31}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":32}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":33}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":34}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":35}
{"index":{}}
{"ts":"2026-01-01 01:45:00","category":"gamma","value":36}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":37}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":38}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":39}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":40}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":41}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":42}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":43}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":44}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":45}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":46}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":47}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":48}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":49}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":50}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":51}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":52}
{"index":{}}
{"ts":"2026-01-01 02:00:00","category":"beta","value":53}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":54}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":55}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":56}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":57}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":58}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":59}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":60}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":61}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":62}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":63}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":64}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":65}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":66}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":67}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":68}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":69}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":70}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":71}
{"index":{}}
{"ts":"2026-01-01 03:00:00","category":"alpha","value":72}
Loading
Loading