From 0f4b3bfe854ce5abf79d8c24b05d4ec7786eb272 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Mon, 31 Aug 2026 15:05:35 +0200 Subject: [PATCH] fix(schema-compiler,query-orchestrator): bound the rollupLambda source query A rollupLambda pre-aggregation with unionWithSourceData issued a live source query on every request. It was lower-bounded by FROM_PARTITION_RANGE and had no upper bound, so it aggregated and CSV-downloaded the entire tail of the source table even when the requested range sat entirely inside already-built partitions - every row of which the outer query then discarded. In the reporter's repro a query returning 696 cost ~330-390 ms at granularity day and ~7.0 s at granularity minute, streaming 1,271,521 rows into a Cube Store inline table per request. Three defects, not the two named in the issue: - buildLambdaQuery() built the sub-query from references.timeDimensions, which carry only {dimension, granularity}, so the requested range never reached it. - loadPreAggregations() downloaded the lambda table whenever a lambdaQuery was present, without checking whether the requested range even reached past the last partition's buildRangeEnd. - Under Tesseract - the default planner - the source query carried no LIMIT at all. query_properties_compiler.rs reads rowLimit as parse::().ok(), so the __MAX_SOURCE_ROW_LIMIT sentinel resolved to None and the maxSourceRowLimit safety valve had silently stopped protecting anything. The row limit is now resolved to a number before the options cross the bridge, and echoed back on LambdaQuery so the orchestrator checks the count against the limit that was actually rendered. Generated source SQL for the reporter's model, dateRange 2024-02-01..2024-02-29: | planner | before | after | | --- | --- | --- | | legacy | `WHERE ts > $1 GROUP BY 1 LIMIT $2` | `WHERE ts > $1 AND (ts >= $2 AND ts <= $3) GROUP BY 1 LIMIT 200000` | | native | `WHERE ts > $1 GROUP BY 1` | `WHERE ts > $1 AND (ts >= $2 AND ts <= $3) GROUP BY 1 LIMIT 200000` | The bound is not a new computation: the matchedTimeDimension/filters block moves out of preAggregationDescriptionFor() into PreAggregations.matchedTimeDimensionDateRangeFor(), so buildLambdaQuery and the orchestrator skip both decide from the range that already selects partitions. lambdaSourceDateRange() wraps it to mirror the multi-usage mergeUsageDateRanges overwrite, and bounds nothing when a usage range is unknown. The inDateRange filter stays separate from the afterDate one on purpose: inDateRange's lower bound is inclusive and would double-count rows sitting exactly at buildRangeEnd. Requests carrying no date range are untouched and still expose all fresh data, which the smoke suite asserts. Fixes #11682 Co-Authored-By: Claude Opus 5 (1M context) --- .../PreAggregationPartitionRangeLoader.ts | 32 +++++- .../src/orchestrator/PreAggregations.ts | 5 + .../test/unit/PreAggregations.test.ts | 105 ++++++++++++++++++ .../src/adapter/BaseQuery.js | 37 ++++-- .../src/adapter/PreAggregations.ts | 105 +++++++++++++----- .../test/unit/pre-aggregations.test.ts | 101 +++++++++++++++++ .../cubejs-testing/test/smoke-lambda.test.ts | 46 ++++++++ 7 files changed, 392 insertions(+), 39 deletions(-) diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts index a19e1924f9220..02c99f2876d21 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts @@ -266,8 +266,17 @@ export class PreAggregationPartitionRangeLoader { if (this.preAggregation.rollupLambdaId) { if (this.lambdaQuery && loadResults.length > 0) { const { buildRangeEnd, targetTableName } = loadResults[loadResults.length - 1]; - const lambdaTypes = await this.loadCache.getTableColumnTypes(this.preAggregation, targetTableName); - lambdaTable = await this.downloadLambdaTable(buildRangeEnd, lambdaTypes); + if (this.lambdaSourceDataCovered(buildRangeEnd)) { + this.logger('Skipping lambda source query', { + preAggregationId: this.preAggregation.preAggregationId, + requestId: this.requestId, + buildRangeEnd, + matchedTimeDimensionDateRange: this.preAggregation.matchedTimeDimensionDateRange, + }); + } else { + const lambdaTypes = await this.loadCache.getTableColumnTypes(this.preAggregation, targetTableName); + lambdaTable = await this.downloadLambdaTable(buildRangeEnd, lambdaTypes); + } } const rollupLambdaResults = this.preAggregationsTablesToTempTables.filter(tempTableResult => tempTableResult[1].rollupLambdaId === this.preAggregation.rollupLambdaId); const filteredResults = loadResults.filter( @@ -368,6 +377,20 @@ export class PreAggregationPartitionRangeLoader { } } + /** + * The lambda query is lower bounded by `buildRangeEnd` (exclusive, via + * `afterDate FROM_PARTITION_RANGE`), so it can only contribute rows strictly after it. + * @see https://github.com/cube-js/cube/issues/11682 + */ + private lambdaSourceDataCovered(buildRangeEnd?: string): boolean { + const matchedRangeEnd = this.preAggregation.matchedTimeDimensionDateRange?.[1]; + if (!matchedRangeEnd || !buildRangeEnd) { + return false; + } + // buildRangeEnd comes back from the DB and may carry a `Z` suffix. + return reformatInIsoLocal(matchedRangeEnd) <= reformatInIsoLocal(buildRangeEnd); + } + /** * Downloads the lambda table from the source DB. */ @@ -399,8 +422,9 @@ export class PreAggregationPartitionRangeLoader { lambdaTypes, } ); - if (data.rowCount === this.options.maxSourceRowLimit) { - throw new Error(`The maximum number of source rows ${this.options.maxSourceRowLimit} was reached for ${this.preAggregation.preAggregationId}`); + const appliedRowLimit = this.lambdaQuery.maxSourceRowLimit ?? this.options.maxSourceRowLimit; + if (data.rowCount === appliedRowLimit) { + throw new Error(`The maximum number of source rows ${appliedRowLimit} was reached for ${this.preAggregation.preAggregationId}`); } return { name: `${LAMBDA_TABLE_PREFIX}_${this.preAggregation.tableName.replace('.', '_')}`, diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts index 084005718b62e..aef7f593aaea5 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts @@ -165,6 +165,11 @@ export type LambdaOptions = { export type LambdaQuery = { sqlAndParams: QueryWithParams, cacheKeyQueries: any[], + /** + * Limit actually rendered into `sqlAndParams`. Undefined for SQL that instead carries the + * MAX_SOURCE_ROW_LIMIT placeholder as a param. + */ + maxSourceRowLimit?: number, }; export type PreAggregationDescription = { diff --git a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts index f1fa0fd82525a..1a14e4331ba4d 100644 --- a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts +++ b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts @@ -796,6 +796,111 @@ describe('PreAggregations', () => { }); }); + // @link https://github.com/cube-js/cube/issues/11682 + describe('lambda source query loading', () => { + const buildRangeEnd = '2024-01-02T23:59:59.999'; + + // The PreAggregationLoader.prototype spy below would otherwise leak into later tests. + afterEach(() => { + jest.restoreAllMocks(); + }); + + const createLambdaLoader = (matchedTimeDimensionDateRange?: [string, string]) => { + const loader = new PreAggregationPartitionRangeLoader( + {} as any, // driverFactory + // eslint-disable-next-line @typescript-eslint/no-empty-function + () => {}, // logger + { options: {} } as any, // queryCache + {} as any, // preAggregations + mockPreAggregation({ + preAggregationId: 'Orders.d', + rollupLambdaId: 'Orders.d_lambda', + lastRollupLambda: true, + unionWithSourceData: true, + matchedTimeDimensionDateRange, + }) as any, + [], // preAggregationsTablesToTempTables + { getTableColumnTypes: jest.fn().mockResolvedValue([{ name: 'ts', type: 'timestamp' }]) } as any, + { + lambdaQuery: { + sqlAndParams: ['SELECT * FROM public.orders WHERE ts > ?', [FROM_PARTITION_RANGE]], + cacheKeyQueries: [], + }, + } as any, + ); + + jest.spyOn(loader as any, 'partitionRanges').mockResolvedValue({ + buildRange: ['2024-01-01T00:00:00.000', buildRangeEnd], + partitionRanges: [['2024-01-02T00:00:00.000', buildRangeEnd]], + }); + jest.spyOn(PreAggregationLoader.prototype, 'loadPreAggregation').mockResolvedValue({ + targetTableName: 'stb_pre_aggregations.orders_d20240102_abc_def', + refreshKeyValues: [], + lastUpdatedAt: 1, + buildRangeEnd, + } as any); + const downloadLambdaTable = jest.spyOn(loader as any, 'downloadLambdaTable').mockResolvedValue({ + name: 'lambda_stb_pre_aggregations_orders_d', + columns: [], + csvRows: '', + }); + + return { loader, downloadLambdaTable }; + }; + + test('skips the source query when the requested range is inside the built range', async () => { + const { loader, downloadLambdaTable } = createLambdaLoader(['2024-01-01T00:00:00.000', buildRangeEnd]); + + const result: any = await loader.loadPreAggregations(); + + expect(downloadLambdaTable).not.toHaveBeenCalled(); + expect(result.lambdaTable).toBeUndefined(); + expect(result.targetTableName).toEqual('stb_pre_aggregations.orders_d20240102_abc_def'); + }); + + test('runs the source query when the requested range extends past the built range', async () => { + const { loader, downloadLambdaTable } = createLambdaLoader(['2024-01-01T00:00:00.000', '2024-01-05T23:59:59.999']); + + const result: any = await loader.loadPreAggregations(); + + expect(downloadLambdaTable).toHaveBeenCalledWith(buildRangeEnd, [{ name: 'ts', type: 'timestamp' }]); + expect(result.lambdaTable?.name).toEqual('lambda_stb_pre_aggregations_orders_d'); + expect(result.targetTableName).toMatch(/UNION ALL SELECT \* FROM lambda_stb_pre_aggregations_orders_d/); + }); + + test('runs the source query when no date range was requested', async () => { + const { loader, downloadLambdaTable } = createLambdaLoader(undefined); + + await loader.loadPreAggregations(); + + expect(downloadLambdaTable).toHaveBeenCalled(); + }); + }); + + describe('lambdaSourceDataCovered', () => { + const covered = (matchedTimeDimensionDateRange: any, buildRangeEnd: any) => ( + createLoader({ matchedTimeDimensionDateRange }) as any + ).lambdaSourceDataCovered(buildRangeEnd); + + test('covered when the requested range ends within the built range', () => { + expect(covered(['2024-01-01T00:00:00.000', '2024-01-02T23:59:59.999'], '2024-01-02T23:59:59.999')).toBe(true); + expect(covered(['2024-01-01T00:00:00.000', '2024-01-02T23:59:59.999'], '2024-01-03T23:59:59.999')).toBe(true); + }); + + test('not covered when the requested range extends past the built range', () => { + expect(covered(['2024-01-01T00:00:00.000', '2024-01-05T23:59:59.999'], '2024-01-03T23:59:59.999')).toBe(false); + }); + + test('normalizes a buildRangeEnd read back from the DB with a Z suffix', () => { + expect(covered(['2024-01-01T00:00:00.000', '2024-01-02T23:59:59.999'], '2024-01-02T23:59:59.999Z')).toBe(true); + }); + + test('keeps the source query when either bound is unknown', () => { + expect(covered(undefined, '2024-01-02T23:59:59.999')).toBe(false); + expect(covered(['2024-01-01T00:00:00.000', '2024-01-02T23:59:59.999'], undefined)).toBe(false); + }); + }); + describe('partitionTableName', () => { test('should generate correct table names for different granularities', () => { const testDateRange: [string, string] = ['2024-01-05T12:34:56.789', '2024-01-05T23:59:59.999']; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 77a772f98fd57..0d454cd94e2bb 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -1093,6 +1093,16 @@ export class BaseQuery { const lambdaPreAgg = preAggForQuery.referencedPreAggregations[preAggForQuery.referencedPreAggregations.length - 1]; // TODO(cristipp) Use source query instead of preaggregation references. const references = this.cubeEvaluator.evaluatePreAggregationReferences(lambdaPreAgg.cube, lambdaPreAgg.preAggregation); + const [timeDimension] = references.timeDimensions; + // @see https://github.com/cube-js/cube/issues/11682 + const sourceDateRange = timeDimension && + this.preAggregations.lambdaSourceDateRange(lambdaPreAgg, preAggForQuery); + // Tesseract renders LIMIT as an inline number and can't carry the MAX_SOURCE_ROW_LIMIT + // placeholder (it parses to None, emitting no LIMIT at all), so resolve it here. + const resolvedRowLimit = this.options.maxSourceRowLimit ?? getEnv('maxSourceRowLimit'); + const maxSourceRowLimit = typeof resolvedRowLimit === 'number' && resolvedRowLimit > 0 + ? resolvedRowLimit + : undefined; const lambdaQuery = this.newSubQuery( { measures: references.measures, @@ -1100,19 +1110,24 @@ export class BaseQuery { timeDimensions: references.timeDimensions, filters: [ ...this.options.filters ?? [], - references.timeDimensions.length > 0 - ? { - member: references.timeDimensions[0].dimension, - operator: 'afterDate', - values: [FROM_PARTITION_RANGE] - } - : [], + ...(timeDimension ? [{ + member: timeDimension.dimension, + operator: 'afterDate', + values: [FROM_PARTITION_RANGE] + }] : []), + // Kept separate from the afterDate filter on purpose: inDateRange's lower bound is + // inclusive and would double count rows sitting exactly at the partition end. + ...(sourceDateRange ? [{ + member: timeDimension.dimension, + operator: 'inDateRange', + values: sourceDateRange + }] : []), ], segments: this.options.segments, order: [], limit: undefined, offset: undefined, - rowLimit: MAX_SOURCE_ROW_LIMIT, + rowLimit: maxSourceRowLimit ?? MAX_SOURCE_ROW_LIMIT, preAggregationQuery: true, } ); @@ -1121,7 +1136,11 @@ export class BaseQuery { () => this.cacheKeyQueries(), { preAggregationQuery: true } ); - result[this.preAggregations.preAggregationId(lambdaPreAgg)] = { sqlAndParams, cacheKeyQueries }; + result[this.preAggregations.preAggregationId(lambdaPreAgg)] = { + sqlAndParams, + cacheKeyQueries, + maxSourceRowLimit, + }; } return result; } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 5fa4174ebbb44..7195dd1172652 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -313,20 +313,24 @@ export class PreAggregations { return []; } - private preAggregationDescriptionFor(cube: string, foundPreAggregation: PreAggregationForQuery): FullPreAggregationDescription { - const { preAggregationName, preAggregation, references } = foundPreAggregation; + /** + * Requested time dimension range this pre-aggregation was matched by, as local + * (timezone naked) ISO strings, e.g. ['2024-02-01T00:00:00.000', '2024-02-29T23:59:59.999']. + * Drives partition selection, and bounds the rollupLambda source query so both halves of + * the union agree. + */ + public matchedTimeDimensionDateRangeFor(foundPreAggregation: PreAggregationForQuery): [string, string] | undefined { + const { preAggregation } = foundPreAggregation; - const tableName = this.preAggregationTableName(cube, preAggregationName, preAggregation); - const invalidateKeyQueries = this.query.preAggregationInvalidateKeyQueries(cube, preAggregation, preAggregationName); - const queryForSqlEvaluation = this.query.preAggregationQueryForSqlEvaluation(cube, preAggregation); - // Atm this is only defined in KsqlQuery but without it partitions are recreated on every refresh - const partitionInvalidateKeyQueries = queryForSqlEvaluation.partitionInvalidateKeyQueries?.(cube, preAggregation); + if (!preAggregation.partitionGranularity) { + return undefined; + } const allBackAliasMembers = this.query.allBackAliasMembers(); let matchedTimeDimension: BaseTimeDimension | undefined; - if (preAggregation.partitionGranularity && !this.hasCumulativeMeasures()) { + if (!this.hasCumulativeMeasures()) { matchedTimeDimension = this.query.timeDimensions.find(td => { if (!td.dateRange) { return false; @@ -349,24 +353,77 @@ export class PreAggregations { }); } - let filters: BaseFilter[] | undefined; + const filters = this.query.filters?.filter((td): td is BaseFilter => { + // TODO support all date operators + if (td.isDateOperator() && 'camelizeOperator' in td && td.camelizeOperator === 'inDateRange') { + if (td.dimension === foundPreAggregation.references.timeDimensions[0].dimension) { + return true; + } - if (preAggregation.partitionGranularity) { - filters = this.query.filters?.filter((td): td is BaseFilter => { - // TODO support all date operators - if (td.isDateOperator() && 'camelizeOperator' in td && td.camelizeOperator === 'inDateRange') { - if (td.dimension === foundPreAggregation.references.timeDimensions[0].dimension) { - return true; - } + // Handling for views + return td.dimension === allBackAliasMembers[foundPreAggregation.references.timeDimensions[0].dimension]; + } - // Handling for views - return td.dimension === allBackAliasMembers[foundPreAggregation.references.timeDimensions[0].dimension]; - } + return false; + }); - return false; - }); + return matchedTimeDimension?.boundaryDateRangeFormatted() || + filters?.[0]?.formattedDateRange() || // TODO intersect all date ranges + undefined; + } + + /** + * Mirrors the merge done in preAggregationDescriptionsForUsageInfos(), otherwise a forward + * shifted usage would be bounded tighter than the partitions it unions with and lose + * source rows. + */ + public lambdaSourceDateRange( + lambdaPreAggregation: PreAggregationForQuery, + rollupLambda: PreAggregationForQuery + ): [string, string] | undefined { + const matchedDateRange = this.matchedTimeDimensionDateRangeFor(lambdaPreAggregation); + + if (!matchedDateRange) { + return undefined; + } + + const usageInfos = (this.preAggregationUsageInfos || []).filter( + usageInfo => usageInfo.cubeName === rollupLambda.cube && + usageInfo.preAggregationName === rollupLambda.preAggregationName + ); + + if (usageInfos.length === 0) { + return matchedDateRange; } + let merged: [string, string] | undefined; + + for (const usageInfo of usageInfos) { + const usageDateRange = PreAggregations.mergeUsageDateRanges(usageInfo.usages); + if (!usageDateRange) { + // An unknown usage range may need anything, so bound nothing. + return undefined; + } + merged = merged + ? [ + usageDateRange[0] < merged[0] ? usageDateRange[0] : merged[0], + usageDateRange[1] > merged[1] ? usageDateRange[1] : merged[1], + ] + : usageDateRange; + } + + return merged; + } + + private preAggregationDescriptionFor(cube: string, foundPreAggregation: PreAggregationForQuery): FullPreAggregationDescription { + const { preAggregationName, preAggregation, references } = foundPreAggregation; + + const tableName = this.preAggregationTableName(cube, preAggregationName, preAggregation); + const invalidateKeyQueries = this.query.preAggregationInvalidateKeyQueries(cube, preAggregation, preAggregationName); + const queryForSqlEvaluation = this.query.preAggregationQueryForSqlEvaluation(cube, preAggregation); + // Atm this is only defined in KsqlQuery but without it partitions are recreated on every refresh + const partitionInvalidateKeyQueries = queryForSqlEvaluation.partitionInvalidateKeyQueries?.(cube, preAggregation); + const uniqueKeyColumnsDefault = () => null; const uniqueKeyColumns = ({ rollup: () => queryForSqlEvaluation.preAggregationUniqueKeyColumns(cube, preAggregation), @@ -403,11 +460,7 @@ export class PreAggregations { preAggregationStartEndQueries: (preAggregation.partitionGranularity || references.timeDimensions[0]?.granularity) && this.refreshRangeQuery(cube).preAggregationStartEndQueries(cube, preAggregation), - matchedTimeDimensionDateRange: - preAggregation.partitionGranularity && ( - matchedTimeDimension?.boundaryDateRangeFormatted() || - filters?.[0]?.formattedDateRange() // TODO intersect all date ranges - ), + matchedTimeDimensionDateRange: this.matchedTimeDimensionDateRangeFor(foundPreAggregation), indexesSql: Object.keys(preAggregation.indexes || {}) .map( index => { diff --git a/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts index 25d3a02d8f00b..403f433cc3212 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts @@ -1,5 +1,6 @@ import fs from 'fs'; import path from 'path'; +import { FROM_PARTITION_RANGE, MAX_SOURCE_ROW_LIMIT } from '@cubejs-backend/shared'; import { prepareJsCompiler, prepareYamlCompiler } from './PrepareCompiler'; import { createECommerceSchema, createSchemaYaml } from './utils'; import { PostgresQuery, queryClass, QueryFactory } from '../../src'; @@ -224,6 +225,106 @@ describe('pre-aggregations', () => { expect(preAggregationsDescription[1].preAggregationId).toEqual('Orders.simple2'); }); + // @link https://github.com/cube-js/cube/issues/11682 + describe('rollupLambda unionWithSourceData source query', () => { + const compileEvents = () => prepareJsCompiler( + ` + cube('Events', { + sql: \`SELECT * FROM public.events\`, + + preAggregations: { + eventsLambda: { + type: \`rollupLambda\`, + unionWithSourceData: true, + rollups: [CUBE.eventsRollup], + }, + eventsRollup: { + measures: [CUBE.count], + timeDimension: CUBE.ts, + granularity: \`day\`, + partitionGranularity: \`month\`, + buildRangeStart: { + sql: \`SELECT DATE '2024-01-01'\`, + }, + buildRangeEnd: { + sql: \`SELECT CURRENT_DATE\`, + }, + }, + }, + + measures: { + count: { + type: \`count\`, + }, + }, + + dimensions: { + id: { + sql: \`id\`, + type: \`number\`, + primaryKey: true, + }, + ts: { + sql: \`ts\`, + type: \`time\`, + }, + }, + }); + ` + ); + + const lambdaQueryFor = async (timeDimensions: any[]) => { + const { compiler, cubeEvaluator, joinGraph } = compileEvents(); + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: ['Events.count'], + timeDimensions, + timezone: 'UTC', + }); + + const lambdaQueries: any = query.buildLambdaQuery(); + const [lambdaQuery] = Object.values(lambdaQueries); + expect(lambdaQuery).toBeDefined(); + + return lambdaQuery; + }; + + it('is bounded by the requested date range', async () => { + const { sqlAndParams: [lambdaSql, lambdaParams] } = await lambdaQueryFor([{ + dimension: 'Events.ts', + dateRange: ['2024-02-01', '2024-02-29'], + }]); + + expect(lambdaParams).toContain(FROM_PARTITION_RANGE); + expect(lambdaParams).toContain('2024-02-29T23:59:59.999Z'); + expect(lambdaParams).toContain('2024-02-01T00:00:00.000Z'); + expect(lambdaSql).toMatch(/<=/); + }); + + it('stays unbounded above without a requested date range', async () => { + const { sqlAndParams: [lambdaSql, lambdaParams] } = await lambdaQueryFor([{ + dimension: 'Events.ts', + granularity: 'day', + }]); + + // With lambda-view we observe all 'fresh' data, with no partition/buildRange limit. + expect(lambdaParams).toEqual([FROM_PARTITION_RANGE]); + expect(lambdaSql).not.toMatch(/<=/); + }); + + it('renders the row limit as a number rather than a placeholder', async () => { + const { sqlAndParams: [lambdaSql, lambdaParams], maxSourceRowLimit } = await lambdaQueryFor([{ + dimension: 'Events.ts', + dateRange: ['2024-02-01', '2024-02-29'], + }]); + + expect(maxSourceRowLimit).toEqual(200000); + expect(lambdaSql).toMatch(/LIMIT 200000/); + expect(lambdaParams).not.toContain(MAX_SOURCE_ROW_LIMIT); + }); + }); + // @link https://github.com/cube-js/cube/issues/6623 it('view and pre-aggregation granularity', async () => { const { compiler, cubeEvaluator, joinGraph } = prepareYamlCompiler( diff --git a/packages/cubejs-testing/test/smoke-lambda.test.ts b/packages/cubejs-testing/test/smoke-lambda.test.ts index 65c0d0c903468..b78af5a8e67f7 100644 --- a/packages/cubejs-testing/test/smoke-lambda.test.ts +++ b/packages/cubejs-testing/test/smoke-lambda.test.ts @@ -342,6 +342,52 @@ describe('lambda', () => { ); }); + // @link https://github.com/cube-js/cube/issues/11682 + test('query with a date range fully inside the build range', async () => { + const response = await client.load({ + measures: ['Orders.count'], + dimensions: ['Orders.status', 'Orders.userId'], + timeDimensions: [ + { + dimension: 'Orders.completedAt', + // ordersByCompletedAtAndUserId is built for 2020-02-07 .. 2020-12-01 + dateRange: ['2020-03-01', '2020-06-30'], + granularity: 'day' + } + ], + filters: [ + { + member: 'Orders.status', + operator: 'equals', + values: ['shipped'] + } + ], + order: { + 'Orders.completedAt': 'desc', + 'Orders.userId': 'asc', + }, + limit: 3 + }); + + // @ts-ignore + const { usedPreAggregations } = response.loadResponse.results[0]; + expect(Object.keys(usedPreAggregations)).toEqual([ + 'dev_pre_aggregations.orders_orders_by_completed_at_and_user_id' + ]); + expect( + usedPreAggregations['dev_pre_aggregations.orders_orders_by_completed_at_and_user_id'].targetTableName + ).not.toMatch(/lambda_/); + + const rows = response.rawData(); + expect(rows.length).toBeGreaterThan(0); + rows.forEach(row => { + const completedAt = row['Orders.completedAt'] as string; + expect(row['Orders.status']).toEqual('shipped'); + expect(completedAt >= '2020-03-01').toBe(true); + expect(completedAt <= '2020-06-30T23:59:59.999').toBe(true); + }); + }); + it('Pre-aggregations API', async () => { const preAggs = await fetch(`${birdbox.configuration.playgroundUrl}/cubejs-system/v1/pre-aggregations`, { method: 'GET',