From f6bf9126e854ae4db4e4a8a0ce65b6eef60a599f Mon Sep 17 00:00:00 2001 From: firasbouzazi <115628206+firasbouzazi@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:10:22 +0000 Subject: [PATCH] [common] Fix auto-partition time validation for quarter and invalid dates Validation of auto-partition time values had two problems: 1. The default QUARTER format "yyyyQ" could never be parsed: the variable-width quarter field cannot follow the adjacent year field, so even the current quarter (e.g. "20244") was rejected. Manually creating a partition or dynamically creating one from the client always failed for QUARTER auto-partitioned tables, and isPastAutoPartition always returned false for them. 2. Parsing used the SMART resolver, which accepts non-existent dates such as "20240230", "20230229" or hour "24", so such partitions could be created on auto-partitioned tables. Parse partition time values with a strict parser that reads a single-letter quarter as a fixed-width field and range-checks it. --- .../apache/fluss/utils/PartitionUtils.java | 56 +++++++++++- .../fluss/utils/PartitionUtilsTest.java | 89 +++++++++++++++++++ 2 files changed, 142 insertions(+), 3 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/PartitionUtils.java b/fluss-common/src/main/java/org/apache/fluss/utils/PartitionUtils.java index fcc584047e0..9871731faf1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/utils/PartitionUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/utils/PartitionUtils.java @@ -35,7 +35,12 @@ import java.time.Instant; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; +import java.time.format.ResolverStyle; +import java.time.temporal.ChronoField; +import java.time.temporal.IsoFields; +import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -469,14 +474,59 @@ private static boolean isValidPartitionTime( AutoPartitionTimeUnit timeUnit, AutoPartitionStrategy autoPartitionStrategy) { try { - DateTimeFormatter.ofPattern(getPartitionTimeFormat(timeUnit, autoPartitionStrategy)) - .parse(time); - return true; + TemporalAccessor parsed = + buildPartitionTimeParser( + getPartitionTimeFormat(timeUnit, autoPartitionStrategy)) + .parse(time); + // a quarter without a day is not range-checked by the resolver + return !parsed.isSupported(IsoFields.QUARTER_OF_YEAR) + || IsoFields.QUARTER_OF_YEAR + .range() + .isValidValue(parsed.getLong(IsoFields.QUARTER_OF_YEAR)); } catch (DateTimeParseException e) { return false; } } + /** + * Builds a strict parser for the given partition time pattern. + * + *

A single-letter quarter field ('Q' or 'q') is parsed as a fixed-width value, because the + * variable-width quarter of {@link DateTimeFormatter#ofPattern(String)} cannot follow an + * adjacent year field, e.g. "yyyyQ" fails to parse "20244". The {@link ResolverStyle#STRICT} + * resolver rejects non-existent times such as "20240230", and the default era lets the + * year-of-era field ('y') resolve under it. + */ + private static DateTimeFormatter buildPartitionTimeParser(String pattern) { + DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); + StringBuilder pendingPattern = new StringBuilder(); + boolean inLiteral = false; + for (int index = 0; index < pattern.length(); index++) { + char patternChar = pattern.charAt(index); + if (patternChar == '\'') { + inLiteral = !inLiteral; + } else if (!inLiteral + && (patternChar == 'Q' || patternChar == 'q') + && (index == 0 || pattern.charAt(index - 1) != patternChar) + && (index + 1 == pattern.length() + || pattern.charAt(index + 1) != patternChar)) { + if (pendingPattern.length() > 0) { + builder.appendPattern(pendingPattern.toString()); + pendingPattern.setLength(0); + } + builder.appendValue(IsoFields.QUARTER_OF_YEAR, 1); + continue; + } + pendingPattern.append(patternChar); + } + if (pendingPattern.length() > 0) { + builder.appendPattern(pendingPattern.toString()); + } + return builder.parseDefaulting(ChronoField.ERA, 1) + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + } + private static String getFormattedTime(ZonedDateTime zonedDateTime, String format) { return DateTimeFormatter.ofPattern(format).format(zonedDateTime); } diff --git a/fluss-common/src/test/java/org/apache/fluss/utils/PartitionUtilsTest.java b/fluss-common/src/test/java/org/apache/fluss/utils/PartitionUtilsTest.java index 720bdecc31b..9d723315fb3 100644 --- a/fluss-common/src/test/java/org/apache/fluss/utils/PartitionUtilsTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/utils/PartitionUtilsTest.java @@ -364,6 +364,95 @@ void testValidateConfiguredTimeFormatWhenAutoPartitionIsDisabled() { .hasMessageContaining("yyyy-MM-dd"); } + @Test + void testValidateAutoPartitionTimeWithQuarterFormat() { + Configuration quarterConf = new Configuration(); + quarterConf.setString(ConfigOptions.TABLE_AUTO_PARTITION_TIME_FORMAT, "yyyyQ"); + AutoPartitionStrategy quarterStrategy = AutoPartitionStrategy.from(quarterConf); + + for (String partitionTime : Arrays.asList("20241", "20244")) { + assertThatNoException() + .isThrownBy( + () -> + validateAutoPartitionTime( + new PartitionSpec( + Collections.singletonMap("dt", partitionTime)), + Collections.singletonList("dt"), + quarterStrategy)); + } + for (String partitionTime : Arrays.asList("20240", "20245", "2024", "202441")) { + assertThatThrownBy( + () -> + validateAutoPartitionTime( + new PartitionSpec( + Collections.singletonMap("dt", partitionTime)), + Collections.singletonList("dt"), + quarterStrategy)) + .isInstanceOf(InvalidPartitionException.class) + .hasMessageContaining(partitionTime); + } + + // the default quarter format is also "yyyyQ" + Configuration defaultQuarterConf = new Configuration(); + defaultQuarterConf.set( + ConfigOptions.TABLE_AUTO_PARTITION_TIME_UNIT, AutoPartitionTimeUnit.QUARTER); + defaultQuarterConf.setString(ConfigOptions.TABLE_AUTO_PARTITION_TIMEZONE, "UTC"); + AutoPartitionStrategy defaultQuarterStrategy = + AutoPartitionStrategy.from(defaultQuarterConf); + Instant now = Instant.parse("2024-11-11T00:00:00Z"); + assertThat(isPastAutoPartition("20243", defaultQuarterStrategy, now)).isTrue(); + assertThat(isPastAutoPartition("20244", defaultQuarterStrategy, now)).isFalse(); + assertThat(isPastAutoPartition("20251", defaultQuarterStrategy, now)).isFalse(); + } + + @Test + void testValidateAutoPartitionTimeRejectsNonExistentDates() { + Configuration dayConf = new Configuration(); + dayConf.setString(ConfigOptions.TABLE_AUTO_PARTITION_TIME_FORMAT, "yyyyMMdd"); + AutoPartitionStrategy dayStrategy = AutoPartitionStrategy.from(dayConf); + Configuration hourConf = new Configuration(); + hourConf.setString(ConfigOptions.TABLE_AUTO_PARTITION_TIME_FORMAT, "yyyyMMddHH"); + AutoPartitionStrategy hourStrategy = AutoPartitionStrategy.from(hourConf); + + assertThatNoException() + .isThrownBy( + () -> + validateAutoPartitionTime( + new PartitionSpec( + Collections.singletonMap("dt", "20240229")), + Collections.singletonList("dt"), + dayStrategy)); + for (String partitionTime : Arrays.asList("20240230", "20230229", "20240431")) { + assertThatThrownBy( + () -> + validateAutoPartitionTime( + new PartitionSpec( + Collections.singletonMap("dt", partitionTime)), + Collections.singletonList("dt"), + dayStrategy)) + .isInstanceOf(InvalidPartitionException.class) + .hasMessageContaining(partitionTime); + } + + assertThatNoException() + .isThrownBy( + () -> + validateAutoPartitionTime( + new PartitionSpec( + Collections.singletonMap("dt", "2024010123")), + Collections.singletonList("dt"), + hourStrategy)); + assertThatThrownBy( + () -> + validateAutoPartitionTime( + new PartitionSpec( + Collections.singletonMap("dt", "2024010124")), + Collections.singletonList("dt"), + hourStrategy)) + .isInstanceOf(InvalidPartitionException.class) + .hasMessageContaining("2024010124"); + } + @Test void testValidateAutoPartitionTimeRetentionBoundaryWithConfiguredDayFormat() { Configuration dashedDayConf = new Configuration();