From 4ab5d638d5f5aa09c14d554514ce3cf346ab214f Mon Sep 17 00:00:00 2001 From: Shangqing Yang Date: Fri, 2 Oct 2026 23:13:58 -0400 Subject: [PATCH 1/4] [Client] Support bounded log scanning --- .../log/AbstractLogFetchCollector.java | 293 +++- .../scanner/log/ArrowLogFetchCollector.java | 48 +- .../table/scanner/log/ArrowScanRecords.java | 61 +- .../table/scanner/log/BucketScanStatus.java | 26 +- .../table/scanner/log/CompletedFetch.java | 56 +- .../scanner/log/DefaultCompletedFetch.java | 1 + .../table/scanner/log/LogFetchCollector.java | 31 +- .../client/table/scanner/log/LogFetcher.java | 160 +- .../client/table/scanner/log/LogScanner.java | 74 + .../table/scanner/log/LogScannerImpl.java | 89 +- .../table/scanner/log/LogScannerStatus.java | 107 +- .../scanner/log/RemoteCompletedFetch.java | 5 + .../table/scanner/log/RemotePendingFetch.java | 4 + .../client/table/scanner/log/ScanRecords.java | 53 +- .../scanner/log/ArrowScanRecordsTest.java | 49 + .../log/BoundedLogScannerRetentionITCase.java | 201 +++ .../log/LogFetchCollectorErrorTest.java | 413 +++++ .../scanner/log/LogFetchCollectorTest.java | 725 +++++++- .../table/scanner/log/LogFetcherTest.java | 304 ++++ .../table/scanner/log/LogScannerITCase.java | 97 ++ .../scanner/log/LogScannerStatusTest.java | 225 +++ .../table/scanner/log/LogScannerTest.java | 225 +++ .../scanner/log/RemoteCompletedFetchTest.java | 92 + .../table/scanner/log/ScanRecordsTest.java | 18 + .../rpc/entity/FetchLogResultForBucket.java | 74 +- .../fluss/rpc/util/CommonRpcMessageUtils.java | 14 +- fluss-rpc/src/main/proto/FlussApi.proto | 5 + .../entity/FetchLogResultForBucketTest.java | 52 + fluss-rust/crates/fluss/proto/FlussApi.proto | 1518 +++++++++++++++++ .../crates/fluss/src/client/table/scanner.rs | 4 + fluss-rust/crates/fluss/src/proto/fluss.rs | 6 + .../apache/fluss/server/log/LogReadInfo.java | 27 + .../apache/fluss/server/replica/Replica.java | 17 +- .../fluss/server/replica/ReplicaManager.java | 62 +- .../server/utils/ServerRpcMessageUtils.java | 5 + .../fluss/server/replica/ReplicaTest.java | 51 + .../utils/ServerRpcMessageUtilsTest.java | 54 +- 37 files changed, 5026 insertions(+), 220 deletions(-) create mode 100644 fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecordsTest.java create mode 100644 fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java create mode 100644 fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java create mode 100644 fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java create mode 100644 fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java create mode 100644 fluss-rpc/src/test/java/org/apache/fluss/rpc/entity/FetchLogResultForBucketTest.java create mode 100644 fluss-rust/crates/fluss/proto/FlussApi.proto diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java index 3ffa640f9c5..a4ff1c17936 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java @@ -38,6 +38,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; + +import static org.apache.fluss.utils.Preconditions.checkState; /** Shared implementation for polling completed fetches into scanner results. */ @ThreadSafe @@ -69,12 +72,10 @@ protected AbstractLogFetchCollector( * defaultResetPolicy is NONE */ public R collectFetch(final LogFetchBuffer logFetchBuffer) { - Map> fetched = new HashMap<>(); - Map consumedUpToOffsets = new HashMap<>(); - int recordsRemaining = maxPollRecords; + PollAccumulator result = new PollAccumulator(); try { - while (recordsRemaining > 0) { + while (result.recordsRemaining > 0) { CompletedFetch nextInLineFetch = logFetchBuffer.nextInLineFetch(); if (nextInLineFetch == null || nextInLineFetch.isConsumed()) { CompletedFetch completedFetch = logFetchBuffer.peek(); @@ -84,18 +85,27 @@ public R collectFetch(final LogFetchBuffer logFetchBuffer) { if (!completedFetch.isInitialized()) { try { - CompletedFetch initialized = initialize(completedFetch); + CompletedFetch initialized = initialize(completedFetch, result); logFetchBuffer.setNextInLineFetch(initialized); if (initialized == null) { completedFetch.drain(); } } catch (Exception e) { - // Remove a completedFetch upon a parse with exception if - // (1) it contains no records, and - // (2) there are no fetched records with actual content - // preceding this exception. - if (fetched.isEmpty() && completedFetch.sizeInBytes == 0) { - logFetchBuffer.poll(); + // Deferred errors remain queued. Immediate failures are discarded only + // when the failure policy says this response must not be retried. + if (result.shouldPropagateImmediately(e) + && result.shouldDiscardFailedFetch(e, completedFetch)) { + CompletedFetch removed = logFetchBuffer.poll(); + checkState( + removed == completedFetch, + "Expected failed fetch %s at the head of the buffer, but found %s.", + completedFetch, + removed); + try { + completedFetch.drain(); + } catch (RuntimeException cleanupException) { + e.addSuppressed(cleanupException); + } } throw e; } @@ -105,56 +115,56 @@ public R collectFetch(final LogFetchBuffer logFetchBuffer) { logFetchBuffer.poll(); } else { - List records = fetchRecords(nextInLineFetch, recordsRemaining); TableBucket tableBucket = nextInLineFetch.tableBucket; - // Always record the advanced next fetch offset for this bucket, even when - // the materialized record list is empty. - consumedUpToOffsets.put(tableBucket, nextInLineFetch.nextFetchOffset()); - if (!records.isEmpty()) { - List currentRecords = fetched.get(tableBucket); - if (currentRecords == null) { - fetched.put(tableBucket, records); - } else { - // this case shouldn't usually happen because we only send one fetch - // at a time per bucket, but it might conceivably happen in some rare - // cases (such as bucket leader changes). we have to copy to a new list - // because the old one may be immutable - List mergedRecords = - new ArrayList<>(records.size() + currentRecords.size()); - mergedRecords.addAll(currentRecords); - mergedRecords.addAll(records); - fetched.put(tableBucket, mergedRecords); - } + long stoppingOffset = logScannerStatus.getBucketStoppingOffset(tableBucket); + boolean bounded = stoppingOffset != LogScanner.NO_STOPPING_OFFSET; + Long offsetBeforeFetch = + bounded ? logScannerStatus.getBucketOffset(tableBucket) : null; + List records = fetchRecords(nextInLineFetch, result.recordsRemaining); - recordsRemaining -= recordCount(records); + if (bounded) { + Long offsetAfterFetch = logScannerStatus.getBucketOffset(tableBucket); + if (offsetBeforeFetch != null + && offsetAfterFetch != null + && !offsetBeforeFetch.equals(offsetAfterFetch)) { + result.recordProgress(tableBucket, offsetAfterFetch, true); + } } else { - fetched.putIfAbsent(tableBucket, Collections.emptyList()); + // Preserve the existing unbounded behavior: every consumed fetch + // contributes its next fetch offset to the poll result. + result.recordProgress( + tableBucket, nextInLineFetch.nextFetchOffset(), false); } + + result.addRecords(tableBucket, records, bounded); } } - } catch (FetchException e) { - if (fetched.isEmpty()) { + } catch (Exception e) { + if (result.shouldPropagateImmediately(e)) { + // Release any off-heap resources held by accumulated records. Pending completion + // events remain in scanner status until a poll successfully returns them. + closeFetchedRecords(result.fetched); throw e; } - } catch (Exception e) { - // Release any off-heap resources (e.g. Arrow buffers) held by - // already-fetched records before propagating the unexpected error. - closeFetchedRecords(fetched); - throw e; } - return toResult(fetched, consumedUpToOffsets); + Set finishedBuckets = logScannerStatus.drainFinishedBuckets(); + return toResult(result.fetched, result.consumedUpToOffsets, finishedBuckets); } /** Initialize a {@link CompletedFetch} object. */ @Nullable - private CompletedFetch initialize(CompletedFetch completedFetch) { + private CompletedFetch initialize(CompletedFetch completedFetch, PollAccumulator result) { TableBucket tb = completedFetch.tableBucket; + if (logScannerStatus.hasReachedStoppingOffset(tb)) { + log.trace("Discarding fetch response for finished bounded bucket {}.", tb); + return null; + } ApiError error = completedFetch.error; try { if (error.isSuccess()) { - return handleInitializeSuccess(completedFetch); + return handleInitializeSuccess(completedFetch, result); } else { handleInitializeErrors( completedFetch, @@ -173,28 +183,54 @@ private CompletedFetch initialize(CompletedFetch completedFetch) { } } - private @Nullable CompletedFetch handleInitializeSuccess(CompletedFetch completedFetch) { + private @Nullable CompletedFetch handleInitializeSuccess( + CompletedFetch completedFetch, PollAccumulator result) { TableBucket tb = completedFetch.tableBucket; - long fetchOffset = completedFetch.nextFetchOffset(); + long requestedFetchOffset = completedFetch.requestedFetchOffset(); // we are interested in this fetch only if the beginning offset matches the // current consumed position. - Long offset = logScannerStatus.getBucketOffset(tb); - if (offset == null) { + Long currentOffset = logScannerStatus.getBucketOffset(tb); + if (currentOffset == null) { log.debug( "Discarding stale fetch response for bucket {} since the expected offset is null which means the bucket has been unsubscribed.", tb); return null; } - if (offset != fetchOffset) { + if (currentOffset != requestedFetchOffset) { log.warn( "Discarding stale fetch response for bucket {} since its offset {} does not match the expected offset {}.", tb, - fetchOffset, - offset); + requestedFetchOffset, + currentOffset); return null; } + long stoppingOffset = logScannerStatus.getBucketStoppingOffset(tb); + boolean bounded = stoppingOffset != LogScanner.NO_STOPPING_OFFSET; + + if (bounded && requestedFetchOffset == LogScanner.EARLIEST_OFFSET) { + long resolvedEarliestOffset = completedFetch.resolvedEarliestOffset(); + if (resolvedEarliestOffset < 0) { + throw new UnsupportedBoundedEarliestException( + "Bounded scanning from EARLIEST_OFFSET requires a server that reports resolved_earliest_offset." + + " Upgrade the server or use an explicit starting offset."); + } + // Physical cursor used while reading this CompletedFetch. + completedFetch.applyResolvedEarliestOffset(); + // Logical bounded cursor owned by LogScannerStatus. + Long logicalOffset = + logScannerStatus.resolveBoundedStartingOffset( + tb, requestedFetchOffset, resolvedEarliestOffset); + if (logicalOffset == null) { + return null; + } + result.recordProgress(tb, logicalOffset, true); + if (logScannerStatus.hasReachedStoppingOffset(tb)) { + return null; + } + } + long highWatermark = completedFetch.highWatermark; if (highWatermark >= 0) { log.trace("Updating high watermark for bucket {} to {}.", tb, highWatermark); @@ -208,7 +244,7 @@ private CompletedFetch initialize(CompletedFetch completedFetch) { private void handleInitializeErrors( CompletedFetch completedFetch, Errors error, String errorMessage, TablePath tablePath) { TableBucket tb = completedFetch.tableBucket; - long fetchOffset = completedFetch.nextFetchOffset(); + long requestedFetchOffset = completedFetch.requestedFetchOffset(); if (error == Errors.NOT_LEADER_OR_FOLLOWER || error == Errors.LOG_STORAGE_EXCEPTION || error == Errors.KV_STORAGE_EXCEPTION @@ -227,25 +263,25 @@ private void handleInitializeErrors( throw new FetchException( String.format( "The fetching offset %s is out of range: %s", - fetchOffset, error.exception(errorMessage))); + requestedFetchOffset, error.exception(errorMessage))); } else if (error == Errors.AUTHORIZATION_EXCEPTION) { throw new AuthorizationException(errorMessage); } else if (error == Errors.UNKNOWN_SERVER_ERROR) { log.warn( "Unknown server error while fetching offset {} for bucket {}: {}", - fetchOffset, + requestedFetchOffset, tb, error.exception(errorMessage)); } else if (error == Errors.CORRUPT_MESSAGE) { throw new FetchException( String.format( "Encountered corrupt message when fetching offset %s for bucket %s: %s", - fetchOffset, tb, error.exception(errorMessage))); + requestedFetchOffset, tb, error.exception(errorMessage))); } else { throw new FetchException( String.format( "Unexpected error code %s while fetching at offset %s from bucket %s: %s", - error, fetchOffset, tb, error.exception(errorMessage))); + error, requestedFetchOffset, tb, error.exception(errorMessage))); } } @@ -256,35 +292,60 @@ protected List fetchRecords(CompletedFetch nextInLineFetch, int maxRecords) { log.debug( "Ignoring fetched records for {} at offset {} since the current offset is null which means the bucket has been unsubscribed.", tb, - nextInLineFetch.fetchOffset()); - } else { - if (nextInLineFetch.nextFetchOffset() == offset) { - List records = doFetchRecords(nextInLineFetch, maxRecords); + nextInLineFetch.requestedFetchOffset()); + } else if (logScannerStatus.hasReachedStoppingOffset(tb)) { + log.trace("Ignoring fetched records for finished bounded bucket {}.", tb); + } else if (nextInLineFetch.nextFetchOffset() == offset) { + long stoppingOffset = logScannerStatus.getBucketStoppingOffset(tb); + boolean bounded = stoppingOffset != LogScanner.NO_STOPPING_OFFSET; + + List records = doFetchRecords(nextInLineFetch, maxRecords); + long rawConsumedUpToOffset = nextInLineFetch.nextFetchOffset(); + + if (bounded && !records.isEmpty()) { + records = trimFetchedRecords(records, stoppingOffset); + } + + log.trace( + "Returning {} fetched records at offset {} for assigned bucket {}.", + records.size(), + offset, + tb); + + long consumedUpToOffset = + bounded + ? Math.min(rawConsumedUpToOffset, stoppingOffset) + : rawConsumedUpToOffset; + + if (consumedUpToOffset > offset) { log.trace( - "Returning {} fetched records at offset {} for assigned bucket {}.", - records.size(), + "Updating fetch offset from {} to {} for bucket {} and returning {} records " + + "from poll()", offset, - tb); + consumedUpToOffset, + tb, + records.size()); - if (nextInLineFetch.nextFetchOffset() > offset) { - log.trace( - "Updating fetch offset from {} to {} for bucket {} and returning {} records from poll()", - offset, - nextInLineFetch.nextFetchOffset(), - tb, - records.size()); - logScannerStatus.updateOffset(tb, nextInLineFetch.nextFetchOffset()); - } - return records; - } else { - // these records aren't next in line based on the last consumed offset, ignore them - // they must be from an obsolete request - log.warn( - "Ignoring fetched records for {} at offset {} since the current offset is {}", - nextInLineFetch.tableBucket, - nextInLineFetch.nextFetchOffset(), - offset); + logScannerStatus.updateOffset(tb, consumedUpToOffset); } + + if (bounded && rawConsumedUpToOffset >= stoppingOffset) { + log.trace( + "Reached stopping offset {} for bucket {}, draining remaining fetched records.", + stoppingOffset, + tb); + nextInLineFetch.drain(); + } + + return records; + } else { + // these records aren't next in line based on the last consumed offset, ignore them + // they must be from an obsolete request + log.warn( + "Ignoring fetched records for {} at offset {} since the current offset is {}", + nextInLineFetch.tableBucket, + nextInLineFetch.nextFetchOffset(), + offset); } log.trace("Draining fetched records for bucket {}", nextInLineFetch.tableBucket); @@ -301,7 +362,11 @@ protected List fetchRecords(CompletedFetch nextInLineFetch, int maxRecords) { protected abstract int recordCount(List fetchedRecords); protected abstract R toResult( - Map> fetchedRecords, Map consumedUpToOffsets); + Map> fetchedRecords, + Map consumedUpToOffsets, + Set finishedBuckets); + + protected abstract List trimFetchedRecords(List fetchedRecords, long stoppingOffset); /** * Release resources held by fetched records on failure. The default implementation is a no-op, @@ -309,4 +374,72 @@ protected abstract R toResult( * off-heap resources (e.g. Arrow buffers) should override this to close them. */ protected void closeFetchedRecords(Map> fetched) {} + + private final class PollAccumulator { + private final Map> fetched = new HashMap<>(); + private final Map consumedUpToOffsets = new HashMap<>(); + + private int recordsRemaining = maxPollRecords; + private boolean hasBoundedAccumulatedResult; + + private boolean hasDeliverableResult() { + return !fetched.isEmpty() + || !consumedUpToOffsets.isEmpty() + || logScannerStatus.hasPendingFinishedBuckets(); + } + + private boolean shouldPropagateImmediately(Exception exception) { + if (exception instanceof FetchException) { + // Preserve the existing FetchException behavior: return any already + // accumulated records, progress, or bounded completion first. + return !hasDeliverableResult(); + } + + // Non-FetchException historically propagates immediately for unbounded + // scans. Defer it only when this poll has already consumed bounded data + // or progress that must be delivered before the error. + return !hasBoundedAccumulatedResult; + } + + private boolean shouldDiscardFailedFetch( + Exception exception, CompletedFetch completedFetch) { + if (exception instanceof UnsupportedBoundedEarliestException) { + return true; + } + + return fetched.isEmpty() + && consumedUpToOffsets.isEmpty() + && completedFetch.sizeInBytes == 0; + } + + private void recordProgress(TableBucket tableBucket, long offset, boolean bounded) { + consumedUpToOffsets.merge(tableBucket, offset, Math::max); + hasBoundedAccumulatedResult |= bounded; + } + + private void addRecords(TableBucket tableBucket, List records, boolean bounded) { + if (records.isEmpty()) { + return; + } + List current = fetched.get(tableBucket); + if (current == null) { + fetched.put(tableBucket, records); + } else { + List merged = new ArrayList<>(current.size() + records.size()); + merged.addAll(current); + merged.addAll(records); + fetched.put(tableBucket, merged); + } + recordsRemaining -= recordCount(records); + hasBoundedAccumulatedResult |= bounded; + } + } + + private static final class UnsupportedBoundedEarliestException + extends UnsupportedOperationException { + + private UnsupportedBoundedEarliestException(String message) { + super(message); + } + } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java index cbe56b094d4..e64d6c07ff0 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowLogFetchCollector.java @@ -22,14 +22,18 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ArrowBatchData; +import org.apache.fluss.utils.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.concurrent.ThreadSafe; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; /** Collects Arrow batches from completed fetches. */ @ThreadSafe @@ -62,8 +66,48 @@ protected int recordCount(List fetchedRecords) { @Override protected ArrowScanRecords toResult( Map> fetchedRecords, - Map consumedUpToOffsets) { - return new ArrowScanRecords(fetchedRecords, consumedUpToOffsets); + Map consumedUpToOffsets, + Set finishedBuckets) { + return new ArrowScanRecords(fetchedRecords, consumedUpToOffsets, finishedBuckets); + } + + @Override + protected List trimFetchedRecords( + List fetchedRecords, long stoppingOffset) { + for (int i = 0; i < fetchedRecords.size(); i++) { + ArrowBatchData batch = fetchedRecords.get(i); + long batchBaseOffset = batch.getBaseLogOffset(); + int batchRecordCount = batch.getRecordCount(); + + if (batchBaseOffset >= stoppingOffset) { + for (int j = i; j < fetchedRecords.size(); j++) { + IOUtils.closeQuietly(fetchedRecords.get(j)); + } + + if (i == 0) { + return Collections.emptyList(); + } + return new ArrayList<>(fetchedRecords.subList(0, i)); + } + + long writableRowCount = stoppingOffset - batchBaseOffset; + if (writableRowCount < batchRecordCount) { + ArrowBatchData truncatedBatch = + batch.truncateAndTransferOwnership((int) writableRowCount); + + List trimmedRecords = new ArrayList<>(i + 1); + trimmedRecords.addAll(fetchedRecords.subList(0, i)); + trimmedRecords.add(truncatedBatch); + + for (int j = i + 1; j < fetchedRecords.size(); j++) { + IOUtils.closeQuietly(fetchedRecords.get(j)); + } + + return trimmedRecords; + } + } + + return fetchedRecords; } @Override diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java index 048eba4e1ae..3e82d0267c2 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -47,6 +48,9 @@ public class ArrowScanRecords implements Iterable, AutoCloseable /** The exclusive upper bound of consumed offsets per polled bucket in this round. */ private final Map consumedUpToOffsets; + /** The bounded buckets that reached their stopping offsets in this poll round. */ + private final Set finishedBuckets; + public ArrowScanRecords(Map> records) { this(records, Collections.emptyMap()); } @@ -54,8 +58,16 @@ public ArrowScanRecords(Map> records) { public ArrowScanRecords( Map> records, Map consumedUpToOffsets) { - this.records = records; + this(records, consumedUpToOffsets, Collections.emptySet()); + } + + ArrowScanRecords( + Map> records, + Map consumedUpToOffsets, + Set finishedBuckets) { + this.records = withProgressOrFinishedBuckets(records, consumedUpToOffsets, finishedBuckets); this.consumedUpToOffsets = consumedUpToOffsets; + this.finishedBuckets = finishedBuckets; } /** Get just the Arrow batches for the given bucket. */ @@ -67,7 +79,10 @@ public List records(TableBucket scanBucket) { return Collections.unmodifiableList(recs); } - /** Returns the buckets that were polled in this round. */ + /** + * Get the buckets that were polled in this round, including buckets whose batch list is empty + * but whose log offset still advanced or whose bounded subscription finished. + */ public Set buckets() { return Collections.unmodifiableSet(records.keySet()); } @@ -99,6 +114,25 @@ public boolean isEmpty() { return records.isEmpty(); } + /** + * Returns {@code true} if this {@code ArrowScanRecords} carries any scanner progress, either by + * returning records, advancing a consumed offset, or completing a bounded subscription. + */ + public boolean hasProgress() { + return count() > 0 || !consumedUpToOffsets.isEmpty() || !finishedBuckets.isEmpty(); + } + + /** + * Returns the bounded buckets that finished in this poll round, including empty ranges. + * + *

Each subscription reports completion only once. A completion event may accompany the final + * records or arrive without any records. Callers must consume all records in this result before + * treating the corresponding buckets as fully read. + */ + public Set finishedBuckets() { + return Collections.unmodifiableSet(finishedBuckets); + } + /** Closes all Arrow batches held by this container, releasing off-heap memory. */ @Override public void close() { @@ -115,6 +149,29 @@ public Iterator iterator() { return new ConcatenatedIterable(records.values()).iterator(); } + /** + * Ensures every bucket with a consumed offset or completion event has a (possibly empty) record + * list entry, so that {@link #buckets()} surfaces progress-only and finished-only buckets. + */ + private static Map> withProgressOrFinishedBuckets( + Map> records, + Map consumedUpToOffsets, + Set finishedBuckets) { + if (records.keySet().containsAll(consumedUpToOffsets.keySet()) + && records.keySet().containsAll(finishedBuckets)) { + return records; + } + + Map> merged = new LinkedHashMap<>(records); + for (TableBucket bucket : consumedUpToOffsets.keySet()) { + merged.putIfAbsent(bucket, Collections.emptyList()); + } + for (TableBucket bucket : finishedBuckets) { + merged.putIfAbsent(bucket, Collections.emptyList()); + } + return merged; + } + private static class ConcatenatedIterable implements Iterable { private final Iterable> iterables; diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java index c99ae783875..3b421bbe32d 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java @@ -24,6 +24,8 @@ class BucketScanStatus { private long offset; // last consumed position private long highWatermark = -1L; // the high watermark from last fetch, -1 if never fetched + // the stopping offset for this bucket, NO_STOPPING_OFFSET if not set + private long stoppingOffset = LogScanner.NO_STOPPING_OFFSET; // TODO add resetStrategy and nextAllowedRetryTimeMs. public BucketScanStatus() { @@ -50,6 +52,26 @@ public void setHighWatermark(Long highWatermark) { this.highWatermark = highWatermark; } + public long getStoppingOffset() { + return stoppingOffset; + } + + public void setStoppingOffset(long stoppingOffset) { + this.stoppingOffset = stoppingOffset; + } + + public boolean isBounded() { + return stoppingOffset != LogScanner.NO_STOPPING_OFFSET; + } + + /** + * Returns whether this subscription has reached its exclusive stopping offset or has an empty + * range. Unbounded subscriptions always return {@code false}. + */ + public boolean hasReachedStoppingOffset() { + return isBounded() && (stoppingOffset == 0 || offset >= stoppingOffset); + } + /** * Returns the number of log records that have not been fetched for this bucket, or 0 if the lag * is unknown, i.e. the offset is still a sentinel offset (like {@link @@ -61,6 +83,8 @@ long recordsLag() { if (offset < 0 || highWatermark < 0) { return 0L; } - return Math.max(highWatermark - offset, 0L); + long effectiveHighWatermark = + isBounded() ? Math.min(highWatermark, stoppingOffset) : highWatermark; + return Math.max(effectiveHighWatermark - offset, 0L); } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java index de9cb1d3f61..2f7688b8f65 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/CompletedFetch.java @@ -44,7 +44,9 @@ import java.util.Iterator; import java.util.List; +import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkState; /** * {@link CompletedFetch} represents the result that was returned from the tablet server via a fetch @@ -61,7 +63,8 @@ public abstract class CompletedFetch { final ApiError error; final int sizeInBytes; final long highWatermark; - private final long fetchOffset; + private final long requestedFetchOffset; + private final long resolvedEarliestOffset; private final long filteredEndOffset; private final boolean isCheckCrcs; @@ -90,7 +93,8 @@ public CompletedFetch( LogRecordReadContext readContext, LogScannerStatus logScannerStatus, boolean isCheckCrcs, - long fetchOffset, + long requestedFetchOffset, + long resolvedEarliestOffset, long filteredEndOffset) { this.tableBucket = tableBucket; this.tablePath = tablePath; @@ -101,16 +105,33 @@ public CompletedFetch( this.readContext = readContext; this.isCheckCrcs = isCheckCrcs; this.logScannerStatus = logScannerStatus; - this.fetchOffset = fetchOffset; checkArgument( - filteredEndOffset == NO_FILTERED_END_OFFSET || filteredEndOffset >= fetchOffset, - "filteredEndOffset (%s) must be %s (NO_FILTERED_END_OFFSET) or >= fetchOffset (%s) for bucket %s.", + resolvedEarliestOffset >= -1L, + "resolvedEarliestOffset (%s) must be -1 or non-negative for bucket %s.", + resolvedEarliestOffset, + tableBucket); + checkArgument( + resolvedEarliestOffset < 0 || requestedFetchOffset == EARLIEST_OFFSET, + "resolvedEarliestOffset (%s) can only be set for an EARLIEST request, " + + "but requestedFetchOffset was %s for bucket %s.", + resolvedEarliestOffset, + requestedFetchOffset, + tableBucket); + this.requestedFetchOffset = requestedFetchOffset; + this.resolvedEarliestOffset = resolvedEarliestOffset; + long effectiveStartOffset = + resolvedEarliestOffset >= 0 ? resolvedEarliestOffset : requestedFetchOffset; + checkArgument( + filteredEndOffset == NO_FILTERED_END_OFFSET + || effectiveStartOffset < 0 + || filteredEndOffset >= effectiveStartOffset, + "filteredEndOffset (%s) must be %s or >= effective start offset (%s) for bucket %s.", filteredEndOffset, NO_FILTERED_END_OFFSET, - fetchOffset, + effectiveStartOffset, tableBucket); this.filteredEndOffset = filteredEndOffset; - this.nextFetchOffset = fetchOffset; + this.nextFetchOffset = requestedFetchOffset; } // TODO: optimize this to avoid deep copying the record. @@ -169,8 +190,25 @@ boolean isInitialized() { return initialized; } - long fetchOffset() { - return fetchOffset; + long requestedFetchOffset() { + return requestedFetchOffset; + } + + long resolvedEarliestOffset() { + return resolvedEarliestOffset; + } + + void applyResolvedEarliestOffset() { + checkState( + requestedFetchOffset == LogScanner.EARLIEST_OFFSET, + "Resolved EARLIEST offset can only be applied to EARLIEST fetches."); + + checkState( + resolvedEarliestOffset >= 0, + "Resolved EARLIEST offset is not available for bucket %s.", + tableBucket); + + nextFetchOffset = resolvedEarliestOffset; } long nextFetchOffset() { diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java index 0384389c283..fa1db1c3c13 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetch.java @@ -61,6 +61,7 @@ public DefaultCompletedFetch( logScannerStatus, isCheckCrc, fetchOffset, + fetchLogResultForBucket.getResolvedEarliestOffset(), fetchLogResultForBucket.getFilteredEndOffset()); this.parsedByteBuf = parsedByteBuf; } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java index abdbede3968..3572c2b9a1b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchCollector.java @@ -30,8 +30,11 @@ import javax.annotation.concurrent.ThreadSafe; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; /* This file is based on source code of Apache Kafka Project (https://kafka.apache.org/), licensed by the Apache * Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE file distributed with this work for @@ -67,7 +70,31 @@ protected int recordCount(List fetchedRecords) { @Override protected ScanRecords toResult( Map> fetchedRecords, - Map consumedUpToOffsets) { - return new ScanRecords(fetchedRecords, consumedUpToOffsets); + Map consumedUpToOffsets, + Set finishedBuckets) { + return new ScanRecords(fetchedRecords, consumedUpToOffsets, finishedBuckets); + } + + @Override + protected List trimFetchedRecords( + List fetchedRecords, long stoppingOffset) { + int validRecordCount = fetchedRecords.size(); + + for (int i = 0; i < fetchedRecords.size(); i++) { + if (fetchedRecords.get(i).logOffset() >= stoppingOffset) { + validRecordCount = i; + break; + } + } + + if (validRecordCount == fetchedRecords.size()) { + return fetchedRecords; + } + + if (validRecordCount == 0) { + return Collections.emptyList(); + } + + return new ArrayList<>(fetchedRecords.subList(0, validRecordCount)); } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java index 28cf324eb46..ded99a56241 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java @@ -78,6 +78,7 @@ import java.util.Optional; import java.util.Set; +import static org.apache.fluss.rpc.entity.FetchLogResultForBucket.NO_RESOLVED_EARLIEST_OFFSET; import static org.apache.fluss.rpc.util.CommonRpcMessageUtils.getFetchLogResultForBucket; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -327,6 +328,7 @@ synchronized void sendFetchRequest(int destination, FetchLogRequest fetchLogRequ private FetchRequestContext createFetchRequestContext(FetchLogRequest fetchLogRequest) { Map requestedTableContexts = new HashMap<>(); + Map requestedOffsets = new HashMap<>(); Set tableIdsInFetchRequest = new HashSet<>(); Set tablePartitionsInFetchRequest = new HashSet<>(); for (PbFetchLogReqForTable fetchTableRequest : fetchLogRequest.getTablesReqsList()) { @@ -340,6 +342,17 @@ private FetchRequestContext createFetchRequestContext(FetchLogRequest fetchLogRe continue; } requestedTableContexts.put(tableId, tableReadContext); + for (PbFetchLogReqForBucket bucketRequest : fetchTableRequest.getBucketsReqsList()) { + TableBucket tableBucket = + new TableBucket( + tableId, + bucketRequest.hasPartitionId() + ? bucketRequest.getPartitionId() + : null, + bucketRequest.getBucketId()); + + requestedOffsets.put(tableBucket, bucketRequest.getFetchOffset()); + } if (!tableReadContext.isPartitioned) { tableIdsInFetchRequest.add(tableId); } else { @@ -352,6 +365,7 @@ private FetchRequestContext createFetchRequestContext(FetchLogRequest fetchLogRe } return new FetchRequestContext( requestedTableContexts, + requestedOffsets, new TableOrPartitions( tableIdsInFetchRequest.isEmpty() ? null : tableIdsInFetchRequest, tablePartitionsInFetchRequest.isEmpty() @@ -441,57 +455,103 @@ private synchronized void handleFetchLogResponse( ? respForBucket.getPartitionId() : null, respForBucket.getBucketId()); + + Long requestedFetchOffset = fetchRequestContext.requestedOffsets.get(tb); + Long currentFetchOffset = logScannerStatus.getBucketOffset(tb); + + // The response must still correspond to the subscription for which + // the request was sent. This also covers an unsubscribed bucket, + // whose current offset is null. + if (requestedFetchOffset == null + || !requestedFetchOffset.equals(currentFetchOffset)) { + LOG.debug( + "Ignoring stale fetch log response for bucket {}. Requested offset was {}, current scanner offset is {}.", + tb, + requestedFetchOffset, + currentFetchOffset); + continue; + } + FetchLogResultForBucket fetchResultForBucket = getFetchLogResultForBucket(tb, tablePathForResp, respForBucket); - // if error code is not NONE, it means the fetch log request failed, we need to - // clear table bucket meta for InvalidMetadataException. + // If the fetch failed, handle metadata-related side effects first. + // The error response is still buffered below so that the collector + // can deliver the exception with the normal poll semantics. if (fetchResultForBucket.getErrorCode() != Errors.NONE.code()) { ApiError error = ApiError.fromErrorMessage(respForBucket); handleFetchLogExceptionForBucket(tb, destination, error); } - Long fetchOffset = logScannerStatus.getBucketOffset(tb); - // if the offset is null, it means the bucket has been unsubscribed, - // we just set a Long.MAX_VALUE as the next fetch offset - if (fetchOffset == null) { - LOG.debug( - "Ignoring fetch log response for bucket {} because the bucket has been " - + "unsubscribed.", - tb); - } else { - if (fetchResultForBucket.fetchFromRemote()) { + long stoppingOffset = logScannerStatus.getBucketStoppingOffset(tb); + + boolean boundedEarliest = + requestedFetchOffset == LogScanner.EARLIEST_OFFSET + && stoppingOffset != LogScanner.NO_STOPPING_OFFSET; + + long resolvedEarliestOffset = fetchResultForBucket.getResolvedEarliestOffset(); + + if (fetchResultForBucket.fetchFromRemote()) { + // For bounded EARLIEST, remote records are useful only when: + // 1. the server supports reporting the resolved EARLIEST offset, and + // 2. that resolved offset is before the stopping offset. + // + // Otherwise we intentionally skip the remote download and buffer a + // metadata-only CompletedFetch below. The collector will either: + // - finish an empty range when resolvedEarliestOffset >= stoppingOffset, or + // - fail fast for an old server when the resolved offset is absent. + boolean needsRemoteRecords = + !boundedEarliest + || (resolvedEarliestOffset != NO_RESOLVED_EARLIEST_OFFSET + && resolvedEarliestOffset < stoppingOffset); + + if (needsRemoteRecords) { pendRemoteFetches( trc, fetchResultForBucket.remoteLogFetchInfo(), - fetchOffset, - fetchResultForBucket.getHighWatermark()); - } else { - LogRecords logRecords = fetchResultForBucket.recordsOrEmpty(); - boolean hasRecords = !MemoryLogRecords.EMPTY.equals(logRecords); - if (hasRecords - || fetchResultForBucket.getErrorCode() != Errors.NONE.code() - || fetchResultForBucket.hasFilteredEndOffset()) { - // Retain the parsed buffer so it stays alive while - // this CompletedFetch's records are being consumed. - if (hasRecords && parsedByteBuf != null) { - parsedByteBuf.retain(); - } - logFetchBuffer.add( - new DefaultCompletedFetch( - tb, - tablePathForResp, - fetchResultForBucket, - trc.readContext, - logScannerStatus, - // skipping CRC check if projection push downed as - // the data is pruned - isCheckCrcs, - fetchOffset, - hasRecords ? parsedByteBuf : null)); - } + requestedFetchOffset, + fetchResultForBucket.getHighWatermark(), + boundedEarliest + ? resolvedEarliestOffset + : NO_RESOLVED_EARLIEST_OFFSET, + stoppingOffset); + continue; } } + + // Local responses, errors, filtered-empty responses, and bounded-EARLIEST + // metadata-only responses all go through CompletedFetch so the collector + // can apply a single set of progress/completion/error semantics. + LogRecords logRecords = fetchResultForBucket.recordsOrEmpty(); + boolean hasRecords = !MemoryLogRecords.EMPTY.equals(logRecords); + boolean shouldBuffer = + hasRecords + || fetchResultForBucket.getErrorCode() != Errors.NONE.code() + || fetchResultForBucket.hasFilteredEndOffset() + || boundedEarliest; + + if (!shouldBuffer) { + continue; + } + + // Retain the parsed buffer so it stays alive while this + // CompletedFetch's records are being consumed. + if (hasRecords && parsedByteBuf != null) { + parsedByteBuf.retain(); + } + + logFetchBuffer.add( + new DefaultCompletedFetch( + tb, + tablePathForResp, + fetchResultForBucket, + trc.readContext, + logScannerStatus, + // Skip CRC check if projection was pushed down because + // the returned data has already been pruned. + isCheckCrcs, + requestedFetchOffset, + hasRecords ? parsedByteBuf : null)); } } } finally { @@ -534,7 +594,9 @@ private void pendRemoteFetches( TableReadContext tableReadContext, RemoteLogFetchInfo remoteLogFetchInfo, long firstFetchOffset, - long highWatermark) { + long highWatermark, + long resolvedEarliestOffset, + long stoppingOffset) { checkNotNull(remoteLogFetchInfo); FsPath remoteLogTabletDir = new FsPath(remoteLogFetchInfo.remoteLogTabletDir()); List remoteLogSegments = remoteLogFetchInfo.remoteLogSegmentList(); @@ -546,6 +608,17 @@ private void pendRemoteFetches( posInLogSegment = 0; fetchOffset = segment.remoteLogStartOffset(); } + long physicalFetchOffset = + i == 0 && resolvedEarliestOffset >= 0 ? resolvedEarliestOffset : fetchOffset; + + // Remote segments are ordered by start offset, so once a segment starts at or + // beyond the stopping offset, all subsequent segments are outside the range. + if (stoppingOffset != LogScanner.NO_STOPPING_OFFSET + && physicalFetchOffset >= stoppingOffset) { + break; + } + long segmentResolvedEarliestOffset = + i == 0 ? resolvedEarliestOffset : NO_RESOLVED_EARLIEST_OFFSET; RemoteLogDownloadFuture downloadFuture = remoteLogDownloader.requestRemoteLog(remoteLogTabletDir, segment); RemotePendingFetch pendingFetch = @@ -555,6 +628,7 @@ private void pendRemoteFetches( tableReadContext.tablePath, posInLogSegment, fetchOffset, + segmentResolvedEarliestOffset, highWatermark, tableReadContext.remoteReadContext, logScannerStatus, @@ -726,15 +800,23 @@ synchronized int getRegisteredTableCount() { return tableReadContexts.size(); } + @VisibleForTesting + int getPendingRemoteLogFetchCount() { + return remoteLogDownloader.getSizeOfSegmentsToFetch(); + } + /** Request-scoped context snapshot. */ private static class FetchRequestContext { private final Map requestedTableContexts; + private final Map requestedOffsets; private final TableOrPartitions tableOrPartitions; private FetchRequestContext( Map requestedTableContexts, + Map requestedOffsets, TableOrPartitions tableOrPartitions) { this.requestedTableContexts = requestedTableContexts; + this.requestedOffsets = requestedOffsets; this.tableOrPartitions = tableOrPartitions; } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java index d255e340a94..a46caeb5c4f 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java @@ -97,6 +97,80 @@ default void subscribeFromBeginning(int bucket) { */ void subscribe(long partitionId, int bucket, long offset); + /** + * Subscribe to the given table bucket in the given offset range dynamically. If the table + * bucket is already subscribed, the offset range will be updated. + * + *

The stopping offset is exclusive, so the subscribed range is {@code [startingOffset, + * stoppingOffset)}. The stopping offset must be non-negative. + * + *

When {@link #EARLIEST_OFFSET} needs to be resolved by the server for a non-empty bounded + * range, the server must report the physical offset to which EARLIEST was resolved. If the + * connected server does not support this capability, polling fails with {@link + * UnsupportedOperationException}. + * + *

The starting offset must be non-negative or {@link #EARLIEST_OFFSET}. A resolved starting + * offset at or beyond the stopping offset represents an empty range. A stopping offset of zero + * always represents an empty range, including when starting from the beginning. + * + *

Completion is reported once per subscription through {@link + * ScanRecords#finishedBuckets()}. The final result can contain records that must be consumed + * before completing the read. The exclusive boundary takes precedence over keeping + * UPDATE_BEFORE and UPDATE_AFTER together. + * + *

Please use {@link #subscribeBounded(long, int, long, long)} to subscribe a partitioned + * table. + * + * @param bucket the table bucket to subscribe. + * @param startingOffset the offset to start from. + * @param stoppingOffset the non-negative exclusive offset to stop at. + * @throws IllegalArgumentException if the starting offset is negative and is not {@link + * #EARLIEST_OFFSET}, or the stopping offset is negative. + * @throws UnsupportedOperationException if this scanner does not support bounded subscriptions. + * @throws java.lang.IllegalStateException if the table is a partitioned table. + */ + default void subscribeBounded(int bucket, long startingOffset, long stoppingOffset) { + throw new UnsupportedOperationException("Bounded log scanning is not supported."); + } + + /** + * Subscribe to the given partitioned table bucket in the given offset range dynamically. If the + * table bucket is already subscribed, the offset range will be updated. + * + *

The stopping offset is exclusive, so the subscribed range is {@code [startingOffset, + * stoppingOffset)}. The stopping offset must be non-negative. + * + *

When {@link #EARLIEST_OFFSET} needs to be resolved by the server for a non-empty bounded + * range, the server must report the physical offset to which EARLIEST was resolved. If the + * connected server does not support this capability, polling fails with {@link + * UnsupportedOperationException}. + * + *

The starting offset must be non-negative or {@link #EARLIEST_OFFSET}. A resolved starting + * offset at or beyond the stopping offset represents an empty range. A stopping offset of zero + * always represents an empty range, including when starting from the beginning. + * + *

Completion is reported once per subscription through {@link + * ScanRecords#finishedBuckets()}. The final result can contain records that must be consumed + * before completing the read. The exclusive boundary takes precedence over keeping + * UPDATE_BEFORE and UPDATE_AFTER together. + * + *

Please use {@link #subscribeBounded(int, long, long)} to subscribe a non-partitioned + * table. + * + * @param partitionId the partition id of the table partition to subscribe. + * @param bucket the table bucket to subscribe. + * @param startingOffset the offset to start from. + * @param stoppingOffset the non-negative exclusive offset to stop at. + * @throws IllegalArgumentException if the starting offset is negative and is not {@link + * #EARLIEST_OFFSET}, or the stopping offset is negative. + * @throws UnsupportedOperationException if this scanner does not support bounded subscriptions. + * @throws java.lang.IllegalStateException if the table is a non-partitioned table. + */ + default void subscribeBounded( + long partitionId, int bucket, long startingOffset, long stoppingOffset) { + throw new UnsupportedOperationException("Bounded log scanning is not supported."); + } + /** * Unsubscribe from the given bucket of given partition dynamically. * diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java index 4637f87f615..d095dcfa846 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java @@ -42,6 +42,8 @@ import java.util.Collections; import java.util.ConcurrentModificationException; +import static org.apache.fluss.utils.Preconditions.checkArgument; + /** * The default impl of {@link LogScanner}. * @@ -195,7 +197,7 @@ public ArrowScanRecords pollRecordBatch(Duration timeout) { long startNanos = System.nanoTime(); do { ArrowScanRecords scanRecords = pollForRecordBatches(); - if (scanRecords.isEmpty()) { + if (!scanRecords.hasProgress()) { try { if (!logFetcher.awaitNotEmpty(startNanos + timeoutNanos)) { return scanRecords; @@ -218,7 +220,7 @@ public ArrowScanRecords pollRecordBatch(Duration timeout) { private ArrowScanRecords pollForRecordBatches() { ArrowScanRecords scanRecords = logFetcher.collectArrowFetch(); - if (!scanRecords.isEmpty()) { + if (scanRecords.hasProgress()) { return scanRecords; } @@ -235,16 +237,8 @@ public void subscribe(int bucket, long offset) { + "\"subscribe(long partitionId, int bucket, long offset)\" to " + "subscribe a partitioned bucket instead."); } - acquireAndEnsureOpen(); - try { - TableBucket tableBucket = new TableBucket(tableId, bucket); - this.logFetcher.registerTable( - new TableScanSpec(tableInfo, projection, recordBatchFilter), schemaGetter); - this.metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath)); - this.logScannerStatus.assignScanBuckets(Collections.singletonMap(tableBucket, offset)); - } finally { - release(); - } + + subscribeRange(new TableBucket(tableId, bucket), offset, NO_STOPPING_OFFSET); } @Override @@ -255,20 +249,36 @@ public void subscribe(long partitionId, int bucket, long offset) { + "\"subscribe(int bucket, long offset)\" to " + "subscribe a non-partitioned bucket instead."); } - acquireAndEnsureOpen(); - try { - TableBucket tableBucket = new TableBucket(tableId, partitionId, bucket); - this.logFetcher.registerTable( - new TableScanSpec(tableInfo, projection, recordBatchFilter), schemaGetter); - // we make assumption that the partition id must belong to the current table - // if we can't find the partition id from the table path, we'll consider the table - // is not exist - metadataUpdater.checkAndUpdatePartitionMetadata( - tablePath, Collections.singleton(partitionId)); - logScannerStatus.assignScanBuckets(Collections.singletonMap(tableBucket, offset)); - } finally { - release(); + + subscribeRange(new TableBucket(tableId, partitionId, bucket), offset, NO_STOPPING_OFFSET); + } + + @Override + public void subscribeBounded(int bucket, long startingOffset, long stoppingOffset) { + if (isPartitionedTable) { + throw new IllegalStateException( + "The table is a partitioned table, please use " + + "\"subscribeBounded(long partitionId, int bucket, long startingOffset, long stoppingOffset)\" to " + + "subscribe a partitioned bucket instead."); } + + validateOffsets(startingOffset, stoppingOffset); + subscribeRange(new TableBucket(tableId, bucket), startingOffset, stoppingOffset); + } + + @Override + public void subscribeBounded( + long partitionId, int bucket, long startingOffset, long stoppingOffset) { + if (!isPartitionedTable) { + throw new IllegalStateException( + "The table is not a partitioned table, please use " + + "\"subscribeBounded(int bucket, long startingOffset, long stoppingOffset)\" to " + + "subscribe a non-partitioned bucket instead."); + } + + validateOffsets(startingOffset, stoppingOffset); + subscribeRange( + new TableBucket(tableId, partitionId, bucket), startingOffset, stoppingOffset); } @Override @@ -302,4 +312,33 @@ public void unsubscribe(int bucket) { release(); } } + + private void subscribeRange(TableBucket tableBucket, long startingOffset, long stoppingOffset) { + Long partitionId = tableBucket.getPartitionId(); + acquireAndEnsureOpen(); + try { + logFetcher.registerTable( + new TableScanSpec(tableInfo, projection, recordBatchFilter), schemaGetter); + if (partitionId == null) { + metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(tablePath)); + } else { + metadataUpdater.checkAndUpdatePartitionMetadata( + tablePath, Collections.singleton(partitionId)); + } + logScannerStatus.assignScanBucket(tableBucket, startingOffset, stoppingOffset); + } finally { + release(); + } + } + + private static void validateOffsets(long startingOffset, long stoppingOffset) { + checkArgument( + startingOffset >= 0 || startingOffset == EARLIEST_OFFSET, + "The starting offset must be non-negative or EARLIEST_OFFSET, but was %s.", + startingOffset); + checkArgument( + stoppingOffset >= 0, + "The stopping offset must be non-negative, but was %s.", + stoppingOffset); + } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java index 8ae1391a9a2..15e3666d6df 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java @@ -25,15 +25,22 @@ import javax.annotation.concurrent.ThreadSafe; import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkState; + /** The status of a {@link LogScanner}. */ @ThreadSafe @Internal public class LogScannerStatus { private final FairBucketStatusMap bucketStatusMap; + private final Set pendingFinishedBuckets = new LinkedHashSet<>(); public LogScannerStatus() { this.bucketStatusMap = new FairBucketStatusMap<>(); @@ -62,7 +69,14 @@ synchronized void updateHighWatermark(TableBucket tableBucket, long highWatermar } synchronized void updateOffset(TableBucket tableBucket, long offset) { - bucketStatus(tableBucket).setOffset(offset); + BucketScanStatus bucketScanStatus = bucketStatus(tableBucket); + + boolean hadReachedStoppingOffset = bucketScanStatus.hasReachedStoppingOffset(); + bucketScanStatus.setOffset(offset); + + if (!hadReachedStoppingOffset && bucketScanStatus.hasReachedStoppingOffset()) { + pendingFinishedBuckets.add(tableBucket); + } } synchronized long recordsLag() { @@ -75,21 +89,34 @@ synchronized long recordsLag() { synchronized void assignScanBuckets(Map scanBucketAndOffsets) { for (Map.Entry entry : scanBucketAndOffsets.entrySet()) { - TableBucket scanBucket = entry.getKey(); - long offset = entry.getValue(); - BucketScanStatus bucketScanStatus = bucketStatusMap.statusValue(scanBucket); - if (bucketScanStatus == null) { - bucketScanStatus = new BucketScanStatus(offset); - } else { - bucketScanStatus.setOffset(offset); - } - bucketStatusMap.update(scanBucket, bucketScanStatus); + assignScanBucket(entry.getKey(), entry.getValue(), LogScanner.NO_STOPPING_OFFSET); + } + } + + synchronized void assignScanBucket( + TableBucket tableBucket, long startingOffset, long stoppingOffset) { + // A new subscription must not inherit an undelivered completion event from + // the previous subscription of the same bucket. + pendingFinishedBuckets.remove(tableBucket); + + BucketScanStatus bucketScanStatus = bucketStatus(tableBucket); + if (bucketScanStatus == null) { + bucketScanStatus = new BucketScanStatus(startingOffset); + } else { + bucketScanStatus.setOffset(startingOffset); + } + bucketScanStatus.setStoppingOffset(stoppingOffset); + bucketStatusMap.update(tableBucket, bucketScanStatus); + + if (bucketScanStatus.hasReachedStoppingOffset()) { + pendingFinishedBuckets.add(tableBucket); } } synchronized void unassignScanBuckets(List buckets) { for (TableBucket bucket : buckets) { bucketStatusMap.remove(bucket); + pendingFinishedBuckets.remove(bucket); } } @@ -99,13 +126,71 @@ synchronized List fetchableBuckets(Predicate isAvailab List result = new ArrayList<>(); bucketStatusMap.forEach( ((tableBucket, bucketScanStatus) -> { - if (isAvailable.test(tableBucket)) { + if (!bucketScanStatus.hasReachedStoppingOffset() + && isAvailable.test(tableBucket)) { result.add(tableBucket); } })); return result; } + @Nullable + synchronized Long resolveBoundedStartingOffset( + TableBucket tableBucket, long requestedOffset, long resolvedEarliestOffset) { + checkArgument( + requestedOffset == LogScanner.EARLIEST_OFFSET, + "Only EARLIEST_OFFSET can be resolved, but requested offset was %s.", + requestedOffset); + + checkArgument( + resolvedEarliestOffset >= 0, + "Resolved EARLIEST offset must be non-negative, but was %s.", + resolvedEarliestOffset); + + BucketScanStatus bucketScanStatus = bucketStatus(tableBucket); + + if (bucketScanStatus == null || bucketScanStatus.getOffset() != requestedOffset) { + return null; + } + + checkState( + bucketScanStatus.isBounded(), + "Cannot resolve an EARLIEST starting offset for unbounded bucket %s.", + tableBucket); + + long logicalOffset = Math.min(resolvedEarliestOffset, bucketScanStatus.getStoppingOffset()); + + updateOffset(tableBucket, logicalOffset); + + return logicalOffset; + } + + synchronized long getBucketStoppingOffset(TableBucket tableBucket) { + BucketScanStatus bucketScanStatus = bucketStatus(tableBucket); + return bucketScanStatus == null + ? LogScanner.NO_STOPPING_OFFSET + : bucketScanStatus.getStoppingOffset(); + } + + synchronized boolean hasReachedStoppingOffset(TableBucket tableBucket) { + BucketScanStatus bucketScanStatus = bucketStatus(tableBucket); + return bucketScanStatus != null && bucketScanStatus.hasReachedStoppingOffset(); + } + + synchronized boolean hasPendingFinishedBuckets() { + return !pendingFinishedBuckets.isEmpty(); + } + + synchronized Set drainFinishedBuckets() { + if (pendingFinishedBuckets.isEmpty()) { + return Collections.emptySet(); + } + + Set finishedBuckets = new LinkedHashSet<>(pendingFinishedBuckets); + pendingFinishedBuckets.clear(); + return finishedBuckets; + } + private BucketScanStatus bucketStatus(TableBucket tableBucket) { return bucketStatusMap.statusValue(tableBucket); } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java index 9d7d1cbc033..6a89aa97e7d 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetch.java @@ -47,6 +47,7 @@ class RemoteCompletedFetch extends CompletedFetch { LogScannerStatus logScannerStatus, boolean isCheckCrc, long fetchOffset, + long resolvedEarliestOffset, Runnable recycleCallback) { super( tableBucket, @@ -59,6 +60,7 @@ class RemoteCompletedFetch extends CompletedFetch { logScannerStatus, isCheckCrc, fetchOffset, + resolvedEarliestOffset, CompletedFetch.NO_FILTERED_END_OFFSET); this.fileLogRecords = fileLogRecords; this.recycleCallback = recycleCallback; @@ -66,6 +68,9 @@ class RemoteCompletedFetch extends CompletedFetch { @Override void drain() { + if (isConsumed()) { + return; + } super.drain(); // close file channel only, don't need to flush the file which is very heavy try { diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java index 5bb6d35681e..95a712bde10 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/RemotePendingFetch.java @@ -35,6 +35,7 @@ class RemotePendingFetch implements PendingFetch { private final int posInLogSegment; private final long fetchOffset; + private final long resolvedEarliestOffset; private final long highWatermark; private final LogRecordReadContext readContext; private final LogScannerStatus logScannerStatus; @@ -46,6 +47,7 @@ class RemotePendingFetch implements PendingFetch { TablePath tablePath, int posInLogSegment, long fetchOffset, + long resolvedEarliestOffset, long highWatermark, LogRecordReadContext readContext, LogScannerStatus logScannerStatus, @@ -55,6 +57,7 @@ class RemotePendingFetch implements PendingFetch { this.tablePath = tablePath; this.posInLogSegment = posInLogSegment; this.fetchOffset = fetchOffset; + this.resolvedEarliestOffset = resolvedEarliestOffset; this.highWatermark = highWatermark; this.readContext = readContext; this.logScannerStatus = logScannerStatus; @@ -83,6 +86,7 @@ public CompletedFetch toCompletedFetch() { logScannerStatus, isCheckCrc, fetchOffset, + resolvedEarliestOffset, downloadFuture.getRecycleCallback()); } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java index ebeab20c266..8870d5bdd00 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java @@ -47,6 +47,9 @@ public class ScanRecords implements Iterable { /** The exclusive upper bound of consumed offsets per polled bucket in this round. */ private final Map consumedUpToOffsets; + /** The bounded buckets that reached their stopping offsets in this poll round. */ + private final Set finishedBuckets; + public ScanRecords(Map> records) { this(records, Collections.emptyMap()); } @@ -54,25 +57,43 @@ public ScanRecords(Map> records) { public ScanRecords( Map> records, Map consumedUpToOffsets) { - this.records = withProgressOnlyBuckets(records, consumedUpToOffsets); + this(records, consumedUpToOffsets, Collections.emptySet()); + } + + ScanRecords( + Map> records, + Map consumedUpToOffsets, + Set finishedBuckets) { + this.records = withProgressOrFinishedBuckets(records, consumedUpToOffsets, finishedBuckets); this.consumedUpToOffsets = consumedUpToOffsets; + this.finishedBuckets = finishedBuckets; } /** - * Ensures every bucket with a consumed offset has a (possibly empty) record list entry, so that - * {@link #buckets()} surfaces progress-only buckets as documented. Returns the original map - * when the invariant already holds (the common case for collector-produced results). + * Ensures every bucket with a consumed offset or completion event has a (possibly empty) record + * list entry, so that {@link #buckets()} surfaces progress-only and finished-only buckets as + * documented. Returns the original map when the invariant already holds (the common case for + * collector-produced results). */ - private static Map> withProgressOnlyBuckets( + private static Map> withProgressOrFinishedBuckets( Map> records, - Map consumedUpToOffsets) { - if (records.keySet().containsAll(consumedUpToOffsets.keySet())) { + Map consumedUpToOffsets, + Set finishedBuckets) { + if (records.keySet().containsAll(consumedUpToOffsets.keySet()) + && records.keySet().containsAll(finishedBuckets)) { return records; } + Map> merged = new LinkedHashMap<>(records); + for (TableBucket bucket : consumedUpToOffsets.keySet()) { merged.putIfAbsent(bucket, Collections.emptyList()); } + + for (TableBucket bucket : finishedBuckets) { + merged.putIfAbsent(bucket, Collections.emptyList()); + } + return merged; } @@ -91,7 +112,7 @@ public List records(TableBucket scanBucket) { /** * Get the bucket ids that were polled in this round, including buckets whose record list is - * empty but whose log offset still advanced. + * empty but whose log offset still advanced or whose bounded subscription finished. */ public Set buckets() { return Collections.unmodifiableSet(records.keySet()); @@ -124,13 +145,25 @@ public boolean isEmpty() { } /** - * Returns {@code true} if this {@code ScanRecords} carries any scanner progress. + * Returns {@code true} if this {@code ScanRecords} carries any scanner progress, either by + * returning records, advancing a consumed offset, or completing a bounded subscription. * *

A result may have progress even when {@link #isEmpty()} is {@code true}, for example when * a bucket returns no materialized records but advances its consumed offset. */ public boolean hasProgress() { - return count() > 0 || !consumedUpToOffsets.isEmpty(); + return count() > 0 || !consumedUpToOffsets.isEmpty() || !finishedBuckets.isEmpty(); + } + + /** + * Returns the bounded buckets that finished in this poll round, including empty ranges. + * + *

Each subscription reports completion only once. A completion event may accompany the final + * records or arrive without any records. Callers must consume all records in this result before + * treating the corresponding buckets as fully read. + */ + public Set finishedBuckets() { + return Collections.unmodifiableSet(finishedBuckets); } @Override diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecordsTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecordsTest.java new file mode 100644 index 00000000000..0cf18356a05 --- /dev/null +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecordsTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.client.table.scanner.log; + +import org.apache.fluss.metadata.TableBucket; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link ArrowScanRecords}. */ +public class ArrowScanRecordsTest { + private static final TableBucket TABLE_BUCKET = new TableBucket(1L, 0); + + @Test + void testFinishedOnlyBucketHasProgress() { + try (ArrowScanRecords records = + new ArrowScanRecords( + Collections.emptyMap(), + Collections.emptyMap(), + Collections.singleton(TABLE_BUCKET))) { + + assertThat(records.count()).isZero(); + assertThat(records.hasProgress()).isTrue(); + assertThat(records.buckets()).containsExactly(TABLE_BUCKET); + assertThat(records.records(TABLE_BUCKET)).isEmpty(); + assertThat(records.consumedUpToOffset(TABLE_BUCKET)).isNull(); + assertThat(records.finishedBuckets()).containsExactly(TABLE_BUCKET); + } + } +} diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java new file mode 100644 index 00000000000..1096465f33d --- /dev/null +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.client.table.scanner.log; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.utils.clock.ManualClock; + +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.Optional; + +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.testutils.DataTestUtils.row; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests bounded subscriptions after retention advances the server's earliest offset. */ +@Execution(ExecutionMode.SAME_THREAD) +class BoundedLogScannerRetentionITCase { + + private static final ManualClock CLOCK = new ManualClock(System.currentTimeMillis()); + + private static final int EXPIRED_RECORDS = 10; + private static final long STOPPING_OFFSET = 5L; + + @RegisterExtension + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfig()) + .setClock(CLOCK) + .build(); + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testEarliestBeyondStoppingOffset(boolean retainRecords) throws Exception { + TablePath tablePath = TablePath.of("bounded_retention", "logs_" + retainRecords); + + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + + admin.createDatabase(tablePath.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); + + admin.createTable( + tablePath, + TableDescriptor.builder() + .schema(DATA1_SCHEMA) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_LOG_TTL, Duration.ofHours(1)) + .build(), + false) + .get(); + + long tableId = admin.getTableInfo(tablePath).get().getTableId(); + + TableBucket bucket = new TableBucket(tableId, 0); + LogTablet logTablet = CLUSTER.waitAndGetLeaderReplica(bucket).getLogTablet(); + + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + appendExpiredRecords(writer); + retry( + Duration.ofSeconds(10), + () -> + assertThat(logTablet.getHighWatermark()) + .isEqualTo((long) EXPIRED_RECORDS)); + + // Move the written records into a closed segment so that retention + // can remove the whole segment. + logTablet.roll(Optional.empty()); + CLOCK.advanceTime(Duration.ofHours(2)); + logTablet.deleteExpiredSegments(); + + assertThat(logTablet.logStartOffset()).isEqualTo((long) EXPIRED_RECORDS); + assertThat(logTablet.localLogEndOffset()).isEqualTo((long) EXPIRED_RECORDS); + + if (retainRecords) { + writer.append(row(EXPIRED_RECORDS, "retained")); + writer.flush(); + retry( + Duration.ofSeconds(10), + () -> + assertThat(logTablet.getHighWatermark()) + .isEqualTo(EXPIRED_RECORDS + 1L)); + } + + assertThat(logTablet.logStartOffset()).isEqualTo((long) EXPIRED_RECORDS); + + // Exercise both scanner APIs against the same retained log state. + assertBoundedScanFinishes(table, bucket, false); + assertBoundedScanFinishes(table, bucket, true); + } + } + } + + private static void appendExpiredRecords(AppendWriter writer) { + for (int i = 0; i < EXPIRED_RECORDS; i++) { + writer.append(row(i, "expired")); + } + + // flush() waits for all previously appended records to complete, avoiding + // one synchronous round trip per record. + writer.flush(); + } + + private static void assertBoundedScanFinishes( + Table table, TableBucket bucket, boolean useArrowPoll) { + try (LogScannerImpl scanner = (LogScannerImpl) table.newScan().createLogScanner()) { + + scanner.subscribeBounded(0, LogScanner.EARLIEST_OFFSET, STOPPING_OFFSET); + long deadline = System.nanoTime() + Duration.ofSeconds(10).toNanos(); + boolean finished = false; + while (!finished) { + assertThat(System.nanoTime()) + .as( + "Bounded scan must finish without waiting for " + + "new records, useArrowPoll=%s", + useArrowPoll) + .isLessThan(deadline); + + finished = pollEmptyRange(scanner, useArrowPoll, bucket); + } + + // Once the bucket is finished, subsequent non-blocking polls must + // not report the same progress again. + if (useArrowPoll) { + try (ArrowScanRecords records = scanner.pollRecordBatch(Duration.ZERO)) { + assertThat(records.hasProgress()).isFalse(); + } + } else { + assertThat(scanner.poll(Duration.ZERO).hasProgress()).isFalse(); + } + } + } + + private static boolean pollEmptyRange( + LogScannerImpl scanner, boolean useArrowPoll, TableBucket bucket) { + if (useArrowPoll) { + try (ArrowScanRecords records = scanner.pollRecordBatch(Duration.ofMillis(100))) { + assertThat(records.count()).isZero(); + if (records.finishedBuckets().contains(bucket)) { + assertThat(records.finishedBuckets()).containsExactly(bucket); + assertThat(records.consumedUpToOffset(bucket)).isEqualTo(STOPPING_OFFSET); + return true; + } + } + } else { + ScanRecords records = scanner.poll(Duration.ofMillis(100)); + assertThat(records.count()).isZero(); + if (records.finishedBuckets().contains(bucket)) { + assertThat(records.finishedBuckets()).containsExactly(bucket); + assertThat(records.consumedUpToOffset(bucket)).isEqualTo(STOPPING_OFFSET); + return true; + } + } + return false; + } + + private static Configuration clusterConfig() { + Configuration conf = new Configuration(); + conf.setInt(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + // Exercise local retention deterministically, without waiting for + // remote uploads. + conf.set(ConfigOptions.REMOTE_LOG_TASK_INTERVAL_DURATION, Duration.ZERO); + return conf; + } +} diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java new file mode 100644 index 00000000000..bed360d4a5d --- /dev/null +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java @@ -0,0 +1,413 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.client.table.scanner.log; + +import org.apache.fluss.client.metadata.TestingMetadataUpdater; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.AuthorizationException; +import org.apache.fluss.exception.FetchException; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.ArrowBatchData; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.rpc.entity.FetchLogResultForBucket; +import org.apache.fluss.rpc.protocol.ApiError; +import org.apache.fluss.rpc.protocol.Errors; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; +import static org.apache.fluss.record.TestData.DATA1; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER; +import static org.apache.fluss.testutils.DataTestUtils.createBasicMemoryLogRecords; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests error delivery alongside progress and completion for both collector formats. */ +class LogFetchCollectorErrorTest { + private final TableBucket finishedBucket = new TableBucket(DATA1_TABLE_ID, 0); + private final TableBucket errorBucket = new TableBucket(DATA1_TABLE_ID, 1); + private final LogScannerStatus status = new LogScannerStatus(); + private final LogFetchBuffer buffer = new LogFetchBuffer(); + private final LogRecordReadContext readContext = + LogRecordReadContext.createArrowReadContext( + DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER); + + @AfterEach + void close() throws Exception { + buffer.close(); + readContext.close(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testAuthorizationErrorDoesNotTrapPendingCompletion(boolean arrow) { + AbstractLogFetchCollector collector = collector(arrow); + status.assignScanBucket(finishedBucket, 0L, 0L); + CompletedFetch failed = addError(Errors.AUTHORIZATION_EXCEPTION); + + assertThatThrownBy(() -> collector.collectFetch(buffer)) + .isInstanceOf(AuthorizationException.class); + assertThat(buffer.peek()).isNull(); + assertThat(failed.isConsumed()).isTrue(); + assertThat(status.hasPendingFinishedBuckets()).isTrue(); + assertThat(collectProgressOnly(collector).finishedBuckets()) + .containsExactly(finishedBucket); + assertThat(collectProgressOnly(collector).hasProgress()).isFalse(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCompletionDefersFetchErrorUntilNextPoll(boolean arrow) { + AbstractLogFetchCollector collector = collector(arrow); + status.assignScanBucket(finishedBucket, 0L, 0L); + CompletedFetch failed = addError(Errors.LOG_OFFSET_OUT_OF_RANGE_EXCEPTION); + + ScanRecords result = collectProgressOnly(collector); + assertThat(result.finishedBuckets()).containsExactly(finishedBucket); + assertThat(result.isEmpty()).isTrue(); + assertThat(buffer.peek()).isSameAs(failed); + assertThat(failed.isConsumed()).isFalse(); + assertThatThrownBy(() -> collector.collectFetch(buffer)).isInstanceOf(FetchException.class); + assertThat(failed.isConsumed()).isTrue(); + assertThat(buffer.peek()).isNull(); + assertThat(collectProgressOnly(collector).hasProgress()).isFalse(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testProgressDefersFetchErrorUntilNextPoll(boolean arrow) { + AbstractLogFetchCollector collector = collector(arrow); + status.assignScanBucket(finishedBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + buffer.add(makeFetch(FetchLogResultForBucket.empty(finishedBucket, 10L, 10L))); + CompletedFetch failed = addError(Errors.CORRUPT_MESSAGE); + + ScanRecords result = collectProgressOnly(collector); + assertThat(result.consumedUpToOffset(finishedBucket)).isEqualTo(10L); + assertThat(result.finishedBuckets()).isEmpty(); + assertThat(result.isEmpty()).isTrue(); + assertThat(buffer.peek()).isSameAs(failed); + assertThatThrownBy(() -> collector.collectFetch(buffer)).isInstanceOf(FetchException.class); + assertThat(failed.isConsumed()).isTrue(); + assertThat(buffer.peek()).isNull(); + assertThat(collectProgressOnly(collector).hasProgress()).isFalse(); + } + + @ParameterizedTest + @CsvSource({"false, false", "false, true", "true, false", "true, true"}) + void testRecordErrorDefersForProgressOrCompletion(boolean arrow, boolean completion) { + AbstractLogFetchCollector collector = collector(arrow); + if (completion) { + status.assignScanBucket(finishedBucket, 0L, 0L); + } else { + status.assignScanBucket(finishedBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + buffer.add(makeFetch(FetchLogResultForBucket.empty(finishedBucket, 10L, 10L))); + } + status.assignScanBucket(errorBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + FetchException failure = new FetchException("record decoding failed"); + CompletedFetch failed = + new DefaultCompletedFetch( + errorBucket, + DATA1_TABLE_PATH, + FetchLogResultForBucket.empty(errorBucket, 10L, -1L), + readContext, + status, + true, + 0L, + null) { + @Override + public List fetchRecords(int maxRecords) { + throw failure; + } + + @Override + List fetchArrowBatches(int maxRecords) { + throw failure; + } + }; + buffer.add(failed); + CompletedFetch queued = makeFetch(FetchLogResultForBucket.empty(errorBucket, 10L, -1L)); + buffer.add(queued); + + ScanRecords result = collectProgressOnly(collector); + assertThat(result.hasProgress()).isTrue(); + assertThat(result.isEmpty()).isTrue(); + if (completion) { + assertThat(result.finishedBuckets()).containsExactly(finishedBucket); + } else { + assertThat(result.consumedUpToOffset(finishedBucket)).isEqualTo(10L); + } + assertThat(buffer.nextInLineFetch()).isSameAs(failed); + assertThat(buffer.peek()).isSameAs(queued); + assertThatThrownBy(() -> collector.collectFetch(buffer)).isSameAs(failure); + // Record errors stay attached to the in-flight fetch; do not dequeue another response. + assertThat(buffer.nextInLineFetch()).isSameAs(failed); + assertThat(buffer.peek()).isSameAs(queued); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testAuthorizationErrorIsDeferredAfterAccumulatedRecords(boolean arrow) throws Exception { + AbstractLogFetchCollector collector = collector(arrow); + + long stoppingOffset = DATA1.size(); + status.assignScanBucket(finishedBucket, 0L, stoppingOffset); + + // Bucket A: consuming these records reaches the bounded stopping offset. + buffer.add( + makeFetch( + FetchLogResultForBucket.records( + finishedBucket, data1Records(), stoppingOffset, -1L, -1L))); + + // Bucket B: processed after bucket A and fails with a non-FetchException. + CompletedFetch failed = addError(Errors.AUTHORIZATION_EXCEPTION); + + long initialMemory = readContext.getBufferAllocator().getAllocatedMemory(); + + // The authorization error must not discard records that were already consumed + // during this poll. Deliver bucket A's records and completion first. + if (arrow) { + try (ArrowScanRecords result = (ArrowScanRecords) collector.collectFetch(buffer)) { + assertThat(result.count()).isEqualTo(DATA1.size()); + assertThat(result.records(finishedBucket)).isNotEmpty(); + assertThat(result.consumedUpToOffset(finishedBucket)).isEqualTo(stoppingOffset); + assertThat(result.finishedBuckets()).containsExactly(finishedBucket); + assertThat(result.hasProgress()).isTrue(); + + // The failed fetch must remain queued for the next poll. + assertThat(buffer.peek()).isSameAs(failed); + assertThat(failed.isConsumed()).isFalse(); + } + + // The successfully delivered Arrow result owns its buffers. Closing that + // result must release them normally. + assertThat(readContext.getBufferAllocator().getAllocatedMemory()) + .isEqualTo(initialMemory); + } else { + ScanRecords result = (ScanRecords) collector.collectFetch(buffer); + + assertThat(result.count()).isEqualTo(DATA1.size()); + assertThat(result.records(finishedBucket)).hasSize(DATA1.size()); + assertThat(result.consumedUpToOffset(finishedBucket)).isEqualTo(stoppingOffset); + assertThat(result.finishedBuckets()).containsExactly(finishedBucket); + assertThat(result.hasProgress()).isTrue(); + + // The failed fetch must remain queued for the next poll. + assertThat(buffer.peek()).isSameAs(failed); + assertThat(failed.isConsumed()).isFalse(); + } + + // Completion was delivered together with bucket A's final records, so it must + // no longer remain pending. + assertThat(status.hasPendingFinishedBuckets()).isFalse(); + + // With no newly accumulated result in this poll, the authorization error is + // now delivered immediately. + assertThatThrownBy(() -> collector.collectFetch(buffer)) + .isInstanceOf(AuthorizationException.class); + + assertThat(failed.isConsumed()).isTrue(); + assertThat(buffer.peek()).isNull(); + + // Neither records, progress, nor completion may be repeated. + assertThat(collectProgressOnly(collector).hasProgress()).isFalse(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testAuthorizationErrorIsDeferredAfterBoundedProgress(boolean arrow) { + AbstractLogFetchCollector collector = collector(arrow); + + status.assignScanBucket(finishedBucket, 0L, 20L); + + // No records, but bounded scanner progress advances from 0 to 10. + buffer.add(makeFetch(FetchLogResultForBucket.empty(finishedBucket, 20L, 10L))); + + CompletedFetch failed = addError(Errors.AUTHORIZATION_EXCEPTION); + + ScanRecords first = collectProgressOnly(collector); + + assertThat(first.isEmpty()).isTrue(); + assertThat(first.hasProgress()).isTrue(); + assertThat(first.consumedUpToOffset(finishedBucket)).isEqualTo(10L); + assertThat(first.finishedBuckets()).isEmpty(); + + assertThat(buffer.peek()).isSameAs(failed); + assertThat(failed.isConsumed()).isFalse(); + + assertThatThrownBy(() -> collector.collectFetch(buffer)) + .isInstanceOf(AuthorizationException.class); + + assertThat(failed.isConsumed()).isTrue(); + assertThat(buffer.peek()).isNull(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testAuthorizationErrorRemainsImmediateAfterUnboundedRecords(boolean arrow) + throws Exception { + AbstractLogFetchCollector collector = collector(arrow); + status.assignScanBucket(finishedBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + + buffer.add( + makeFetch( + FetchLogResultForBucket.records( + finishedBucket, data1Records(), DATA1.size(), -1L, -1L))); + + CompletedFetch failed = addError(Errors.AUTHORIZATION_EXCEPTION); + + long initialMemory = readContext.getBufferAllocator().getAllocatedMemory(); + + // Legacy unbounded behavior: the authorization error is propagated + // immediately even though records were already accumulated. + assertThatThrownBy(() -> collector.collectFetch(buffer)) + .isInstanceOf(AuthorizationException.class); + + // Scanner progress from A is preserved even though A's records were not + // returned from this poll. + assertThat(status.getBucketOffset(finishedBucket)).isEqualTo(DATA1.size()); + + if (arrow) { + // Accumulated Arrow records must be released when the poll fails. + assertThat(readContext.getBufferAllocator().getAllocatedMemory()) + .isEqualTo(initialMemory); + } + + // Preserve the parent queue semantics: because this poll had already + // accumulated a result, the zero-byte authorization error remains queued. + assertThat(failed.isConsumed()).isFalse(); + assertThat(buffer.peek()).isSameAs(failed); + + // With no accumulated result in the next poll, the same error is propagated + // again and is now removed from the queue. + assertThatThrownBy(() -> collector.collectFetch(buffer)) + .isInstanceOf(AuthorizationException.class); + + assertThat(failed.isConsumed()).isTrue(); + assertThat(buffer.peek()).isNull(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testOffsetOutOfRangeReportsRequestedOffset(boolean arrow) { + AbstractLogFetchCollector collector = collector(arrow); + long requestedOffset = 123L; + + status.assignScanBucket(errorBucket, requestedOffset, LogScanner.NO_STOPPING_OFFSET); + + CompletedFetch failed = + makeFetch( + FetchLogResultForBucket.error( + errorBucket, + new ApiError(Errors.LOG_OFFSET_OUT_OF_RANGE_EXCEPTION, "failure")), + requestedOffset); + + buffer.add(failed); + + assertThatThrownBy(() -> collector.collectFetch(buffer)) + .isInstanceOf(FetchException.class) + .hasMessageContaining("fetching offset 123"); + } + + private AbstractLogFetchCollector collector(boolean arrow) { + TestingMetadataUpdater metadata = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + return arrow + ? new ArrowLogFetchCollector(status, new Configuration(), metadata) + : new LogFetchCollector(status, new Configuration(), metadata); + } + + private CompletedFetch addError(Errors error) { + status.assignScanBucket(errorBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + CompletedFetch fetch = + makeFetch( + FetchLogResultForBucket.error(errorBucket, new ApiError(error, "failure"))); + buffer.add(fetch); + return fetch; + } + + private CompletedFetch makeFetch(FetchLogResultForBucket result) { + return makeFetch(result, 0L); + } + + private CompletedFetch makeFetch(FetchLogResultForBucket result, long requestedOffset) { + return new DefaultCompletedFetch( + result.getTableBucket(), + DATA1_TABLE_PATH, + result, + readContext, + status, + true, + requestedOffset, + null); + } + + private ScanRecords collectProgressOnly(AbstractLogFetchCollector collector) { + Object result = collector.collectFetch(buffer); + if (result instanceof ScanRecords) { + return (ScanRecords) result; + } + try (ArrowScanRecords records = (ArrowScanRecords) result) { + assertThat(records.count()).isZero(); + Map offsets = new HashMap<>(); + for (TableBucket bucket : records.buckets()) { + Long offset = records.consumedUpToOffset(bucket); + if (offset != null) { + offsets.put(bucket, offset); + } + } + return new ScanRecords(Collections.emptyMap(), offsets, records.finishedBuckets()); + } + } + + private MemoryLogRecords data1Records() throws Exception { + return createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 0L, + 0L, + LogRecordBatch.CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + Collections.nCopies(DATA1.size(), ChangeType.APPEND_ONLY), + DATA1, + LogFormat.ARROW, + DEFAULT_COMPRESSION, + true); + } +} diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java index 064512a568a..d020bd7eccd 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java @@ -20,11 +20,11 @@ import org.apache.fluss.client.metadata.MetadataUpdater; import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.table.scanner.ScanRecord; -import org.apache.fluss.compression.ArrowCompressionInfo; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.ArrowBatchData; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.LogRecordBatch; import org.apache.fluss.record.LogRecordReadContext; @@ -34,13 +34,23 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.LongSupplier; +import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; +import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; import static org.apache.fluss.record.TestData.DATA1; import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; @@ -51,6 +61,7 @@ import static org.apache.fluss.testutils.DataTestUtils.createBasicMemoryLogRecords; import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link LogFetchCollector}. */ public class LogFetchCollectorTest { @@ -86,6 +97,310 @@ void afterEach() { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testBoundedEarliestRequiresResolvedOffset(boolean arrow) { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, EARLIEST_OFFSET, 50L); + logFetchBuffer.add( + makeCompletedFetch( + tb, FetchLogResultForBucket.empty(tb, 100L, -1L), EARLIEST_OFFSET)); + AbstractLogFetchCollector collector = + arrow + ? new ArrowLogFetchCollector( + logScannerStatus, + new Configuration(), + new TestingMetadataUpdater( + Collections.singletonMap( + DATA1_TABLE_PATH, DATA1_TABLE_INFO))) + : logFetchCollector; + + assertThatThrownBy(() -> collector.collectFetch(logFetchBuffer)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("resolved_earliest_offset"); + assertThat(logFetchBuffer.peek()).isNull(); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(EARLIEST_OFFSET); + } + + @ParameterizedTest + @CsvSource({ + // arrow, startingOffset, stoppingOffset, resolvedOffset, expectedOffset, finished + "false, -2, 50, 100, 50, true", + "true, -2, 50, 100, 50, true", + "false, -2, 100, 100, 100, true", + "true, -2, 100, 100, 100, true", + "false, -2, 150, 100, 100, false", + "true, -2, 150, 100, 100, false", + "false, -2, 50, 0, 0, false", + "true, -2, 50, 0, 0, false", + "false, 10, 50, -1, 10, false", + "true, 10, 50, -1, 10, false" + }) + void testResolvedBoundedStartingOffset( + boolean arrow, + long startingOffset, + long stoppingOffset, + long resolvedOffset, + long expectedOffset, + boolean finished) { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, startingOffset, stoppingOffset); + CompletedFetch fetch = + makeCompletedFetch( + tb, + FetchLogResultForBucket.records( + tb, MemoryLogRecords.EMPTY, 100L, -1L, -1L, resolvedOffset), + startingOffset); + + logFetchBuffer.add(fetch); + boolean expectedProgress = startingOffset != expectedOffset; + + if (arrow) { + ArrowLogFetchCollector collector = + new ArrowLogFetchCollector( + logScannerStatus, + new Configuration(), + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO))); + + try (ArrowScanRecords result = collector.collectFetch(logFetchBuffer)) { + assertThat(result.count()).isZero(); + assertThat(result.records(tb)).isEmpty(); + assertThat(result.hasProgress()).isEqualTo(expectedProgress); + if (finished) { + assertThat(result.finishedBuckets().contains(tb)); + } else { + assertThat(result.finishedBuckets().isEmpty()); + } + if (expectedProgress) { + assertThat(result.consumedUpToOffset(tb)).isEqualTo(expectedOffset); + } else { + assertThat(result.consumedUpToOffset(tb)).isNull(); + } + } + + // Completion/progress from this response must be one-shot. + try (ArrowScanRecords result = collector.collectFetch(logFetchBuffer)) { + assertThat(result.hasProgress()).isFalse(); + } + } else { + ScanRecords result = logFetchCollector.collectFetch(logFetchBuffer); + assertThat(result.isEmpty()).isTrue(); + assertThat(result.records(tb)).isEmpty(); + assertThat(result.hasProgress()).isEqualTo(expectedProgress); + if (finished) { + assertThat(result.finishedBuckets().contains(tb)); + } else { + assertThat(result.finishedBuckets().isEmpty()); + } + if (expectedProgress) { + assertThat(result.consumedUpToOffset(tb)).isEqualTo(expectedOffset); + } else { + assertThat(result.consumedUpToOffset(tb)).isNull(); + } + + // Completion/progress from this response must be one-shot. + assertThat(logFetchCollector.collectFetch(logFetchBuffer).hasProgress()).isFalse(); + } + + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(expectedOffset); + assertThat(logScannerStatus.hasReachedStoppingOffset(tb)).isEqualTo(finished); + assertThat(fetch.isConsumed()).isTrue(); + } + + @ParameterizedTest + @ValueSource(longs = {-1L, 100L}) + void testUnboundedEarliestKeepsEmptyFetchBehavior(long resolvedOffset) { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBuckets(Collections.singletonMap(tb, EARLIEST_OFFSET)); + logFetchBuffer.add( + makeCompletedFetch( + tb, + FetchLogResultForBucket.records( + tb, MemoryLogRecords.EMPTY, 100L, -1L, -1L, resolvedOffset), + EARLIEST_OFFSET)); + + assertThat(logFetchCollector.collectFetch(logFetchBuffer).hasProgress()).isTrue(); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(EARLIEST_OFFSET); + } + + @Test + void testDiscardStaleResolvedStartingOffset() { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, 10L, 50L); + CompletedFetch fetch = + makeCompletedFetch( + tb, + FetchLogResultForBucket.records( + tb, MemoryLogRecords.EMPTY, 100L, -1L, -1L, 100L), + EARLIEST_OFFSET); + logFetchBuffer.add(fetch); + + assertThat(logFetchCollector.collectFetch(logFetchBuffer).hasProgress()).isFalse(); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(10L); + assertThat(fetch.isConsumed()).isTrue(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testResolvedStartingOffsetInsideBatch(boolean arrow) throws Exception { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + + // The logical starting offset is EARLIEST_OFFSET, while the server resolves it to 100. + // The bounded subscription should stop before offset 103. + logScannerStatus.assignScanBucket(tb, EARLIEST_OFFSET, 103L); + + // The physical batch starts from offset 95, which means records before the resolved + // starting offset 100 must be skipped by the collector. + MemoryLogRecords records = + createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 95L, + 0L, + LogRecordBatch.CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + Collections.nCopies(DATA1.size(), ChangeType.APPEND_ONLY), + DATA1, + LogFormat.ARROW, + DEFAULT_COMPRESSION, + true); + + FetchLogResultForBucket result = + FetchLogResultForBucket.records(tb, records, 105L, -1L, -1L, 100L); + + CompletedFetch fetch = makeCompletedFetch(tb, result, EARLIEST_OFFSET); + + logFetchBuffer.add(fetch); + + MetadataUpdater metadata = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + + if (arrow) { + ArrowLogFetchCollector collector = + new ArrowLogFetchCollector(logScannerStatus, new Configuration(), metadata); + + try (ArrowScanRecords scanRecords = collector.collectFetch(logFetchBuffer)) { + + assertThat(scanRecords.count()).isEqualTo(3); + assertThat(scanRecords.records(tb)).hasSize(1); + + ArrowBatchData batch = scanRecords.records(tb).get(0); + assertThat(batch.getBaseLogOffset()).isEqualTo(100L); + assertThat(batch.getRecordCount()).isEqualTo(3); + assertThat(batch.getVectorSchemaRoot().getRowCount()).isEqualTo(3); + + assertThat(scanRecords.consumedUpToOffset(tb)).isEqualTo(103L); + assertThat(scanRecords.finishedBuckets()).containsExactly(tb); + } + } else { + LogFetchCollector collector = + new LogFetchCollector(logScannerStatus, new Configuration(), metadata); + + ScanRecords scanRecords = collector.collectFetch(logFetchBuffer); + + assertThat(scanRecords.records(tb)) + .extracting(ScanRecord::logOffset) + .containsExactly(100L, 101L, 102L); + + assertThat(scanRecords.consumedUpToOffset(tb)).isEqualTo(103L); + assertThat(scanRecords.finishedBuckets()).containsExactly(tb); + } + + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(103L); + assertThat(fetch.isConsumed()).isTrue(); + } + + @Test + void testResolvedStartingOffsetInsideBatchWithMaxPollRecords() throws Exception { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + + logScannerStatus.assignScanBucket(tb, EARLIEST_OFFSET, 103L); + + MemoryLogRecords records = + createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 95L, + 0L, + LogRecordBatch.CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + Collections.nCopies(DATA1.size(), ChangeType.APPEND_ONLY), + DATA1, + LogFormat.ARROW, + DEFAULT_COMPRESSION, + true); + + FetchLogResultForBucket result = + FetchLogResultForBucket.records(tb, records, 105L, -1L, -1L, 100L); + + CompletedFetch fetch = makeCompletedFetch(tb, result, EARLIEST_OFFSET); + + logFetchBuffer.add(fetch); + + Configuration conf = new Configuration(); + conf.setInt(ConfigOptions.CLIENT_SCANNER_LOG_MAX_POLL_RECORDS, 2); + + MetadataUpdater metadata = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + + LogFetchCollector collector = new LogFetchCollector(logScannerStatus, conf, metadata); + + // Only two records can be returned in the first poll. Although the physical batch + // starts at 95, records before the resolved starting offset 100 must not count + // towards max.poll.records. + ScanRecords first = collector.collectFetch(logFetchBuffer); + + assertThat(first.records(tb)).extracting(ScanRecord::logOffset).containsExactly(100L, 101L); + assertThat(first.consumedUpToOffset(tb)).isEqualTo(102L); + assertThat(first.finishedBuckets()).isEmpty(); + + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(102L); + assertThat(fetch.isConsumed()).isFalse(); + + // The remaining record before the bounded end offset is returned by the next poll. + ScanRecords second = collector.collectFetch(logFetchBuffer); + + assertThat(second.records(tb)).extracting(ScanRecord::logOffset).containsExactly(102L); + assertThat(second.consumedUpToOffset(tb)).isEqualTo(103L); + assertThat(second.finishedBuckets()).containsExactly(tb); + + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(103L); + assertThat(fetch.isConsumed()).isTrue(); + } + + @Test + void testBoundedEarliestWithoutResolutionDrainsNonEmptyFetch() throws Exception { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + + logScannerStatus.assignScanBucket(tb, LogScanner.EARLIEST_OFFSET, 200L); + + // Intentionally use the old overload to simulate a response from an old server: + // records are present, but resolved_earliest_offset is absent. + FetchLogResultForBucket result = + FetchLogResultForBucket.records( + tb, genMemoryLogRecordsByObject(DATA1), 200L, -1L, -1L); + + CompletedFetch fetch = makeCompletedFetch(tb, result, LogScanner.EARLIEST_OFFSET); + + logFetchBuffer.add(fetch); + + assertThatThrownBy(() -> logFetchCollector.collectFetch(logFetchBuffer)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("resolved_earliest_offset"); + + assertThat(fetch.isConsumed()).isTrue(); + assertThat(logFetchBuffer.peek()).isNull(); + assertThat(logFetchBuffer.isEmpty()).isTrue(); + + // Failed resolution must not mutate the scanner position. + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(LogScanner.EARLIEST_OFFSET); + } + @Test void testNormal() throws Exception { long fetchOffset = 0L; @@ -319,12 +634,12 @@ void testUpdateBeforeAndAfterNeverSplitAcrossPolls() throws Exception { 0L, System.currentTimeMillis(), LogRecordBatch.CURRENT_LOG_MAGIC_VALUE, - org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID, - org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, changeTypes, objects, LogFormat.ARROW, - ArrowCompressionInfo.DEFAULT_COMPRESSION); + DEFAULT_COMPRESSION); Configuration conf = new Configuration(); conf.setInt(ConfigOptions.CLIENT_SCANNER_LOG_MAX_POLL_RECORDS, 1); @@ -359,4 +674,406 @@ void testUpdateBeforeAndAfterNeverSplitAcrossPolls() throws Exception { assertThat(records3).hasSize(1); assertThat(records3.get(0).getChangeType()).isEqualTo(ChangeType.INSERT); } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testEmptyRangeFromBeginningReportsCompletion(boolean arrow) { + TableBucket bucket = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(bucket, EARLIEST_OFFSET, 0L); + + MetadataUpdater metadata = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + + if (arrow) { + ArrowLogFetchCollector collector = + new ArrowLogFetchCollector(logScannerStatus, new Configuration(), metadata); + try (ArrowScanRecords records = collector.collectFetch(logFetchBuffer)) { + assertThat(records.count()).isZero(); + assertThat(records.hasProgress()).isTrue(); + assertThat(records.finishedBuckets()).containsExactly(bucket); + } + try (ArrowScanRecords records = collector.collectFetch(logFetchBuffer)) { + assertThat(records.hasProgress()).isFalse(); + } + } else { + ScanRecords records = logFetchCollector.collectFetch(logFetchBuffer); + assertThat(records.isEmpty()).isTrue(); + assertThat(records.hasProgress()).isTrue(); + assertThat(records.finishedBuckets()).containsExactly(bucket); + assertThat(logFetchCollector.collectFetch(logFetchBuffer).hasProgress()).isFalse(); + } + } + + @ParameterizedTest + @CsvSource({ + // arrow, stoppingOffset, filteredEndOffset, expectedOffset, finished + "false, 5, 20, 5, true", + "true, 5, 20, 5, true", + "false, 10, 5, 5, false", + "true, 10, 5, 5, false" + }) + void testBoundedFilteredEmptyResponse( + boolean arrow, + long stoppingOffset, + long filteredEndOffset, + long expectedOffset, + boolean finished) { + + MetadataUpdater metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, 0L, stoppingOffset); + + FetchLogResultForBucket filteredEmpty = + FetchLogResultForBucket.empty(tb, 10L, filteredEndOffset); + CompletedFetch completedFetch = makeCompletedFetch(tb, filteredEmpty, 0L); + logFetchBuffer.add(completedFetch); + + if (arrow) { + ArrowLogFetchCollector collector = + new ArrowLogFetchCollector( + logScannerStatus, new Configuration(), metadataUpdater); + + try (ArrowScanRecords records = collector.collectFetch(logFetchBuffer)) { + assertThat(records.count()).isZero(); + assertThat(records.hasProgress()).isTrue(); + assertThat(records.records(tb)).isEmpty(); + assertThat(records.buckets()).containsExactly(tb); + assertThat(records.consumedUpToOffset(tb)).isEqualTo(expectedOffset); + + if (finished) { + assertThat(records.finishedBuckets()).containsExactly(tb); + } else { + assertThat(records.finishedBuckets()).isEmpty(); + } + } + } else { + LogFetchCollector collector = + new LogFetchCollector(logScannerStatus, new Configuration(), metadataUpdater); + + ScanRecords records = collector.collectFetch(logFetchBuffer); + + assertThat(records.isEmpty()).isTrue(); + assertThat(records.hasProgress()).isTrue(); + assertThat(records.records(tb)).isEmpty(); + assertThat(records.consumedUpToOffset(tb)).isEqualTo(expectedOffset); + + if (finished) { + assertThat(records.finishedBuckets()).containsExactly(tb); + } else { + assertThat(records.finishedBuckets()).isEmpty(); + } + } + + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(expectedOffset); + assertThat(logScannerStatus.hasReachedStoppingOffset(tb)).isEqualTo(finished); + assertThat(completedFetch.isConsumed()).isTrue(); + } + + @Test + void testBoundedFetchStopsAtStoppingOffset() throws Exception { + Configuration conf = new Configuration(); + conf.setInt(ConfigOptions.CLIENT_SCANNER_LOG_MAX_POLL_RECORDS, 2); + + MetadataUpdater metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + LogFetchCollector collector = + new LogFetchCollector(logScannerStatus, conf, metadataUpdater); + + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, 0L, 5L); + + FetchLogResultForBucket result = + FetchLogResultForBucket.records( + tb, genMemoryLogRecordsByObject(DATA1), 10L, -1L, -1L); + CompletedFetch completedFetch = makeCompletedFetch(tb, result, 0L); + logFetchBuffer.add(completedFetch); + + ScanRecords firstPoll = collector.collectFetch(logFetchBuffer); + assertThat(firstPoll.records(tb)).extracting(ScanRecord::logOffset).containsExactly(0L, 1L); + assertThat(firstPoll.consumedUpToOffset(tb)).isEqualTo(2L); + assertThat(firstPoll.finishedBuckets()).isEmpty(); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(2L); + assertThat(completedFetch.isConsumed()).isFalse(); + + ScanRecords secondPoll = collector.collectFetch(logFetchBuffer); + assertThat(secondPoll.records(tb)) + .extracting(ScanRecord::logOffset) + .containsExactly(2L, 3L); + assertThat(secondPoll.consumedUpToOffset(tb)).isEqualTo(4L); + assertThat(secondPoll.finishedBuckets()).isEmpty(); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(4L); + assertThat(completedFetch.isConsumed()).isFalse(); + + ScanRecords thirdPoll = collector.collectFetch(logFetchBuffer); + assertThat(thirdPoll.records(tb)).extracting(ScanRecord::logOffset).containsExactly(4L); + assertThat(thirdPoll.consumedUpToOffset(tb)).isEqualTo(5L); + assertThat(thirdPoll.finishedBuckets()).containsExactly(tb); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(5L); + assertThat(completedFetch.isConsumed()).isTrue(); + } + + @Test + void testStoppingOffsetTakesPrecedenceOverUpdatePairGrouping() throws Exception { + List changeTypes = + Arrays.asList( + ChangeType.INSERT, + ChangeType.UPDATE_BEFORE, + ChangeType.UPDATE_AFTER, + ChangeType.INSERT); + + List objects = DATA1.subList(0, 4); + + MemoryLogRecords records = + createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 0L, + System.currentTimeMillis(), + LogRecordBatch.CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + changeTypes, + objects, + LogFormat.ARROW, + DEFAULT_COMPRESSION); + + Configuration conf = new Configuration(); + conf.setInt(ConfigOptions.CLIENT_SCANNER_LOG_MAX_POLL_RECORDS, 1); + + MetadataUpdater metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + LogFetchCollector collector = + new LogFetchCollector(logScannerStatus, conf, metadataUpdater); + + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, 0L, 2L); + + FetchLogResultForBucket result = FetchLogResultForBucket.records(tb, records, 4L, -1L, -1L); + CompletedFetch completedFetch = makeCompletedFetch(tb, result, 0L); + logFetchBuffer.add(completedFetch); + + ScanRecords firstPoll = collector.collectFetch(logFetchBuffer); + assertThat(firstPoll.records(tb)).hasSize(1); + assertThat(firstPoll.records(tb).get(0).getChangeType()).isEqualTo(ChangeType.INSERT); + assertThat(firstPoll.finishedBuckets()).isEmpty(); + + ScanRecords secondPoll = collector.collectFetch(logFetchBuffer); + + assertThat(secondPoll.records(tb)).hasSize(1); + assertThat(secondPoll.records(tb).get(0).getChangeType()) + .isEqualTo(ChangeType.UPDATE_BEFORE); + assertThat(secondPoll.records(tb).get(0).logOffset()).isEqualTo(1L); + + assertThat(secondPoll.consumedUpToOffset(tb)).isEqualTo(2L); + assertThat(secondPoll.finishedBuckets()).containsExactly(tb); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(2L); + assertThat(completedFetch.isConsumed()).isTrue(); + } + + @Test + void testDiscardBufferedFetchForFinishedBucket() throws Exception { + Configuration conf = new Configuration(); + MetadataUpdater metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + LogFetchCollector collector = + new LogFetchCollector(logScannerStatus, conf, metadataUpdater); + + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, 0L, 5L); + + FetchLogResultForBucket firstResult = + FetchLogResultForBucket.records( + tb, genMemoryLogRecordsByObject(DATA1), 10L, -1L, -1L); + CompletedFetch firstFetch = makeCompletedFetch(tb, firstResult, 0L); + + FetchLogResultForBucket staleResult = + FetchLogResultForBucket.records( + tb, genMemoryLogRecordsByObject(DATA1), 20L, -1L, -1L); + CompletedFetch staleFetch = makeCompletedFetch(tb, staleResult, 10L); + + logFetchBuffer.add(firstFetch); + logFetchBuffer.add(staleFetch); + + ScanRecords firstPoll = collector.collectFetch(logFetchBuffer); + + assertThat(firstPoll.finishedBuckets()).containsExactly(tb); + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(5L); + assertThat(firstFetch.isConsumed()).isTrue(); + + ScanRecords secondPoll = collector.collectFetch(logFetchBuffer); + + assertThat(secondPoll.hasProgress()).isFalse(); + assertThat(staleFetch.isConsumed()).isTrue(); + } + + @Test + void testUnboundedArrowProgressOnlyPreservesLegacyResultShape() { + MetadataUpdater metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + + ArrowLogFetchCollector collector = + new ArrowLogFetchCollector(logScannerStatus, new Configuration(), metadataUpdater); + + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, 0L, LogScanner.NO_STOPPING_OFFSET); + + FetchLogResultForBucket filteredEmpty = FetchLogResultForBucket.empty(tb, 10L, 5L); + CompletedFetch completedFetch = makeCompletedFetch(tb, filteredEmpty, 0L); + logFetchBuffer.add(completedFetch); + + try (ArrowScanRecords records = collector.collectFetch(logFetchBuffer)) { + assertThat(records.count()).isZero(); + + // Preserve the legacy Arrow collector result shape for an unbounded + // progress-only fetch. + assertThat(records.isEmpty()).isFalse(); + assertThat(records.buckets()).containsExactly(tb); + assertThat(records.records(tb)).isEmpty(); + + assertThat(records.hasProgress()).isTrue(); + assertThat(records.consumedUpToOffset(tb)).isEqualTo(5L); + assertThat(records.finishedBuckets()).isEmpty(); + } + + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(5L); + assertThat(completedFetch.isConsumed()).isTrue(); + } + + @ParameterizedTest + @ValueSource(longs = {5L, 10L}) + void testBoundedArrowFetchRespectsStoppingOffset(long stoppingOffset) throws Exception { + MetadataUpdater metadataUpdater = + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO)); + + ArrowLogFetchCollector collector = + new ArrowLogFetchCollector(logScannerStatus, new Configuration(), metadataUpdater); + + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + logScannerStatus.assignScanBucket(tb, 0L, stoppingOffset); + + MemoryLogRecords records = + createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 0L, + 0L, + LogRecordBatch.CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + Collections.nCopies(DATA1.size(), ChangeType.APPEND_ONLY), + DATA1, + LogFormat.ARROW, + DEFAULT_COMPRESSION, + true); + + FetchLogResultForBucket result = + FetchLogResultForBucket.records(tb, records, DATA1.size(), -1L, -1L); + + CompletedFetch completedFetch = makeCompletedFetch(tb, result, 0L); + logFetchBuffer.add(completedFetch); + + try (ArrowScanRecords scanRecords = collector.collectFetch(logFetchBuffer)) { + int expectedRecordCount = (int) stoppingOffset; + + assertThat(scanRecords.count()).isEqualTo(expectedRecordCount); + assertThat(scanRecords.records(tb)).hasSize(1); + + ArrowBatchData batch = scanRecords.records(tb).get(0); + assertThat(batch.getBaseLogOffset()).isZero(); + assertThat(batch.getRecordCount()).isEqualTo(expectedRecordCount); + assertThat(batch.getVectorSchemaRoot().getRowCount()).isEqualTo(expectedRecordCount); + + assertThat(scanRecords.consumedUpToOffset(tb)).isEqualTo(stoppingOffset); + assertThat(scanRecords.finishedBuckets()).containsExactly(tb); + assertThat(scanRecords.hasProgress()).isTrue(); + } + + assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(stoppingOffset); + assertThat(completedFetch.isConsumed()).isTrue(); + } + + @Test + void testBoundedArrowTruncationReleasesAllBatchMemory() throws Exception { + TableBucket bucket = new TableBucket(DATA1_TABLE_ID, 0); + int batchSize = DATA1.size(); + long stoppingOffset = batchSize + batchSize / 2; + logScannerStatus.assignScanBucket(bucket, 0L, stoppingOffset); + ArrowLogFetchCollector collector = + new ArrowLogFetchCollector( + logScannerStatus, + new Configuration(), + new TestingMetadataUpdater( + Collections.singletonMap(DATA1_TABLE_PATH, DATA1_TABLE_INFO))); + + List batches = new ArrayList<>(); + int totalBytes = 0; + for (int i = 0; i < 3; i++) { + MemoryLogRecords records = + createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + (long) i * batchSize, + System.currentTimeMillis(), + LogRecordBatch.CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + Collections.nCopies(batchSize, ChangeType.APPEND_ONLY), + DATA1, + LogFormat.ARROW, + DEFAULT_COMPRESSION, + true); + batches.add(records); + totalBytes += records.sizeInBytes(); + } + ByteBuffer buffer = ByteBuffer.allocate(totalBytes); + for (MemoryLogRecords batch : batches) { + buffer.put(batch.getMemorySegment().wrap(0, batch.sizeInBytes())); + } + buffer.flip(); + CompletedFetch fetch = + makeCompletedFetch( + bucket, + FetchLogResultForBucket.records( + bucket, + MemoryLogRecords.pointToByteBuffer(buffer), + 3L * batchSize, + -1L, + -1L), + 0L); + logFetchBuffer.add(fetch); + + LongSupplier allocatedBytes; + try (ArrowScanRecords records = collector.collectFetch(logFetchBuffer)) { + assertThat(records.records(bucket)).hasSize(2); + assertThat(records.count()).isEqualTo((int) stoppingOffset); + assertThat(records.finishedBuckets()).containsExactly(bucket); + assertThat(fetch.isConsumed()).isTrue(); + ArrowBatchData first = records.records(bucket).get(0); + ArrowBatchData truncated = records.records(bucket).get(1); + assertThat(first.getRecordCount()).isEqualTo(batchSize); + assertThat(truncated.getBaseLogOffset()).isEqualTo(batchSize); + assertThat(truncated.getRecordCount()).isEqualTo(batchSize / 2); + // Both batches remain readable after draining the fetch and transferring ownership. + assertThat(first.getVectorSchemaRoot().getVector(0).getObject(0)) + .isEqualTo(DATA1.get(0)[0]); + assertThat(truncated.getVectorSchemaRoot().getVector(0).getObject(batchSize / 2 - 1)) + .isEqualTo(DATA1.get(batchSize / 2 - 1)[0]); + allocatedBytes = + first.getVectorSchemaRoot().getVector(0).getAllocator()::getAllocatedMemory; + assertThat(allocatedBytes.getAsLong()).isPositive(); + } + // Check the read-side allocator before closing readContext: closing the result must + // release retained slices as well as the discarded third batch and the original second + // batch. + assertThat(allocatedBytes.getAsLong()).isZero(); + } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java index 82f8a139816..97b0182ce41 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java @@ -49,6 +49,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.nio.file.Path; @@ -68,6 +70,7 @@ import static org.apache.fluss.client.metadata.TestingMetadataUpdater.NODE1; import static org.apache.fluss.client.metadata.TestingMetadataUpdater.NODE2; import static org.apache.fluss.client.metadata.TestingMetadataUpdater.NODE3; +import static org.apache.fluss.record.MemoryLogRecords.EMPTY; import static org.apache.fluss.record.TestData.DATA1_SCHEMA; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; @@ -165,6 +168,167 @@ void testUnregisterTableDiscardsBufferedFetches() { assertThat(logFetcher.getRegisteredTableCount()).isZero(); } + @ParameterizedTest + @ValueSource(longs = {100L, -1L}) + void testBoundedEmptyEarliestResponse(long resolvedOffset) { + IOUtils.closeQuietly(logFetcher); + + EarliestTabletServerGateway gateway = + new EarliestTabletServerGateway(resolvedOffset, false); + + metadataUpdater = initializeMetadataUpdater(gateway); + logFetcher = createLogFetcher(new Configuration()); + + logFetcher.getLogScannerStatus().assignScanBucket(tb1, LogScanner.EARLIEST_OFFSET, 50L); + + Map requestMap = + logFetcher.prepareFetchLogRequests(Collections.singletonList(tb1)); + + logFetcher.sendFetchRequest(1, requestMap.get(1)); + + assertThat(logFetcher.hasAvailableFetches()).isTrue(); + + if (resolvedOffset == FetchLogResultForBucket.NO_RESOLVED_EARLIEST_OFFSET) { + assertThatThrownBy(logFetcher::collectFetch) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("resolved_earliest_offset"); + + assertThat(logFetcher.hasAvailableFetches()).isFalse(); + assertThat(logFetcher.getLogScannerStatus().getBucketOffset(tb1)) + .isEqualTo(LogScanner.EARLIEST_OFFSET); + return; + } + + ScanRecords records = logFetcher.collectFetch(); + + assertThat(records.count()).isZero(); + assertThat(records.hasProgress()).isTrue(); + assertThat(records.consumedUpToOffset(tb1)).isEqualTo(50L); + assertThat(records.finishedBuckets()).containsExactly(tb1); + + assertThat(logFetcher.getLogScannerStatus().getBucketOffset(tb1)).isEqualTo(50L); + } + + @Test + void testDiscardResponseAfterStartingOffsetChanges() { + IOUtils.closeQuietly(logFetcher); + DelayedTabletServerGateway gateway = new DelayedTabletServerGateway(); + metadataUpdater = initializeMetadataUpdater(gateway); + logFetcher = createLogFetcher(new Configuration()); + logFetcher.getLogScannerStatus().assignScanBucket(tb1, LogScanner.EARLIEST_OFFSET, 50L); + logFetcher.sendFetchRequest( + 1, logFetcher.prepareFetchLogRequests(Collections.singletonList(tb1)).get(1)); + logFetcher.getLogScannerStatus().assignScanBucket(tb1, 10L, 50L); + + gateway.completeResponse(); + + assertThat(logFetcher.getCompletedFetchesSize()).isZero(); + assertThat(logFetcher.getLogScannerStatus().getBucketOffset(tb1)).isEqualTo(10L); + } + + @Test + void testBoundedEarliestBeyondStoppingOffsetSkipsRemoteDownload() throws Exception { + IOUtils.closeQuietly(logFetcher); + EarliestTabletServerGateway gateway = new EarliestTabletServerGateway(100L, true); + metadataUpdater = initializeMetadataUpdater(gateway); + + try (RemoteFileDownloader failingDownloader = + new RemoteFileDownloader(1) { + @Override + protected long downloadFile(Path targetFilePath, FsPath remoteFilePath) + throws IOException { + throw new IOException("Remote download must not happen"); + } + }; + LogFetcher fetcher = + createLogFetcher(new Configuration(), metadataUpdater, failingDownloader)) { + + fetcher.getLogScannerStatus().assignScanBucket(tb1, LogScanner.EARLIEST_OFFSET, 50L); + Map requestMap = + fetcher.prepareFetchLogRequests(Collections.singletonList(tb1)); + fetcher.sendFetchRequest(1, requestMap.get(1)); + + // The remote response becomes a metadata-only CompletedFetch. + assertThat(fetcher.hasAvailableFetches()).isTrue(); + + ScanRecords records = fetcher.collectFetch(); + + assertThat(records.isEmpty()).isTrue(); + assertThat(records.consumedUpToOffset(tb1)).isEqualTo(50L); + assertThat(records.finishedBuckets()).containsExactly(tb1); + } + } + + @Test + void testBoundedEarliestBeforeStoppingOffsetDownloadsRemoteRecords() throws Exception { + IOUtils.closeQuietly(logFetcher); + EarliestTabletServerGateway gateway = new EarliestTabletServerGateway(20L, true); + metadataUpdater = initializeMetadataUpdater(gateway); + + try (RemoteFileDownloader failingDownloader = + new RemoteFileDownloader(1) { + @Override + protected long downloadFile(Path targetFilePath, FsPath remoteFilePath) + throws IOException { + throw new IOException("Expected remote download"); + } + }; + LogFetcher fetcher = + createLogFetcher(new Configuration(), metadataUpdater, failingDownloader)) { + + fetcher.getLogScannerStatus().assignScanBucket(tb1, LogScanner.EARLIEST_OFFSET, 50L); + Map requestMap = + fetcher.prepareFetchLogRequests(Collections.singletonList(tb1)); + fetcher.sendFetchRequest(1, requestMap.get(1)); + + retry(Duration.ofSeconds(30), () -> assertThat(fetcher.hasAvailableFetches()).isTrue()); + + assertThatThrownBy(fetcher::collectFetch) + .isInstanceOf(FetchException.class) + .rootCause() + .hasMessageContaining("Expected remote download"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBoundedRemoteDoesNotScheduleSegmentsAtOrBeyondStoppingOffset(boolean earliest) + throws Exception { + IOUtils.closeQuietly(logFetcher); + + long physicalStartingOffset = 20L; + long stoppingOffset = 25L; + + TwoSegmentRemoteTabletServerGateway gateway = + new TwoSegmentRemoteTabletServerGateway(physicalStartingOffset); + + metadataUpdater = initializeMetadataUpdater(gateway); + + Configuration conf = new Configuration(); + + // Prevent the RemoteLogDownloader worker from consuming the queue. + // We want to inspect exactly how many segments LogFetcher scheduled. + conf.setInt(ConfigOptions.CLIENT_SCANNER_REMOTE_LOG_PREFETCH_NUM, 0); + + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogFetcher fetcher = createLogFetcher(conf, metadataUpdater, downloader)) { + long requestedStartingOffset = + earliest ? LogScanner.EARLIEST_OFFSET : physicalStartingOffset; + fetcher.getLogScannerStatus() + .assignScanBucket(tb1, requestedStartingOffset, stoppingOffset); + Map requestMap = + fetcher.prepareFetchLogRequests(Collections.singletonList(tb1)); + fetcher.sendFetchRequest(1, requestMap.get(1)); + + // Remote response contains: + // segment 0: [20, 100) + // segment 1: [100, 200) + // + // Since stop=25, only segment 0 is relevant. + assertThat(fetcher.getPendingRemoteLogFetchCount()).isEqualTo(1); + } + } + @Test void testDiscardStaleResponseAfterTableReregistered() { IOUtils.closeQuietly(logFetcher); @@ -367,6 +531,146 @@ private void completeResponse() { } } + private static class EarliestTabletServerGateway extends TestTabletServerGateway { + private final long resolvedEarliestOffset; + private final boolean remote; + + private EarliestTabletServerGateway(long resolvedEarliestOffset, boolean remote) { + super(false, Collections.emptySet()); + this.resolvedEarliestOffset = resolvedEarliestOffset; + this.remote = remote; + } + + @Override + public CompletableFuture fetchLog(FetchLogRequest request) { + Map fetchLogData = getFetchLogData(request); + Map results = new HashMap<>(); + + fetchLogData.forEach( + (tableBucket, fetchReqInfo) -> { + FetchLogResultForBucket result; + if (remote) { + RemoteLogSegment segment = + RemoteLogSegment.Builder.builder() + .tableBucket(tableBucket) + .physicalTablePath( + PhysicalTablePath.of(DATA1_TABLE_PATH)) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(resolvedEarliestOffset) + .remoteLogEndOffset(resolvedEarliestOffset + 100L) + .maxTimestamp(1000L) + .segmentSizeInBytes(1024) + .build(); + + RemoteLogFetchInfo remoteLogFetchInfo = + new RemoteLogFetchInfo( + "/tmp/test-tablet-dir", + null, + Collections.singletonList(segment), + 0); + + result = + FetchLogResultForBucket.remote( + tableBucket, + remoteLogFetchInfo, + 100L, + resolvedEarliestOffset); + + } else if (resolvedEarliestOffset + == FetchLogResultForBucket.NO_RESOLVED_EARLIEST_OFFSET) { + + // Simulates an old server: successful empty response without + // resolved_earliest_offset. + result = + FetchLogResultForBucket.records( + tableBucket, EMPTY, 100L, -1L, -1L); + + } else { + result = + FetchLogResultForBucket.records( + tableBucket, + EMPTY, + 100L, + -1L, + -1L, + resolvedEarliestOffset); + } + + results.put(tableBucket, result); + }); + + return CompletableFuture.completedFuture(makeFetchLogResponse(results)); + } + } + + private static class TwoSegmentRemoteTabletServerGateway extends TestTabletServerGateway { + private final long resolvedEarliestOffset; + + private TwoSegmentRemoteTabletServerGateway(long resolvedEarliestOffset) { + super(false, Collections.emptySet()); + this.resolvedEarliestOffset = resolvedEarliestOffset; + } + + @Override + public CompletableFuture fetchLog(FetchLogRequest request) { + Map fetchLogData = getFetchLogData(request); + Map results = new HashMap<>(); + fetchLogData.forEach( + (tableBucket, fetchReqInfo) -> { + RemoteLogSegment firstSegment = + RemoteLogSegment.Builder.builder() + .tableBucket(tableBucket) + .physicalTablePath(PhysicalTablePath.of(DATA1_TABLE_PATH)) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(resolvedEarliestOffset) + .remoteLogEndOffset(100L) + .maxTimestamp(1000L) + .segmentSizeInBytes(1024) + .build(); + + RemoteLogSegment secondSegment = + RemoteLogSegment.Builder.builder() + .tableBucket(tableBucket) + .physicalTablePath(PhysicalTablePath.of(DATA1_TABLE_PATH)) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(100L) + .remoteLogEndOffset(200L) + .maxTimestamp(2000L) + .segmentSizeInBytes(1024) + .build(); + + RemoteLogFetchInfo remoteLogFetchInfo = + new RemoteLogFetchInfo( + "/tmp/test-tablet-dir", + null, + Arrays.asList(firstSegment, secondSegment), + 0); + + FetchLogResultForBucket result; + + if (fetchReqInfo.getFetchOffset() == LogScanner.EARLIEST_OFFSET) { + + result = + FetchLogResultForBucket.remote( + tableBucket, + remoteLogFetchInfo, + 200L, + resolvedEarliestOffset); + } else { + // Explicit offset request must not carry + // resolved_earliest_offset. + result = + FetchLogResultForBucket.remote( + tableBucket, remoteLogFetchInfo, 200L); + } + + results.put(tableBucket, result); + }); + + return CompletableFuture.completedFuture(makeFetchLogResponse(results)); + } + } + private static class BlockingMetadataUpdater extends TestingMetadataUpdater { private final CountDownLatch metadataUpdateStarted = new CountDownLatch(1); private final CountDownLatch continueMetadataUpdate = new CountDownLatch(1); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java index 7f6700acf3f..f9d3268d855 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerITCase.java @@ -564,6 +564,103 @@ void testPollArrowBatchesWithSchemaEvolution() throws Exception { } } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBoundedLogScan(boolean isPartitioned) throws Exception { + TablePath tablePath = TablePath.of("test_db_1", "test_bounded_log_scan_" + isPartitioned); + TableDescriptor.Builder builder = + TableDescriptor.builder().schema(DATA1_SCHEMA).distributedBy(1); + if (isPartitioned) { + builder.partitionedBy("b"); + } + long tableId = createTable(tablePath, builder.build(), false); + Long partitionId = null; + if (isPartitioned) { + admin.createPartition(tablePath, newPartitionSpec("b", "p1"), false).get(); + partitionId = + FLUSS_CLUSTER_EXTENSION.waitUntilPartitionAllReady(tablePath, 1).get("p1"); + assertThat(partitionId).isNotNull(); + FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady(tableId, partitionId); + } else { + waitAllReplicasReady(tableId, 1); + } + + TableBucket tableBucket = new TableBucket(tableId, partitionId, 0); + try (Table table = conn.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + for (int i = 0; i < 10; i++) { + writer.append(row(i, "p1")).get(); + } + writer.flush(); + + try (LogScanner logScanner = table.newScan().createLogScanner()) { + if (isPartitioned) { + logScanner.subscribeBounded(partitionId, 0, 2L, 5L); + } else { + logScanner.subscribeBounded(0, 2L, 5L); + } + + List offsets = new ArrayList<>(); + ScanRecords lastScanRecords = ScanRecords.EMPTY; + boolean finished = false; + long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos(); + + while (!finished) { + assertThat(System.nanoTime()) + .as("Timed out waiting for bounded log scan to finish") + .isLessThan(deadline); + + ScanRecords scanRecords = logScanner.poll(Duration.ofSeconds(1)); + lastScanRecords = scanRecords; + + for (ScanRecord scanRecord : scanRecords) { + offsets.add(scanRecord.logOffset()); + } + + Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); + if (consumedUpToOffset != null) { + assertThat(consumedUpToOffset).isLessThanOrEqualTo(5L); + } + + finished = scanRecords.finishedBuckets().contains(tableBucket); + } + + assertThat(offsets).containsExactly(2L, 3L, 4L); + assertThat(lastScanRecords.consumedUpToOffset(tableBucket)).isEqualTo(5L); + assertThat(lastScanRecords.finishedBuckets()).containsExactly(tableBucket); + } + } + } + + @Test + void testEmptyBoundedArrowRangeReturnsCompletion() throws Exception { + TablePath tablePath = + TablePath.of("test_db_1", "test_empty_bounded_arrow_range_completion"); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(DATA1_SCHEMA) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .build(); + long tableId = createTable(tablePath, descriptor, false); + waitAllReplicasReady(tableId, 1); + + TableBucket tableBucket = new TableBucket(tableId, 0); + try (Table table = conn.getTable(tablePath); + LogScannerImpl scanner = (LogScannerImpl) table.newScan().createLogScanner()) { + scanner.subscribeBounded(0, 5L, 5L); + + try (ArrowScanRecords scanRecords = scanner.pollRecordBatch(Duration.ofSeconds(1))) { + assertThat(scanRecords.hasProgress()).isTrue(); + assertThat(scanRecords.count()).isZero(); + assertThat(scanRecords.buckets()).containsExactly(tableBucket); + assertThat(scanRecords.records(tableBucket)).isEmpty(); + assertThat(scanRecords.consumedUpToOffset(tableBucket)).isNull(); + assertThat(scanRecords.finishedBuckets()).containsExactly(tableBucket); + } + } + } + private void pollAndVerifyArrowBatches( LogScannerImpl scanner, int expectedRecords, int minExpectedOffset) { int count = 0; diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java new file mode 100644 index 00000000000..96d1eab2d02 --- /dev/null +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.client.table.scanner.log; + +import org.apache.fluss.metadata.TableBucket; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.apache.fluss.client.table.scanner.log.LogScanner.NO_STOPPING_OFFSET; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link LogScannerStatus}. */ +public class LogScannerStatusTest { + + private static final TableBucket TABLE_BUCKET = new TableBucket(1L, 0); + + @Test + void testAssignBoundedBucket() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 20L); + + assertThat(scannerStatus.getBucketOffset(TABLE_BUCKET)).isEqualTo(10L); + assertThat(scannerStatus.getBucketStoppingOffset(TABLE_BUCKET)).isEqualTo(20L); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); + } + + @Test + void testResubscribeUpdatesStoppingOffset() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 20L); + scannerStatus.assignScanBucket(TABLE_BUCKET, 20L, 30L); + + assertThat(scannerStatus.getBucketOffset(TABLE_BUCKET)).isEqualTo(20L); + assertThat(scannerStatus.getBucketStoppingOffset(TABLE_BUCKET)).isEqualTo(30L); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); + } + + @Test + void testResubscribeAsUnboundedClearsStoppingOffset() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 20L); + scannerStatus.assignScanBucket(TABLE_BUCKET, 20L, NO_STOPPING_OFFSET); + + assertThat(scannerStatus.getBucketOffset(TABLE_BUCKET)).isEqualTo(20L); + assertThat(scannerStatus.getBucketStoppingOffset(TABLE_BUCKET)) + .isEqualTo(NO_STOPPING_OFFSET); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); + } + + @Test + void testBoundedRecordsLag() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 90L, 100L); + scannerStatus.updateHighWatermark(TABLE_BUCKET, 1000L); + + assertThat(scannerStatus.recordsLag()).isEqualTo(10L); + } + + @Test + void testUnboundedRecordsLag() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 90L, NO_STOPPING_OFFSET); + scannerStatus.updateHighWatermark(TABLE_BUCKET, 1000L); + + assertThat(scannerStatus.recordsLag()).isEqualTo(910L); + } + + @Test + void testEmptyBoundedRangeIsImmediatelyFinished() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 10L); + + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isTrue(); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isTrue(); + + assertThat(scannerStatus.drainFinishedBuckets()).containsExactly(TABLE_BUCKET); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isFalse(); + } + + @Test + void testReachingStoppingOffsetReportsCompletionOnce() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 20L); + + scannerStatus.updateOffset(TABLE_BUCKET, 19L); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isFalse(); + + scannerStatus.updateOffset(TABLE_BUCKET, 20L); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isTrue(); + assertThat(scannerStatus.drainFinishedBuckets()).containsExactly(TABLE_BUCKET); + + scannerStatus.updateOffset(TABLE_BUCKET, 20L); + assertThat(scannerStatus.drainFinishedBuckets()).isEmpty(); + } + + @Test + void testFinishedBucketIsNotFetchable() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 20L); + + assertThat(scannerStatus.fetchableBuckets(ignored -> true)).containsExactly(TABLE_BUCKET); + + scannerStatus.updateOffset(TABLE_BUCKET, 20L); + + assertThat(scannerStatus.fetchableBuckets(ignored -> true)).isEmpty(); + } + + @Test + void testResubscribeClearsPendingCompletion() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 10L); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isTrue(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 20L); + + assertThat(scannerStatus.hasPendingFinishedBuckets()).isFalse(); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); + assertThat(scannerStatus.getBucketStoppingOffset(TABLE_BUCKET)).isEqualTo(20L); + } + + @Test + void testUnassignClearsPendingCompletion() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 10L); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isTrue(); + + scannerStatus.unassignScanBuckets(Collections.singletonList(TABLE_BUCKET)); + + assertThat(scannerStatus.hasPendingFinishedBuckets()).isFalse(); + assertThat(scannerStatus.getBucketOffset(TABLE_BUCKET)).isNull(); + } + + @Test + void testFinishedBucketDoesNotBlockOtherBuckets() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + TableBucket first = new TableBucket(1L, 0); + TableBucket second = new TableBucket(1L, 1); + + scannerStatus.assignScanBucket(first, 0L, 5L); + scannerStatus.assignScanBucket(second, 0L, 10L); + + scannerStatus.updateOffset(first, 5L); + + assertThat(scannerStatus.fetchableBuckets(ignored -> true)).containsExactly(second); + assertThat(scannerStatus.drainFinishedBuckets()).containsExactly(first); + } + + @Test + void testResolveBoundedEarliestBeyondStoppingOffset() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, LogScanner.EARLIEST_OFFSET, 50L); + + Long resolved = + scannerStatus.resolveBoundedStartingOffset( + TABLE_BUCKET, LogScanner.EARLIEST_OFFSET, 100L); + + assertThat(resolved).isEqualTo(50L); + assertThat(scannerStatus.getBucketOffset(TABLE_BUCKET)).isEqualTo(50L); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isTrue(); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isTrue(); + assertThat(scannerStatus.drainFinishedBuckets()).containsExactly(TABLE_BUCKET); + } + + @Test + void testResolveBoundedEarliestBeforeStoppingOffset() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, LogScanner.EARLIEST_OFFSET, 150L); + + Long resolved = + scannerStatus.resolveBoundedStartingOffset( + TABLE_BUCKET, LogScanner.EARLIEST_OFFSET, 100L); + + assertThat(resolved).isEqualTo(100L); + assertThat(scannerStatus.getBucketOffset(TABLE_BUCKET)).isEqualTo(100L); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isFalse(); + } + + @Test + void testResolveBoundedEarliestIgnoresStaleRequest() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + scannerStatus.assignScanBucket(TABLE_BUCKET, LogScanner.EARLIEST_OFFSET, 50L); + // Resubscribe before the old EARLIEST response arrives. + scannerStatus.assignScanBucket(TABLE_BUCKET, 10L, 50L); + Long resolved = + scannerStatus.resolveBoundedStartingOffset( + TABLE_BUCKET, LogScanner.EARLIEST_OFFSET, 20L); + + assertThat(resolved).isNull(); + assertThat(scannerStatus.getBucketOffset(TABLE_BUCKET)).isEqualTo(10L); + assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); + assertThat(scannerStatus.hasPendingFinishedBuckets()).isFalse(); + } +} diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java new file mode 100644 index 00000000000..e7563030ca4 --- /dev/null +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.client.table.scanner.log; + +import org.apache.fluss.client.metadata.TestingMetadataUpdater; +import org.apache.fluss.client.table.scanner.RemoteFileDownloader; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.Collections; + +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; +import static org.apache.fluss.record.TestData.PARTITION_TABLE_INFO; +import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests compatibility with implementations of the original {@link LogScanner} interface. */ +class LogScannerTest { + + @Test + void testLegacySubscriptionWithIntegerArguments() { + LegacyLogScanner scanner = new LegacyLogScanner(); + LogScanner api = scanner; + api.subscribe(0, 2, 5L); + assertThat(scanner.partitionId).isZero(); + assertThat(scanner.bucket).isEqualTo(2); + assertThat(scanner.offset).isEqualTo(5L); + } + + @Test + void testLegacyScannerRejectsBoundedSubscriptions() { + LogScanner scanner = new LegacyLogScanner(); + assertThatThrownBy(() -> scanner.subscribeBounded(0, 2, 5)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> scanner.subscribeBounded(1L, 0, 2, 5)) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testInvalidOffsetsDoNotReplaceSubscription() throws Exception { + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogScanner scanner = createScanner(downloader)) { + scanner.subscribeBounded(0, 0, 0); + assertThatThrownBy(() -> scanner.subscribeBounded(0, 0, -1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("stopping offset"); + assertThatThrownBy(() -> scanner.subscribeBounded(0, -1, 5)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("starting offset"); + assertThatThrownBy(() -> scanner.subscribeBounded(0, 0, LogScanner.NO_STOPPING_OFFSET)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("stopping offset"); + assertThat(scanner.poll(Duration.ZERO).finishedBuckets()).hasSize(1); + } + } + + @ParameterizedTest + @CsvSource({"0, 0", "5, 5", "5, 3", "-2, 0"}) + void testEmptyBoundedSubscription(long startingOffset, long stoppingOffset) throws Exception { + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogScannerImpl scanner = createScanner(downloader)) { + scanner.subscribeBounded(0, startingOffset, stoppingOffset); + ScanRecords records = scanner.poll(Duration.ZERO); + assertThat(records.isEmpty()).isTrue(); + assertThat(records.finishedBuckets()) + .containsExactly(new TableBucket(DATA1_TABLE_ID, 0)); + assertThat(scanner.poll(Duration.ZERO).hasProgress()).isFalse(); + } + } + + @Test + void testUnboundedSubscriptionClearsPendingCompletion() throws Exception { + TableBucket bucket = new TableBucket(DATA1_TABLE_ID, 0); + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogScannerImpl scanner = createScanner(downloader)) { + scanner.subscribeBounded(0, 0, 0); + scanner.subscribe(0, LogScanner.EARLIEST_OFFSET); + assertThat(scanner.logScannerStatus.getBucketStoppingOffset(bucket)) + .isEqualTo(LogScanner.NO_STOPPING_OFFSET); + assertThat(scanner.logScannerStatus.hasReachedStoppingOffset(bucket)).isFalse(); + assertThat(scanner.logScannerStatus.hasPendingFinishedBuckets()).isFalse(); + + // Preserve the original API's deferred validation of out-of-range offsets. + scanner.subscribe(0, Long.MIN_VALUE); + assertThat(scanner.logScannerStatus.getBucketOffset(bucket)).isEqualTo(Long.MIN_VALUE); + } + } + + @Test + void testPartitionedSubscriptionRejectsInvalidOffsets() throws Exception { + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogScannerImpl scanner = createScanner(downloader, PARTITION_TABLE_INFO)) { + assertThatThrownBy(() -> scanner.subscribeBounded(1L, 0, 0, -1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("stopping offset"); + assertThatThrownBy( + () -> scanner.subscribeBounded(1L, 0, 0, LogScanner.NO_STOPPING_OFFSET)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("stopping offset"); + assertThatThrownBy(() -> scanner.subscribeBounded(1L, 0, -1, 5)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("starting offset"); + assertThat(scanner.logScannerStatus.prepareToPoll()).isFalse(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testSubscriptionRejectsWrongTableType(boolean bounded) throws Exception { + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogScanner scanner = createScanner(downloader); + LogScanner partitionedScanner = createScanner(downloader, PARTITION_TABLE_INFO)) { + String nonPartitionedMethod = + bounded + ? "subscribeBounded(int bucket, long startingOffset, long stoppingOffset)" + : "subscribe(int bucket, long offset)"; + String partitionedMethod = + bounded + ? "subscribeBounded(long partitionId, int bucket, long startingOffset, long stoppingOffset)" + : "subscribe(long partitionId, int bucket, long offset)"; + assertThatThrownBy( + () -> { + if (bounded) { + scanner.subscribeBounded(1L, 0, 0, 5); + } else { + scanner.subscribe(1L, 0, 0); + } + }) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "The table is not a partitioned table, please use \"%s\" to subscribe a non-partitioned bucket instead.", + nonPartitionedMethod); + assertThatThrownBy( + () -> { + if (bounded) { + partitionedScanner.subscribeBounded(0, 0, 5); + } else { + partitionedScanner.subscribe(0, 0); + } + }) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "The table is a partitioned table, please use \"%s\" to subscribe a partitioned bucket instead.", + partitionedMethod); + } + } + + private static LogScannerImpl createScanner(RemoteFileDownloader downloader) { + return createScanner(downloader, DATA1_TABLE_INFO); + } + + private static LogScannerImpl createScanner( + RemoteFileDownloader downloader, TableInfo tableInfo) { + return new LogScannerImpl( + new Configuration(), + tableInfo, + new TestingMetadataUpdater( + Collections.singletonMap(tableInfo.getTablePath(), tableInfo)), + TestingClientMetricGroup.newInstance(), + downloader, + null, + TEST_SCHEMA_GETTER, + null); + } + + private static class LegacyLogScanner implements LogScanner { + private long partitionId; + private int bucket; + private long offset; + + @Override + public ScanRecords poll(Duration timeout) { + return ScanRecords.EMPTY; + } + + @Override + public void subscribe(int bucket, long offset) { + this.bucket = bucket; + this.offset = offset; + } + + @Override + public void subscribe(long partitionId, int bucket, long offset) { + this.partitionId = partitionId; + subscribe(bucket, offset); + } + + @Override + public void unsubscribe(long partitionId, int bucket) {} + + @Override + public void unsubscribe(int bucket) {} + + @Override + public void wakeup() {} + + @Override + public void close() {} + } +} diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java index c01ea7e2fc5..caecd5fb4b6 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java @@ -17,7 +17,9 @@ package org.apache.fluss.client.table.scanner.log; +import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.config.Configuration; import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; @@ -53,7 +55,9 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.fluss.record.TestData.DATA2; import static org.apache.fluss.record.TestData.DATA2_PHYSICAL_TABLE_PATH; @@ -269,6 +273,93 @@ void testProjection(String format) throws Exception { } } + @Test + void testBoundedEarliestResolutionPreservedThroughRemotePendingFetch() throws Exception { + TableBucket tableBucket = new TableBucket(DATA2_TABLE_ID, 0); + + long resolvedEarliestOffset = 20L; + long stoppingOffset = 25L; + + logScannerStatus.assignScanBucket(tableBucket, LogScanner.EARLIEST_OFFSET, stoppingOffset); + + RemoteLogSegment remoteLogSegment = + RemoteLogSegment.Builder.builder() + .tableBucket(tableBucket) + .physicalTablePath(DATA2_PHYSICAL_TABLE_PATH) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(resolvedEarliestOffset) + .remoteLogEndOffset(resolvedEarliestOffset + DATA2.size()) + .segmentSizeInBytes(Integer.MAX_VALUE) + .build(); + + File logFile = + genRemoteLogSegmentFile( + DATA2_ROW_TYPE, + tempDir, + remoteLogSegment, + DATA2, + resolvedEarliestOffset, + LogFormat.ARROW); + + AtomicInteger recycleCount = new AtomicInteger(); + + RemoteLogDownloadFuture downloadFuture = + new RemoteLogDownloadFuture( + CompletableFuture.completedFuture(logFile), recycleCount::incrementAndGet); + + RemotePendingFetch pendingFetch = + new RemotePendingFetch( + remoteLogSegment, + downloadFuture, + DATA2_TABLE_PATH, + 0, + LogScanner.EARLIEST_OFFSET, + resolvedEarliestOffset, + resolvedEarliestOffset + DATA2.size(), + remoteReadContext, + logScannerStatus, + true); + + assertThat(pendingFetch.isCompleted()).isTrue(); + + CompletedFetch completedFetch = pendingFetch.toCompletedFetch(); + + // Request identity and physical resolution must remain separate. + assertThat(completedFetch.requestedFetchOffset()).isEqualTo(LogScanner.EARLIEST_OFFSET); + assertThat(completedFetch.resolvedEarliestOffset()).isEqualTo(resolvedEarliestOffset); + + // Resolution is not applied until collector initialization. + assertThat(completedFetch.nextFetchOffset()).isEqualTo(LogScanner.EARLIEST_OFFSET); + + try (LogFetchBuffer fetchBuffer = new LogFetchBuffer()) { + fetchBuffer.add(completedFetch); + + LogFetchCollector collector = + new LogFetchCollector( + logScannerStatus, + new Configuration(), + new TestingMetadataUpdater( + Collections.singletonMap(DATA2_TABLE_PATH, DATA2_TABLE_INFO))); + + ScanRecords records = collector.collectFetch(fetchBuffer); + + assertThat(records.records(tableBucket)) + .extracting(ScanRecord::logOffset) + .containsExactly(20L, 21L, 22L, 23L, 24L); + + assertThat(records.count()).isEqualTo(5); + assertThat(records.consumedUpToOffset(tableBucket)).isEqualTo(stoppingOffset); + assertThat(records.finishedBuckets()).containsExactly(tableBucket); + + assertThat(logScannerStatus.getBucketOffset(tableBucket)).isEqualTo(stoppingOffset); + assertThat(logScannerStatus.hasReachedStoppingOffset(tableBucket)).isTrue(); + + assertThat(completedFetch.isConsumed()).isTrue(); + assertThat(recycleCount.get()).isEqualTo(1); + } + assertThat(recycleCount.get()).isEqualTo(1); + } + private FileLogRecords createFileLogRecords( TableBucket tableBucket, PhysicalTablePath physicalTablePath, @@ -312,6 +403,7 @@ private RemoteCompletedFetch makeCompletedFetch( logScannerStatus, true, fetchOffset, + -1L, recycle); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java index 47f1355f8fc..13798b54c9a 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/ScanRecordsTest.java @@ -37,6 +37,8 @@ /** Test for {@link ScanRecords}. */ public class ScanRecordsTest { + private static final TableBucket TABLE_BUCKET = new TableBucket(1L, 0); + @Test void iterator() { Map> records = new LinkedHashMap<>(); @@ -132,4 +134,20 @@ void progressOnlyBucketsSurfacedWithoutRecordEntries() { assertThat(scanRecords.count()).isEqualTo(1); assertThat(scanRecords.hasProgress()).isTrue(); } + + @Test + void testFinishedOnlyBucketHasProgress() { + ScanRecords records = + new ScanRecords( + Collections.emptyMap(), + Collections.emptyMap(), + Collections.singleton(TABLE_BUCKET)); + + assertThat(records.isEmpty()).isTrue(); + assertThat(records.hasProgress()).isTrue(); + assertThat(records.buckets()).containsExactly(TABLE_BUCKET); + assertThat(records.records(TABLE_BUCKET)).isEmpty(); + assertThat(records.consumedUpToOffset(TABLE_BUCKET)).isNull(); + assertThat(records.finishedBuckets()).containsExactly(TABLE_BUCKET); + } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java index 11f6d7e73b8..729e5ea6410 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java @@ -33,11 +33,14 @@ /** Result of {@link FetchLogRequest} for each table bucket. */ @Internal public class FetchLogResultForBucket extends ResultForBucket { + public static final long NO_RESOLVED_EARLIEST_OFFSET = -1L; + private final @Nullable RemoteLogFetchInfo remoteLogFetchInfo; private final @Nullable LogRecords records; private final long highWatermark; private final long filteredEndOffset; private final long minRetainOffset; + private final long resolvedEarliestOffset; private FetchLogResultForBucket( TableBucket tableBucket, @@ -46,6 +49,7 @@ private FetchLogResultForBucket( long highWatermark, long filteredEndOffset, long minRetainOffset, + long resolvedEarliestOffset, ApiError error) { super(tableBucket, error); this.remoteLogFetchInfo = remoteLogFetchInfo; @@ -53,6 +57,7 @@ private FetchLogResultForBucket( this.highWatermark = highWatermark; this.filteredEndOffset = filteredEndOffset; this.minRetainOffset = minRetainOffset; + this.resolvedEarliestOffset = resolvedEarliestOffset; } /** Creates a successful local fetch result. */ @@ -62,7 +67,34 @@ public static FetchLogResultForBucket records( long highWatermark, long filteredEndOffset, long minRetainOffset) { + return records( + tableBucket, + records, + highWatermark, + filteredEndOffset, + minRetainOffset, + NO_RESOLVED_EARLIEST_OFFSET); + } + + /** + * Creates a successful local fetch result with an EARLIEST offset resolution. + * + *

{@code resolvedEarliestOffset} must be non-negative when supplied, or {@link + * #NO_RESOLVED_EARLIEST_OFFSET} when this fetch did not resolve a symbolic starting offset. + */ + public static FetchLogResultForBucket records( + TableBucket tableBucket, + LogRecords records, + long highWatermark, + long filteredEndOffset, + long minRetainOffset, + long resolvedEarliestOffset) { checkArgument(minRetainOffset >= -1L, "Min retain offset must be at least -1."); + + checkArgument( + resolvedEarliestOffset >= NO_RESOLVED_EARLIEST_OFFSET, + "Resolved earliest offset must be at least %s.", + NO_RESOLVED_EARLIEST_OFFSET); return new FetchLogResultForBucket( tableBucket, null, @@ -70,12 +102,26 @@ public static FetchLogResultForBucket records( highWatermark, filteredEndOffset, minRetainOffset, + resolvedEarliestOffset, ApiError.NONE); } /** Creates a successful remote fetch result. */ public static FetchLogResultForBucket remote( TableBucket tableBucket, RemoteLogFetchInfo remoteLogFetchInfo, long highWatermark) { + return remote(tableBucket, remoteLogFetchInfo, highWatermark, NO_RESOLVED_EARLIEST_OFFSET); + } + + /** Creates a successful remote fetch result with an EARLIEST offset resolution. */ + public static FetchLogResultForBucket remote( + TableBucket tableBucket, + RemoteLogFetchInfo remoteLogFetchInfo, + long highWatermark, + long resolvedEarliestOffset) { + checkArgument( + resolvedEarliestOffset >= NO_RESOLVED_EARLIEST_OFFSET, + "Resolved earliest offset must be at least %s.", + NO_RESOLVED_EARLIEST_OFFSET); return new FetchLogResultForBucket( tableBucket, checkNotNull(remoteLogFetchInfo, "remote log fetch info can not be null"), @@ -83,6 +129,7 @@ public static FetchLogResultForBucket remote( highWatermark, -1L, -1L, + resolvedEarliestOffset, ApiError.NONE); } @@ -90,12 +137,20 @@ public static FetchLogResultForBucket remote( public static FetchLogResultForBucket empty( TableBucket tableBucket, long highWatermark, long filteredEndOffset) { return new FetchLogResultForBucket( - tableBucket, null, null, highWatermark, filteredEndOffset, -1L, ApiError.NONE); + tableBucket, + null, + null, + highWatermark, + filteredEndOffset, + -1L, + NO_RESOLVED_EARLIEST_OFFSET, + ApiError.NONE); } /** Creates a failed fetch result. */ public static FetchLogResultForBucket error(TableBucket tableBucket, ApiError error) { - return new FetchLogResultForBucket(tableBucket, null, null, -1L, -1L, -1L, error); + return new FetchLogResultForBucket( + tableBucket, null, null, -1L, -1L, -1L, NO_RESOLVED_EARLIEST_OFFSET, error); } /** @@ -125,6 +180,21 @@ public LogRecords recordsOrEmpty() { return remoteLogFetchInfo; } + /** + * Returns whether this response contains the physical offset resolved from an EARLIEST request. + */ + public boolean hasResolvedEarliestOffset() { + return resolvedEarliestOffset >= 0; + } + + /** + * Returns the physical offset resolved from an EARLIEST request, or {@link + * #NO_RESOLVED_EARLIEST_OFFSET} when absent. + */ + public long getResolvedEarliestOffset() { + return resolvedEarliestOffset; + } + public long getHighWatermark() { return highWatermark; } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 8733056f1f8..b421f8b7fd0 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -56,6 +56,8 @@ import java.util.UUID; import java.util.stream.Collectors; +import static org.apache.fluss.rpc.entity.FetchLogResultForBucket.NO_RESOLVED_EARLIEST_OFFSET; + /** * Utils for making rpc request/response from inner object or convert inner class to rpc * request/response for client and server. @@ -197,6 +199,10 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( fetchLogResultForBucket = FetchLogResultForBucket.error(tb, ApiError.fromErrorMessage(respForBucket)); } else { + long resolvedEarliestOffset = + respForBucket.hasResolvedEarliestOffset() + ? respForBucket.getResolvedEarliestOffset() + : NO_RESOLVED_EARLIEST_OFFSET; if (respForBucket.hasRemoteLogFetchInfo()) { PbRemoteLogFetchInfo pbRlfInfo = respForBucket.getRemoteLogFetchInfo(); String partitionName = @@ -231,7 +237,10 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( pbRlfInfo.getFirstStartPos()); fetchLogResultForBucket = FetchLogResultForBucket.remote( - tb, rlFetchInfo, respForBucket.getHighWatermark()); + tb, + rlFetchInfo, + respForBucket.getHighWatermark(), + resolvedEarliestOffset); } else { ByteBuffer recordsBuffer = toByteBuffer(respForBucket.getRecordsSlice()); LogRecords records = @@ -249,7 +258,8 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( hasFilteredEndOffset ? respForBucket.getFilteredEndOffset() : -1L, respForBucket.hasMinRetainOffset() ? respForBucket.getMinRetainOffset() - : -1L); + : -1L, + resolvedEarliestOffset); } } diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index 5a5294c614e..82269875d4d 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -1082,6 +1082,11 @@ message PbFetchLogRespForBucket { // The safe local log retention boundary confirmed by a committed KV snapshot. This is only // returned for KV follower fetches and is distinct from the physical log_start_offset. optional int64 min_retain_offset = 10; + // The log start offset to which an EARLIEST_OFFSET request was resolved. + // This field is present only when the request used EARLIEST_OFFSET. + // Its presence also allows a client to distinguish a server that reports the + // resolved offset from an older server that does not. + optional int64 resolved_earliest_offset = 11; } message PbPutKvReqForBucket { diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/entity/FetchLogResultForBucketTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/entity/FetchLogResultForBucketTest.java new file mode 100644 index 00000000000..19e6cfbf579 --- /dev/null +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/entity/FetchLogResultForBucketTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.rpc.entity; + +import org.apache.fluss.metadata.TableBucket; + +import org.junit.jupiter.api.Test; + +import static org.apache.fluss.record.MemoryLogRecords.EMPTY; +import static org.apache.fluss.rpc.entity.FetchLogResultForBucket.NO_RESOLVED_EARLIEST_OFFSET; +import static org.assertj.core.api.Assertions.assertThat; + +/** Test FetchLogResultForBucket. */ +public class FetchLogResultForBucketTest { + @Test + void testResolvedEarliestOffsetPresence() { + TableBucket bucket = new TableBucket(1L, 0); + + FetchLogResultForBucket absent = + FetchLogResultForBucket.records(bucket, EMPTY, 10L, -1L, -1L); + + assertThat(absent.hasResolvedEarliestOffset()).isFalse(); + assertThat(absent.getResolvedEarliestOffset()).isEqualTo(NO_RESOLVED_EARLIEST_OFFSET); + + FetchLogResultForBucket zero = + FetchLogResultForBucket.records(bucket, EMPTY, 10L, -1L, -1L, 0L); + + assertThat(zero.hasResolvedEarliestOffset()).isTrue(); + assertThat(zero.getResolvedEarliestOffset()).isZero(); + + FetchLogResultForBucket retained = + FetchLogResultForBucket.records(bucket, EMPTY, 100L, -1L, -1L, 100L); + + assertThat(retained.getResolvedEarliestOffset()).isEqualTo(100L); + } +} diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto new file mode 100644 index 00000000000..74f4664240d --- /dev/null +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -0,0 +1,1518 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; + +package fluss; +option java_package = "org.apache.fluss.rpc.messages"; +option optimize_for = LITE_RUNTIME; + +/** +* NOTICE: To ensure a smooth upgrade to Proto3 syntax while maintaining backward compatibility in +* the future, please adhere to the following limitations: +* 1. DO NOT USE "default" keyword, proto3 removes the support of "default" +* 2. DO NOT USE "enum" type + */ + +/** +* NOTICE: The Rust client checks in the code generated from this file. After changing it, run +* fluss-rust/crates/fluss/regen.sh and commit the result; the proto-up-to-date job of the Rust +* CI fails otherwise. + */ + +// the message indicates a failed request of a server failure. +message ErrorResponse { + required int32 error_code = 1; + optional string error_message = 2; +} + +message ApiVersionsRequest { + required string client_software_name = 1; + required string client_software_version = 2; +} + +message ApiVersionsResponse { + repeated PbApiVersion api_versions = 1; + optional int32 server_type = 2; +} + +// get schema request and response +message GetTableSchemaRequest { + required PbTablePath table_path = 1; + optional int32 schema_id = 2; +} + +message GetTableSchemaResponse { + required int32 schema_id = 1; + required bytes schema_json = 2; +} + +// create database request and response +message CreateDatabaseRequest { + required string database_name = 1; + required bool ignore_if_exists = 2; + optional bytes database_json = 3; +} + +message CreateDatabaseResponse { +} + +// alter database request and response +message AlterDatabaseRequest { + required string database_name = 1; + required bool ignore_if_not_exists = 2; + repeated PbAlterConfig config_changes = 3; + optional string comment = 4; +} + +message AlterDatabaseResponse { +} + +// get table request and response +message GetDatabaseInfoRequest { + required string database_name = 1; +} + +message GetDatabaseInfoResponse { + required bytes database_json = 3; + required int64 created_time = 4; + required int64 modified_time = 5; +} + +// drop database request and response +message DropDatabaseRequest { + required string database_name = 1; + required bool ignore_if_not_exists = 2; + required bool cascade = 3; +} +message DropDatabaseResponse { +} + +// database exists request and response +message DatabaseExistsRequest { + required string database_name = 1; +} +message DatabaseExistsResponse { + required bool exists = 1; +} + +// list databases request and response +message ListDatabasesRequest { + optional bool include_summary = 1; +} + +message ListDatabasesResponse { + repeated string database_name = 1; + repeated PbDatabaseSummary database_summary = 2; +} + +// create table request and response +message CreateTableRequest { + required PbTablePath table_path = 1; + required bytes table_json = 2; + required bool ignore_if_exists = 3; +} + +message CreateTableResponse { +} + +// alter table request and response +message AlterTableRequest { + required PbTablePath table_path = 1; + required bool ignore_if_not_exists = 2; + repeated PbAlterConfig config_changes = 3; + repeated PbAddColumn add_columns = 4; + repeated PbDropColumn drop_columns = 5; + repeated PbRenameColumn rename_columns = 6; + repeated PbModifyColumn modify_columns = 7; + optional PbModifyBucketCount modify_bucket_count = 8; +} + +message PbModifyBucketCount { + required int32 new_bucket_count = 1; +} + +message AlterTableResponse { +} + + + +// get table request and response +message GetTableInfoRequest { + required PbTablePath table_path = 1; +} + +message GetTableInfoResponse { + required int64 table_id = 1; + required int32 schema_id = 2; + required bytes table_json = 3; + required int64 created_time = 4; + required int64 modified_time = 5; + optional string remote_data_dir = 6; + optional int64 bucket_count_epoch = 7; +} + +// describe buckets request and response +message DescribeBucketsRequest { + required PbTablePath table_path = 1; + optional PbPartitionSpec partition_spec = 2; +} + +message DescribeBucketsResponse { + required PbTablePath table_path = 1; + required int64 table_id = 2; + repeated PbBucketInfo bucket_info = 3; +} + +// list tables request and response +message ListTablesRequest { + required string database_name = 1; +} + +message ListTablesResponse { + repeated string table_name = 1; +} + +// drop table request and response +message DropTableRequest { + required PbTablePath table_path = 1; + required bool ignore_if_not_exists = 2; +} + +message DropTableResponse { +} + +// table exists request and response +message TableExistsRequest { + required PbTablePath table_path = 1; +} + +message TableExistsResponse { + required bool exists = 1; +} + +// metadata request and response, request send from client to each server. +message MetadataRequest { + repeated PbTablePath table_path = 1; + repeated PbPhysicalTablePath partitions_path = 2; + + // note: currently, we assume the partition ids must belong to the table_paths in the + // metadata request + // todo: we won't need the assumption after we introduce metadata cache in server + repeated int64 partitions_id = 3 [packed = true]; +} + +message MetadataResponse { + optional PbServerNode coordinator_server = 1; + repeated PbServerNode tablet_servers = 2; + repeated PbTableMetadata table_metadata = 3; + repeated PbPartitionMetadata partition_metadata = 4; +} + +// update metadata request and response, only send between server. +message UpdateMetadataRequest { + optional PbServerNode coordinator_server = 1; + repeated PbServerNode tablet_servers = 2; + repeated PbTableMetadata table_metadata = 3; + repeated PbPartitionMetadata partition_metadata = 4; + optional int32 coordinator_epoch = 5; +} + +message UpdateMetadataResponse { +} + +// produce log request and response +message ProduceLogRequest { + required int32 acks = 1; + required int64 table_id = 2; + required int32 timeout_ms = 3; + repeated PbProduceLogReqForBucket buckets_req = 4; +} + +message ProduceLogResponse { + repeated PbProduceLogRespForBucket buckets_resp = 1; +} + +// fetch log request and response +message FetchLogRequest { + required int32 follower_server_id = 1; // value -1 indicate the request from client. + required int32 max_bytes = 2; + repeated PbFetchLogReqForTable tables_req = 3; + optional int32 max_wait_ms = 4; + optional int32 min_bytes = 5; + optional int32 read_preference = 6; +} + +message FetchLogResponse { + repeated PbFetchLogRespForTable tables_resp = 1; +} + +// Aggregation mode constants for write operations (using int32 instead of enum for proto3 compatibility) +// AGG_MODE_AGGREGATE = 0: Data is aggregated through server-side merge engine (default) +// AGG_MODE_OVERWRITE = 1: Bypass merge engine, directly replace values (for undo recovery) +// AGG_MODE_LOCAL_AGGREGATE = 2: Client-side local aggregation (reserved for future implementation) + +// put kv request and response +message PutKvRequest { + required int32 acks = 1; + required int64 table_id = 2; + required int32 timeout_ms = 3; + // the indexes for the columns to write, + // if empty, means write all columns + repeated int32 target_columns = 4 [packed = true]; + repeated PbPutKvReqForBucket buckets_req = 5; + // Aggregation mode for this request (see AGG_MODE_* constants above) + // 0 = AGGREGATE (default), 1 = OVERWRITE, 2 = LOCAL_AGGREGATE (not yet supported) + optional int32 agg_mode = 6; +} + +message PutKvResponse { + repeated PbPutKvRespForBucket buckets_resp = 1; +} + +// lookup request and response +message LookupRequest { + required int64 table_id = 1; + repeated PbLookupReqForBucket buckets_req = 2; + optional bool insert_if_not_exists = 3; + optional int32 acks = 4; + optional int32 timeout_ms = 5; +} + +message LookupResponse { + repeated PbLookupRespForBucket buckets_resp = 1; +} + +// Prefix Lookup request and response +message PrefixLookupRequest { + required int64 table_id = 1; + repeated PbPrefixLookupReqForBucket buckets_req = 2; +} + +message PrefixLookupResponse { + repeated PbPrefixLookupRespForBucket buckets_resp = 1; +} + + +// limit scan request and response +message LimitScanRequest { + required int64 table_id = 2; + optional int64 partition_id = 3; + required int32 bucket_id = 4; + required int32 limit = 5; + optional int32 routing_bucket_count = 6; +} + +message LimitScanResponse{ + optional int32 error_code = 1; + optional string error_message = 2; + // flag to indicate the table type + optional bool is_log_table = 3; + // LogRecordBatch if is_log_table is true, otherwise KvRecordBatch + optional bytes records = 4; +} + + +// Full KV scan request and response. +// A new scan is initiated with bucket_scan_req; subsequent batches use scanner_id. +message PbScanReqForBucket { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + // If set, stops returning rows after this many records. + optional int64 limit = 4; + optional int32 routing_bucket_count = 5; +} + +message ScanKvRequest { + // Mutually exclusive: either scanner_id (continuation) or bucket_scan_req (new scan). + optional bytes scanner_id = 1; + optional PbScanReqForBucket bucket_scan_req = 2; + // Monotonically increasing sequence number for in-order delivery validation. + optional int32 call_seq_id = 3; + // Maximum number of bytes of record data to return in this batch. + optional int32 batch_size_bytes = 4; + // If true, the server closes the scanner session immediately. + optional bool close_scanner = 5; +} + +message ScanKvResponse { + optional int32 error_code = 1; + optional string error_message = 2; + // Opaque server-assigned session identifier; echoed on every response. + optional bytes scanner_id = 3; + // False when the scan is complete and the scanner session has been closed. + optional bool has_more_results = 4; + // Serialised DefaultValueRecordBatch; absent when the batch is empty. + optional bytes records = 5; + // Log high-watermark at the time the RocksDB snapshot was opened. + // Only set on the first response (new scan initiation). + optional int64 log_offset = 6; +} + + +// Get table statistics request and response. +// Sent to TabletServer to get per-bucket statistics. +message GetTableStatsRequest { + required int64 table_id = 1; + repeated PbTableStatsReqForBucket buckets_req = 2; + // Column indexes for column-level stats (e.g., ndv, sum). + // If empty, column stats are not collected. + // Reserved for future use. + repeated int32 target_columns = 3 [packed = true]; +} + +message GetTableStatsResponse { + repeated PbTableStatsRespForBucket buckets_resp = 1; +} + +// notify bucket leader and isr request +message NotifyLeaderAndIsrRequest { + required int32 coordinator_epoch = 1; + repeated PbNotifyLeaderAndIsrReqForBucket notify_buckets_leader_req = 2; +} + +// response for notify bucket leader and isr request +message NotifyLeaderAndIsrResponse { + repeated PbNotifyLeaderAndIsrRespForBucket notify_buckets_leader_resp = 1; +} + +// stop bucket replica request +message StopReplicaRequest { + required int32 coordinator_epoch = 1; + repeated PbStopReplicaReqForBucket stop_replicas_req = 2; +} + +// response for stop bucket replica request +message StopReplicaResponse { + repeated PbStopReplicaRespForBucket stop_replicas_resp = 1; +} + +// adjust isr request and response +message AdjustIsrRequest { + required int32 serverId = 1; + repeated PbAdjustIsrReqForTable tables_req = 2; +} + +message AdjustIsrResponse { + repeated PbAdjustIsrRespForTable tables_resp = 1; +} + +// list offsets request and response +message ListOffsetsRequest { + required int32 follower_server_id = 1; // value -1 indicate the request from client. + required int32 offset_type = 2; // value can be 0,1,2 (see ListOffsetsParam for more details) + required int64 table_id = 3; + optional int64 partition_id = 4; + repeated int32 bucket_id = 5 [packed = true]; // it is recommended to use packed for repeated numerics to get more efficient encoding + optional int64 startTimestamp = 6; + optional int32 routing_bucket_count = 7; +} +message ListOffsetsResponse { + repeated PbListOffsetsRespForBucket buckets_resp = 1; +} + +// commit kv snapshot request and response +message CommitKvSnapshotRequest { + required bytes completed_snapshot = 1; + required int32 coordinator_epoch = 2; + required int32 bucket_leader_epoch = 3; +} + +message CommitKvSnapshotResponse { +} + +// notify the log offset about kv snapshot +message NotifyKvSnapshotOffsetRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int32 coordinator_epoch = 4; + required int64 min_retain_offset = 5; +} + +message NotifyKvSnapshotOffsetResponse { +} + +message GetLatestKvSnapshotsRequest { + required PbTablePath table_path = 1; + optional string partition_name = 2; +} + +message GetLatestKvSnapshotsResponse { + required int64 table_id = 1; + // null if it is a non-partitioned table, otherwise, it must be not null + optional int64 partition_id = 2; + repeated PbKvSnapshot latest_snapshots = 3; +} + +message GetKvSnapshotMetadataRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int64 snapshot_id = 4; +} + +message GetKvSnapshotMetadataResponse { + required int64 log_offset = 1; + repeated PbRemotePathAndLocalFile snapshot_files = 2; +} + +message AcquireKvSnapshotLeaseRequest { + required string lease_id = 1; + required int64 lease_duration_ms = 2; + repeated PbKvSnapshotLeaseForTable snapshots_to_lease = 3; +} + +message AcquireKvSnapshotLeaseResponse { + repeated PbKvSnapshotLeaseForTable unavailable_snapshots = 1; +} + +message ReleaseKvSnapshotLeaseRequest { + required string lease_id = 1; + repeated PbTableBucket buckets_to_release = 2; +} + +message ReleaseKvSnapshotLeaseResponse {} + +message DropKvSnapshotLeaseRequest { + required string lease_id = 1; +} + +message DropKvSnapshotLeaseResponse {} + +message GetLakeSnapshotRequest { + required PbTablePath table_path = 1; + optional int64 snapshot_id = 2; // If not set, returns the latest snapshot + optional bool readable = 3; // If true, returns the latest readable snapshot +} + +message GetLakeSnapshotResponse { + required int64 table_id = 1; + required int64 snapshotId = 2; + repeated PbLakeSnapshotForBucket bucket_snapshots = 3; +} + +message GetFileSystemSecurityTokenRequest { +} + +message GetFileSystemSecurityTokenResponse { + required string schema = 1; + required bytes token = 2; + optional int64 expiration_time = 3; + repeated PbKeyValue addition_info = 4; +} + +// init writer request and response +message InitWriterRequest { + repeated PbTablePath table_path = 1; +} + +message InitWriterResponse { + required int64 writer_id = 1; +} + +message ListPartitionInfosRequest { + required PbTablePath table_path = 1; + optional PbPartitionSpec partial_partition_spec = 2; + optional bool include_system_partitions = 3; +} + +message ListPartitionInfosResponse { + repeated PbPartitionInfo partitions_info = 1; + optional bool system_partitions_included = 2; +} + +// list remote log manifest entries (one per bucket of a table or partition) +message ListRemoteLogManifestsRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; // required if table is partitioned +} + +message ListRemoteLogManifestsResponse { + repeated PbRemoteLogManifestEntry manifests = 1; +} + +message PbRemoteLogManifestEntry { + required PbTableBucket table_bucket = 1; + required string remote_log_manifest_path = 2; + required int64 remote_log_end_offset = 3; +} + +// list active KV snapshot dirs (retained_N + still-in-use) for a unit +message ListKvSnapshotsRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; +} + +message ListKvSnapshotsResponse { + required int64 table_id = 1; + // null if it is a non-partitioned table, otherwise, it must be not null + optional int64 partition_id = 2; + // Active snapshots = retained_N ∪ still-in-use; multiple entries per bucket allowed. + repeated PbKvSnapshot active_snapshots = 3; +} + +// create partition request and response +message CreatePartitionRequest { + required PbTablePath table_path = 1; + required PbPartitionSpec partition_spec = 2; + required bool ignore_if_not_exists = 3; +} + +message CreatePartitionResponse {} + +// drop partition request and response +message DropPartitionRequest { + required PbTablePath table_path = 1; + required PbPartitionSpec partition_spec = 2; + required bool ignore_if_not_exists = 3; +} + +message DropPartitionResponse {} + +// commit remote log manifest request and response +message CommitRemoteLogManifestRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required string remote_log_manifest_path = 4; + required int64 remote_log_start_offset = 5; + required int64 remote_log_end_offset = 6; + required int32 coordinator_epoch = 7; + required int32 bucket_leader_epoch = 8; + optional int64 highest_copied_end_offset = 9; +} + +message CommitRemoteLogManifestResponse { + required bool commitSuccess = 1; +} + +// notify remote log offsets request and response +message NotifyRemoteLogOffsetsRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int32 coordinator_epoch = 4; + required int64 remote_start_offset = 5; + required int64 remote_end_offset = 6; + optional int64 highest_copied_end_offset = 7; +} + +message NotifyRemoteLogOffsetsResponse { +} + +message PrepareLakeTableSnapshotRequest { + repeated PbTableOffsets bucket_offsets = 1; + // If true, ignore previous table offsets and do not merge with existing lake table offsets. + // If false (default), the system will merge the provided bucket offsets with the previous + // lake table offsets to ensure continuity. This is useful when you want to replace all offsets + // instead of merging with existing ones. + optional bool ignore_previous_table_offsets = 2; +} + +message PrepareLakeTableSnapshotResponse { + repeated PbPrepareLakeTableRespForTable prepare_lake_table_resp = 1; +} + +message CommitLakeTableSnapshotRequest { + // Deprecated: PbLakeTableSnapshotInfo is no longer used for committing lake table snapshots. + // Currently, it is only used to allow the coordinator to notify tablet servers about the current + // synchronized log end offsets, which are then reported to metrics. In the future, we plan to + // have tiering directly report to metrics, and this field will be fully removed. + // Still reserve it for protocol compatibility. + repeated PbLakeTableSnapshotInfo tables_req = 1; + // The metadata for lake table snapshots to be committed. Each entry contains the table ID, + // snapshot ID, and the file paths where the snapshot data (containing bucket log end offset + // information) is stored. The tiered_snapshot_file_path points to the file storing tiered log + // end offsets, while readable_snapshot_file_path (if present) points to the file storing readable + // log end offsets. + repeated PbLakeTableSnapshotMetadata lake_table_snapshot_metadata = 2; +} + +message CommitLakeTableSnapshotResponse { + repeated PbCommitLakeTableSnapshotRespForTable table_resp = 1; +} + +message PbCommitLakeTableSnapshotRespForTable { + optional int32 error_code = 1; + optional string error_message = 2; + required int64 table_id = 3; +} + +message NotifyLakeTableOffsetRequest { + required int32 coordinator_epoch = 1; + repeated PbNotifyLakeTableOffsetReqForBucket notify_buckets_req = 2; +} + +message PbNotifyLakeTableOffsetReqForBucket { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int64 snapshot_id = 4; + optional int64 log_start_offset = 5; + optional int64 log_end_offset = 6; + optional int64 max_timestamp = 7; +} + +message NotifyLakeTableOffsetResponse { +} + +message AuthenticateRequest { + required string protocol = 1; + required bytes token = 2; +} + +message AuthenticateResponse { + optional bytes challenge = 1; +} + +// acl related requests and responses +message ListAclsRequest{ + required PbAclFilter acl_filter = 1; +} + +message ListAclsResponse{ + repeated PbAclInfo acl = 1; +} + +message CreateAclsRequest{ + repeated PbAclInfo acl = 1; +} + +message CreateAclsResponse{ + repeated PbCreateAclRespInfo aclRes = 1; +} + +message DropAclsRequest{ + repeated PbAclFilter acl_filter = 1; +} + +message DropAclsResponse{ + repeated PbDropAclsFilterResult filter_results = 1; +} + +message LakeTieringHeartbeatRequest { + repeated PbHeartbeatReqForTable tiering_tables = 1; + repeated PbHeartbeatReqForTable finished_tables = 2; + repeated PbHeartbeatReqForTable failed_tables = 3; + // whether to request a table + optional bool request_table = 4; + // Table IDs that were force finished due to reaching the maximum tiering duration. + // When a table's tiering operation exceeds the max duration (data lake freshness), + // it will be force finished to prevent it from blocking other tables' tiering operations. + // These table IDs must also appear in finished_tables. The coordinator uses this field to + // distinguish between normally finished tables and force finished tables. + repeated int64 force_finished_tables = 5; +} + +message LakeTieringHeartbeatResponse { + // coordinator epoch + required int32 coordinator_epoch = 1; + // the returned table to tier, empty when no table is needed by the lake tiering service + optional PbLakeTieringTableInfo tiering_table = 2; + repeated PbHeartbeatRespForTable tiering_table_resp = 3; + repeated PbHeartbeatRespForTable finished_table_resp = 4; + repeated PbHeartbeatRespForTable failed_table_resp = 5; +} + +message ControlledShutdownRequest { + required int32 tablet_server_id = 1; + required int32 tablet_server_epoch = 2; +} + +message ControlledShutdownResponse { + repeated PbTableBucket remaining_leader_buckets = 1; +} + +message DescribeClusterConfigsRequest{ +} + + +message DescribeClusterConfigsResponse{ + repeated PbDescribeConfig configs = 1; +} + +message AlterClusterConfigsRequest{ + repeated PbAlterConfig alter_configs = 1; +} + +message AlterClusterConfigsResponse{ +} + +message AddServerTagRequest { + repeated int32 server_ids = 1 [packed = true]; + required int32 server_tag = 2; +} + +message AddServerTagResponse { +} + +message AddServerTagByRackRequest { + repeated string racks = 1; + required int32 server_tag = 2; +} + +message AddServerTagByRackResponse { +} + +message RemoveServerTagRequest { + repeated int32 server_ids = 1 [packed = true]; + required int32 server_tag = 2; +} + +message RemoveServerTagResponse { +} + +message RemoveServerTagByRackRequest { + repeated string racks = 1; + required int32 server_tag = 2; +} + +message RemoveServerTagByRackResponse { +} + +message RebalanceRequest { + repeated int32 goals = 1 [packed = true]; +} + +message RebalanceResponse { + required string rebalance_id = 1; +} + +message ListRebalanceProgressRequest { + optional string rebalance_id = 1; +} + +message ListRebalanceProgressResponse { + optional string rebalance_id = 1; + optional int32 rebalance_status = 2; + repeated PbRebalanceProgressForTable table_progress = 3; +} + +message CancelRebalanceRequest { + optional string rebalance_id = 1; +} + +message CancelRebalanceResponse { +} + +// ------------------------------------------------------------------------------------------ +// Producer Offset Management +// ------------------------------------------------------------------------------------------ + +// Register producer offsets request and response +message RegisterProducerOffsetsRequest { + required string producer_id = 1; + repeated PbProducerTableOffsets table_offsets = 2; + optional int64 ttl_ms = 3; +} + +message RegisterProducerOffsetsResponse { + // Result of registration: 0 = CREATED (new snapshot), 1 = ALREADY_EXISTS (snapshot existed) + optional int32 result = 1; +} + +// Get producer offsets request and response +message GetProducerOffsetsRequest { + required string producer_id = 1; +} + +message GetProducerOffsetsResponse { + optional string producer_id = 1; + optional int64 expiration_time = 2; + repeated PbProducerTableOffsets table_offsets = 3; +} + +// Delete producer offsets request and response +message DeleteProducerOffsetsRequest { + required string producer_id = 1; +} + +message DeleteProducerOffsetsResponse { +} + +message GetClusterHealthRequest { } + +message GetClusterHealthResponse { + required int32 num_replicas = 1; + required int32 in_sync_replicas = 2; + required int32 num_leader_replicas = 3; + required int32 active_leader_replicas = 4; + required int32 status = 5; // PbClusterHealthStatus: GREEN=0, YELLOW=1, RED=2, UNKNOWN=3 +} + + +// --------------- Inner classes ---------------- +message PbApiVersion { + required int32 api_key = 1; + required int32 min_version = 2; + required int32 max_version = 3; +} + +message PbTablePath { + required string database_name = 1; + required string table_name = 2; +} + +message PbPhysicalTablePath { + required string database_name = 1; + required string table_name = 2; + optional string partition_name = 3; +} + +// For MetadataResponse, host and port are still used for all versions. +// For UpdateMetadataRequest, +// * versions <= 0.6: host and port are used. +// * versions >= 0.7: listeners is used to replace host and port. +// For MetadataResponse and UpdateMetadataRequest: Fluss versions >= 0.7: we add rack for each tabletServer +message PbServerNode { + required int32 node_id = 1; + required string host = 2; + required int32 port = 3; + optional string listeners = 4; + optional string rack = 5; +} + +message PbTableMetadata { + required PbTablePath table_path = 1; + required int64 table_id = 2; + required int32 schema_id = 3; + // TODO Starting from version 0.9, fluss-client no longer get table info from metadata response. However, for backward + // compatibility, we will still retain the field in the request. Once metadataV2 is introduced, table info will be + // officially omitted from the new metadata response. + required bytes table_json = 4; + repeated PbBucketMetadata bucket_metadata = 5; + required int64 created_time = 6; + required int64 modified_time = 7; + optional string remote_data_dir = 8; + // A table-level, monotonically increasing version for bucket.num changes. + optional int64 bucket_count_epoch = 9; + + // TODO add a new filed 'deleted_table' to indicate this table is deleted in UpdateMetadataRequest. + // trace by: https://github.com/apache/fluss/issues/981 +} + +message PbPartitionMetadata { + required int64 table_id = 1; + // the partition name and id for the partition + required string partition_name = 2; + required int64 partition_id = 3; + repeated PbBucketMetadata bucket_metadata = 4; + // the actual bucket count for this partition, used for per-partition bucket rescale + optional int32 bucket_count = 5; +} + +message PbBucketMetadata { + required int32 bucket_id = 1; + // optional as some time the leader may not elected yet + optional int32 leader_id = 2; + repeated int32 replica_id = 3 [packed = true]; + optional int32 leader_epoch = 4; + // Generation of the complete leader/ISR state. + // Absence indicates legacy metadata; -1 indicates no leader/ISR state exists. + optional int32 bucket_epoch = 5; + repeated int32 isr = 6 [packed = true]; +} + +message PbBucketInfo { + optional int64 partition_id = 1; + optional string partition_name = 2; + required int32 bucket_id = 3; + // optional as the leader may not be elected yet + optional int32 leader_id = 4; + repeated int32 replica_id = 5 [packed = true]; + optional int32 leader_epoch = 6; + // Generation of the complete leader/ISR state. + // Absence indicates legacy metadata; -1 indicates no leader/ISR state exists. + optional int32 bucket_epoch = 7; + repeated int32 isr = 8 [packed = true]; +} + +message PbProduceLogReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required bytes records = 3; + // The original partition name for a historical write; unset for a normal write. + optional string original_partition_name = 4; + optional int32 routing_bucket_count = 5; +} + +message PbProduceLogRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + optional int64 base_offset = 5; + // The original partition name echoed from a historical write request; unset for a normal write. + optional string original_partition_name = 6; +} + +message PbFetchLogReqForTable { + required int64 table_id = 1; + required bool projection_pushdown_enabled = 2; + repeated int32 projected_fields = 3 [packed = true]; + repeated PbFetchLogReqForBucket buckets_req = 4; + // filter_predicate and filter_schema_id MUST be set together. + // filter_predicate is the serialized predicate; filter_schema_id identifies + // the schema against which the predicate was compiled. + optional PbPredicate filter_predicate = 5; + optional int32 filter_schema_id = 6; +} + +message PbFetchLogReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + // TODO leader epoch + required int64 fetch_offset = 3; + required int32 max_fetch_bytes = 4; + optional int32 routing_bucket_count = 5; +} + +message PbFetchLogRespForTable { + required int64 table_id = 1; + repeated PbFetchLogRespForBucket buckets_resp = 2; +} +message PbFetchLogRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + optional int64 high_watermark = 5; + optional int64 log_start_offset = 6; // TODO now we don't introduce log start offset, but remain it in protobuf + optional PbRemoteLogFetchInfo remote_log_fetch_info = 7; + optional bytes records = 8; + // The offset the server scanned up to while filtering out batches, which may accompany a + // non-empty records field when only trailing batches were filtered. The client should start + // its next fetch from the later of this offset and the end of the records it received. + optional int64 filtered_end_offset = 9; + // The safe local log retention boundary confirmed by a committed KV snapshot. This is only + // returned for KV follower fetches and is distinct from the physical log_start_offset. + optional int64 min_retain_offset = 10; + // The actual offset used for this fetch after resolving an EARLIEST request. + // This field is present only when the request used EARLIEST_OFFSET. + // Its presence also allows a new client to distinguish a server that supports + // EARLIEST resolution from an older server that does not expose the resolved offset. + optional int64 resolved_earliest_offset = 11; +} + +message PbPutKvReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required bytes records = 3; + // The original partition name for historical PK writes. It is unset for normal writes. + optional string original_partition_name = 4; + // the bucket count the sender used to form bucket_id; the server compares it against the + // actual count to detect stale routing. Not authoritative metadata. + optional int32 routing_bucket_count = 5; +} + +message PbPutKvRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + // the log end offset (LEO) of the changelog after this write + // this is the offset of the next record to be written, used for exactly-once semantics + optional int64 log_end_offset = 5; + // backpressure signal from RocksDB write stall detection: + // 0 = normal (no pressure), (0,1) = DELAYED zone (normalized L0 ratio) + // introduced in v1.0 + optional float pressure = 6; + // The original partition name for historical PK writes. It is unset for normal writes. + optional string original_partition_name = 7; +} + +message PbLookupReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + repeated bytes keys = 3; + // The original partition name for historical lookup. It is unset for normal lookup. + optional string original_partition_name = 4; + optional int32 routing_bucket_count = 5; +} + +message PbLookupRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + repeated PbValue values = 5; + // The original partition name for historical lookup. It is unset for normal lookup. + optional string original_partition_name = 6; +} + +message PbValue { + // optional, if empty, means no value + optional bytes values = 1; +} + +message PbValueList { + repeated bytes values = 1; +} + +message PbPrefixLookupReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + repeated bytes keys = 3; + optional int32 routing_bucket_count = 4; +} + +message PbPrefixLookupRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + repeated PbValueList value_lists = 5; +} + +message PbTableBucket { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; +} + +message PbAdjustIsrReqForTable { + required int64 table_id = 1; + repeated PbAdjustIsrReqForBucket buckets_req = 2; +} + + +message PbAdjustIsrReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required int32 leader_epoch = 3; + repeated int32 new_isr = 4 [packed = true]; + required int32 coordinator_epoch = 5; + required int32 bucket_epoch = 6; + repeated int32 standby_replicas = 7 [packed = true]; +} + +message PbAdjustIsrRespForTable { + required int64 table_id = 1; + repeated PbAdjustIsrRespForBucket buckets_resp = 2; +} + +message PbAdjustIsrRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + optional int32 leader_id = 5; + optional int32 leader_epoch = 6; + repeated int32 isr = 7 [packed = true]; + optional int32 bucket_epoch = 8; + optional int32 coordinator_epoch = 9; + repeated int32 standby_replicas = 10 [packed = true]; +} + +message PbListOffsetsRespForBucket { + required int32 bucket_id = 1; + optional int32 error_code = 2; + optional string error_message = 3; + optional int64 offset = 4; +} + +message PbNotifyLeaderAndIsrReqForBucket { + // we need table path for tablet server to create data dir for the bucket, + // although tablet server don't need it after it has created data dir. + // For simplicity, we always pass table path. + required PbPhysicalTablePath physical_table_path = 1; + required PbTableBucket table_bucket = 2; + required int32 leader = 3; + required int32 leader_epoch = 4; + repeated int32 replicas = 5 [packed = true]; + repeated int32 isr = 6 [packed = true]; + required int32 bucket_epoch = 7; + repeated int32 standby_replicas = 8 [packed = true]; + optional int32 bucket_count = 9; + optional int64 bucket_count_epoch = 10; +} + +message PbNotifyLeaderAndIsrRespForBucket { + required PbTableBucket table_bucket = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbStopReplicaReqForBucket { + required PbTableBucket table_bucket = 1; + required int32 leader_epoch = 2; + // delete means remove local replica data (i.e., data stored on the current node). + required bool delete = 3; + // deleteRemote means remove remote replica data (e.g., data in object storage) and was introduced in v0.9. + // For backward compatibility, if a request does not include the deleteRemote flag, the system treats delete as + // deleteRemote (i.e., it falls back to remote deletion). This ensures older CoordinatorServer continues to function + // correctly with newer TabletServers. + optional bool deleteRemote = 4; +} + +message PbStopReplicaRespForBucket { + required PbTableBucket table_bucket = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbKvSnapshot { + required int32 bucket_id = 1; + // null if there is no snapshot for this bucket + optional int64 snapshot_id = 2; + // null if there is no snapshot for this bucket, then read from EARLIEST + optional int64 log_offset = 3; +} + +message PbLakeSnapshotForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int64 log_offset = 3; + optional string partition_name = 4; +} + +message PbRemotePathAndLocalFile { + required string remote_path = 1; + required string local_file_name = 2; +} + +message PbKeyValue { + required string key = 1; + required string value = 2; +} + +message PbRemoteLogFetchInfo { + required string remote_log_tablet_dir = 1; + optional string partition_name = 2; + repeated PbRemoteLogSegment remote_log_segments = 3; + optional int32 first_start_pos = 4; +} + +message PbRemoteLogSegment { + required string remote_log_segment_id = 1; + required int64 remote_log_start_offset = 2; + required int64 remote_log_end_offset = 3; + required int32 segment_size_in_bytes = 4; + optional int64 max_timestamp = 5; +} + +message PbPartitionInfo { + required int64 partition_id = 1; + required PbPartitionSpec partition_spec = 2; + optional string remote_data_dir = 3; + // the actual bucket count for this partition, used for per-partition bucket rescale + optional int32 bucket_count = 4; +} + +message PbPartitionSpec { + repeated PbKeyValue partition_key_values = 1; +} + +message PbCreateAclRespInfo { + required PbAclInfo acl = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbAclInfo { + required string resource_name = 1; + required int32 resource_type = 2; + required string principal_name = 3; + required string principal_type = 4; + required string host = 5; + required int32 operation_type = 6; + required int32 permission_type = 7; +} + +message PbAclFilter { + optional string resource_name = 1; + required int32 resource_type = 2; + optional string principal_name = 3; + optional string principal_type = 4; + optional string host = 5; + required int32 operation_type = 6; + required int32 permission_type = 7; +} + +message PbDropAclsFilterResult { + repeated PbDropAclsMatchingAcl matching_acls = 1; + optional int32 error_code = 2; + optional string error_message = 3; + +} + +message PbDropAclsMatchingAcl { + required PbAclInfo acl = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbLakeTieringTableInfo { + required int64 table_id = 1; + required PbTablePath table_path = 2; + required int64 tiering_epoch = 3; +} + +message PbHeartbeatReqForTable { + required int64 table_id = 1; + // the coordinator epoch when the table is assigned to be tiering service + required int32 coordinator_epoch = 2; + // the tiering epoch when the table is assigned to be tiering service + required int64 tiering_epoch = 3; + // stats for the last tiering round; only set for finished_tables in LakeTieringHeartbeatRequest + optional PbLakeTieringStats lake_tiering_stats = 4; +} + +// Stats reported by the tiering service when a table finishes a tiering round. +message PbLakeTieringStats { + // cumulative total file size (bytes) of the lake table after tiering + optional int64 file_size = 1; + // cumulative total record count of the lake table after tiering + optional int64 record_count = 2; +} + +message PbHeartbeatRespForTable { + required int64 table_id = 1; + optional ErrorResponse error = 2; +} + +message PbAlterConfig { + required string config_key = 1; + optional string config_value = 2; + required int32 op_type = 3; // SET=0, DELETE=1, APPEND=2, SUBTRACT=3 +} + +message PbAddColumn { + required string column_name = 1; + required bytes data_type_json = 2; + optional string comment = 3; + required int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3 + optional string agg_function_type = 5; + repeated PbKeyValue agg_function_params = 6; +} + +message PbDropColumn { + required string column_name = 1; +} + +message PbRenameColumn { + required string old_column_name = 1; + required string new_column_name = 2; +} + +message PbModifyColumn { + required string column_name = 1; + optional bytes data_type_json = 2; + optional string comment = 3; + optional int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3 +} + +message PbDescribeConfig { + required string config_key = 1; + optional string config_value = 2; + required string config_source = 3; +} + +message PbRebalancePlanForTable { + required int64 table_id = 1; + repeated PbRebalancePlanForBucket buckets_plan = 2; +} + +message PbRebalanceProgressForTable { + required int64 table_id = 1; + repeated PbRebalanceProgressForBucket buckets_progress = 2; +} + +message PbRebalanceProgressForBucket { + required PbRebalancePlanForBucket rebalance_plan = 1; + required int32 rebalance_status = 2; +} + +message PbRebalancePlanForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 original_leader = 3; + optional int32 new_leader = 4; + repeated int32 original_replicas = 5 [packed = true]; + repeated int32 new_replicas = 6 [packed = true]; +} + +message PbLakeTableSnapshotMetadata { + required int64 table_id = 1; + required int64 snapshot_id = 2; + required string tiered_bucket_offsets_file_path = 3; + optional string readable_bucket_offsets_file_path = 4; + // Snapshot retention policy: The earliest snapshot ID to retain. + // 1. If set, the system will keep all snapshots in the range [earliest_snapshot_id_to_keep, current_snapshot_id]. + // 2. If not set, the system defaults to a "Single Snapshot Retention" policy, keeping only the snapshot specified in this request. + optional int64 earliest_snapshot_id_to_keep = 5; +} + +message PbLakeTableSnapshotInfo { + optional int64 table_id = 1; + required int64 snapshot_id = 2; + repeated PbLakeTableOffsetForBucket buckets_req = 3; + // add table path to reduce get table_path by table id + optional PbTablePath table_path = 4; +} + +message PbLakeTableOffsetForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + // Deprecated: log_start_offset is no longer used. Field number 3 is reserved for protocol compatibility. + // optional int64 log_start_offset = 3; + optional int64 log_end_offset = 4; + // Deprecated: partition_name is no longer used. Field number 5 is reserved for protocol compatibility. + // optional string partition_name = 5; + optional int64 max_timestamp = 6; +} + +message PbPrepareLakeTableRespForTable { + optional int32 error_code = 1; + optional string error_message = 2; + optional int64 table_id = 3; + optional string lake_table_offsets_path = 4; +} + +message PbTableOffsets { + required int64 table_id = 1; + required PbTablePath table_path = 2; + repeated PbBucketOffset bucket_offsets = 3; +} + +message PbBucketOffset { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int64 log_end_offset = 4; +} + +// Helper message for table offsets in producer offsets (without table_path) +message PbProducerTableOffsets { + required int64 table_id = 1; + repeated PbBucketOffset bucket_offsets = 2; +} + +message PbDatabaseSummary { + required string database_name = 1; + required int64 created_time = 2; + required int32 table_count = 3; +} + +message PbKvSnapshotLeaseForTable { + required int64 table_id = 1; + repeated PbKvSnapshotLeaseForBucket bucket_snapshots = 2; +} + +message PbKvSnapshotLeaseForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required int64 snapshot_id = 3; +} + +message PbTableStatsReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 routing_bucket_count = 3; +} + +message PbTableStatsRespForBucket { + optional int32 error_code = 1; + optional string error_message = 2; + optional int64 partition_id = 3; + required int32 bucket_id = 4; + + // --- Table-level stats --- + // The number of rows in this bucket. + // For KV tables: the number of unique keys (live rows). + // For Log tables: total number of log records (highWatermark - logStartOffset). + // Absent if row count is not available (e.g., WAL changelog mode or legacy tables). + optional int64 row_count = 5; + + // --- data size stats (future) --- + // The data size in bytes of this bucket. + // For KV tables: the size of the KV store. + // For Log tables: the size of the log segments. + // Reserved for future use. + // optional int64 data_size_bytes = 6; + + // --- Column-level stats (future) --- + // Per-column statistics, keyed by column index. + // Only populated when target_columns is specified in the request. + // repeated PbColumnStats column_stats = 10; +} + +/** +* Definition of predicate system related message +*/ + +// PbPredicateType constants (using int32 instead of enum for proto3 compatibility) +// LEAF = 0, COMPOUND = 1 + +// Represents a predicate that can be serialized and transmitted across +// languages +message PbPredicate { + // See PbPredicateType constants: LEAF = 0, COMPOUND = 1 + required int32 type = 1; + optional PbLeafPredicate leaf = 2; + optional PbCompoundPredicate compound = 3; +} + +// Represents a leaf predicate that compares a field with literals +message PbLeafPredicate { + // The function to apply (see PbLeafFunction constants: EQUAL=0 ... NOT_IN=12) + required int32 function = 1; + // The schema field id of the referenced top-level field + required int32 field_id = 2; + // The literals to compare with + repeated PbLiteralValue literals = 3; +} + +// Represents a compound predicate that combines multiple predicates +message PbCompoundPredicate { + // The function to apply (see PbCompoundFunction constants: AND=0, OR=1) + required int32 function = 1; + // The child predicates + repeated PbPredicate children = 2; +} + +// PbLeafFunction constants (using int32 instead of enum for proto3 compatibility) +// EQUAL=0, NOT_EQUAL=1, LESS_THAN=2, LESS_OR_EQUAL=3, GREATER_THAN=4, +// GREATER_OR_EQUAL=5, IS_NULL=6, IS_NOT_NULL=7, STARTS_WITH=8, CONTAINS=9, +// END_WITH=10, IN=11, NOT_IN=12 + +// PbCompoundFunction constants (using int32 instead of enum for proto3 compatibility) +// AND=0, OR=1 + +// PbDataTypeRoot constants (using int32 instead of enum for proto3 compatibility) +// BOOLEAN=0, TINYINT=1, SMALLINT=2, INT=3, BIGINT=4, FLOAT=5, DOUBLE=6, +// CHAR=7, VARCHAR=8, DECIMAL=9, DATE=10, TIME_WITHOUT_TIME_ZONE=11, +// TIMESTAMP_WITHOUT_TIME_ZONE=12, TIMESTAMP_WITH_LOCAL_TIME_ZONE=13, +// BINARY=14, BYTES=15 + +// Represents a literal value +message PbLiteralValue { + // See PbDataTypeRoot constants: indicates which value field is populated + required int32 literal_type = 1; + required bool is_null = 2; + optional bool boolean_value = 3; + optional int32 int_value = 4; + optional int64 bigint_value = 5; + optional float float_value = 6; + optional double double_value = 7; + optional string string_value = 8; + optional bytes binary_value = 9; + optional int64 decimal_value = 10; // Serialized decimal (compact mode) + optional int64 timestamp_millis_value = 11; // Epoch millis + optional int32 timestamp_nano_of_millis_value = 12; // Nano of millis + optional bytes decimal_bytes = 13; // Serialized decimal (non-compact mode) +} diff --git a/fluss-rust/crates/fluss/src/client/table/scanner.rs b/fluss-rust/crates/fluss/src/client/table/scanner.rs index a56a536b312..feff43debb1 100644 --- a/fluss-rust/crates/fluss/src/client/table/scanner.rs +++ b/fluss-rust/crates/fluss/src/client/table/scanner.rs @@ -3054,6 +3054,7 @@ mod tests { records: None, filtered_end_offset, min_retain_offset: None, + resolved_earliest_offset: None, }], }], } @@ -3109,6 +3110,7 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + resolved_earliest_offset: None, }], }], }; @@ -3169,6 +3171,7 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + resolved_earliest_offset: None, }], }], }; @@ -3519,6 +3522,7 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + resolved_earliest_offset: None, }], }], }; diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs index 5afd9d99156..d7a39a80140 100644 --- a/fluss-rust/crates/fluss/src/proto/fluss.rs +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -1436,6 +1436,12 @@ pub struct PbFetchLogRespForBucket { /// returned for KV follower fetches and is distinct from the physical log_start_offset. #[prost(int64, optional, tag = "10")] pub min_retain_offset: ::core::option::Option, + /// The log start offset to which an EARLIEST_OFFSET request was resolved. + /// This field is present only when the request used EARLIEST_OFFSET. + /// Its presence also allows a client to distinguish a server that reports the + /// resolved offset from an older server that does not. + #[prost(int64, optional, tag = "11")] + pub resolved_earliest_offset: ::core::option::Option, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PbPutKvReqForBucket { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java index 0d14fb2412e..d7e539ecbfa 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java @@ -22,21 +22,46 @@ /** Structure used for lower level reads. */ @Internal public class LogReadInfo { + public static final long NO_RESOLVED_EARLIEST_OFFSET = -1L; private final FetchDataInfo fetchedData; private final long highWatermark; private final long logEndOffset; private final long minRetainOffset; + private final long resolvedEarliestOffset; public LogReadInfo( FetchDataInfo fetchedData, long highWatermark, long logEndOffset, long minRetainOffset) { + this( + fetchedData, + highWatermark, + logEndOffset, + minRetainOffset, + NO_RESOLVED_EARLIEST_OFFSET); + } + + public LogReadInfo( + FetchDataInfo fetchedData, + long highWatermark, + long logEndOffset, + long minRetainOffset, + long resolvedEarliestOffset) { this.fetchedData = fetchedData; this.highWatermark = highWatermark; this.logEndOffset = logEndOffset; this.minRetainOffset = minRetainOffset; + this.resolvedEarliestOffset = resolvedEarliestOffset; + } + + /** + * Returns the physical log offset resolved from an EARLIEST fetch request, or {@link + * #NO_RESOLVED_EARLIEST_OFFSET} if this read did not resolve EARLIEST. + */ + public long getResolvedEarliestOffset() { + return resolvedEarliestOffset; } public FetchDataInfo getFetchedData() { @@ -70,6 +95,8 @@ public String toString() { + logEndOffset + ", minRetainOffset=" + minRetainOffset + + ", resolvedEarliestOffset=" + + resolvedEarliestOffset + ')'; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index e02a6f26824..a9651cdbd1b 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -148,6 +148,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.apache.fluss.server.log.FetchParams.FETCH_FROM_EARLIEST_OFFSET; +import static org.apache.fluss.server.log.LogReadInfo.NO_RESOLVED_EARLIEST_OFFSET; import static org.apache.fluss.utils.PartitionUtils.HISTORICAL_PARTITION_VALUE; import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.fluss.utils.Preconditions.checkState; @@ -2030,10 +2032,11 @@ private LogReadInfo readRecords(FetchParams fetchParams, LogTablet logTablet) long initialLogEndOffset = logTablet.localLogEndOffset(); long initialMinRetainOffset = fetchParams.isFromFollower() && isKvTable() ? logTablet.getMinRetainOffset() : -1L; - long readOffset = - fetchParams.fetchOffset() == FetchParams.FETCH_FROM_EARLIEST_OFFSET - ? logTablet.logStartOffset() - : fetchParams.fetchOffset(); + long requestedFetchOffset = fetchParams.fetchOffset(); + boolean resolvingEarliest = requestedFetchOffset == FETCH_FROM_EARLIEST_OFFSET; + long readOffset = resolvingEarliest ? logTablet.logStartOffset() : requestedFetchOffset; + + long resolvedEarliestOffset = resolvingEarliest ? readOffset : NO_RESOLVED_EARLIEST_OFFSET; // todo validate fetched epoch. @@ -2057,7 +2060,11 @@ private LogReadInfo readRecords(FetchParams fetchParams, LogTablet logTablet) } } return new LogReadInfo( - fetchDataInfo, initialHighWatermark, initialLogEndOffset, initialMinRetainOffset); + fetchDataInfo, + initialHighWatermark, + initialLogEndOffset, + initialMinRetainOffset, + resolvedEarliestOffset); } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 33c3849ef9e..b5c041fdc2d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -133,13 +133,11 @@ import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.fluss.utils.concurrent.Scheduler; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; - import java.io.File; import java.io.IOException; import java.nio.file.DirectoryNotEmptyException; @@ -167,6 +165,7 @@ import java.util.stream.Stream; import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; +import static org.apache.fluss.rpc.entity.FetchLogResultForBucket.NO_RESOLVED_EARLIEST_OFFSET; import static org.apache.fluss.server.TabletManagerBase.getPartitionRegistration; import static org.apache.fluss.server.TabletManagerBase.getSchemaInfo; import static org.apache.fluss.server.TabletManagerBase.getTableRegistration; @@ -1869,9 +1868,8 @@ public Map readFromLog( fetchedData.hasFilteredEndOffset() ? fetchedData.getFilteredEndOffset() : -1L, - readInfo.hasMinRetainOffset() - ? readInfo.getMinRetainOffset() - : -1L); + readInfo.hasMinRetainOffset() ? readInfo.getMinRetainOffset() : -1L, + readInfo.getResolvedEarliestOffset()); logReadResult.put( tb, new LogReadResult(fetchLogResult, fetchedData.getFetchOffsetMetadata())); @@ -1908,7 +1906,7 @@ public Map readFromLog( } private @Nullable FetchLogResultForBucket tryFetchRemoteFirst( - Replica replica, long fetchOffset) { + Replica replica, long requestedFetchOffset) { if (!replica.isLeader()) { throw new NotLeaderOrFollowerException( String.format( @@ -1917,20 +1915,24 @@ public Map readFromLog( } TableBucket tb = replica.getTableBucket(); - long normalizedFetchOffset = - fetchOffset == FetchParams.FETCH_FROM_EARLIEST_OFFSET - ? replica.getLogStartOffset() - : fetchOffset; - if (!canFetchFromRemoteLog(replica, normalizedFetchOffset)) { + boolean resolvingEarliest = requestedFetchOffset == FetchParams.FETCH_FROM_EARLIEST_OFFSET; + long effectiveFetchOffset = + resolvingEarliest ? replica.getLogStartOffset() : requestedFetchOffset; + long resolvedEarliestOffset = + resolvingEarliest ? effectiveFetchOffset : NO_RESOLVED_EARLIEST_OFFSET; + if (!canFetchFromRemoteLog(replica, effectiveFetchOffset)) { return null; } try { RemoteLogFetchInfo remoteLogFetchInfo = - fetchLogFromRemote(replica, normalizedFetchOffset); + fetchLogFromRemote(replica, effectiveFetchOffset); if (remoteLogFetchInfo != null) { return FetchLogResultForBucket.remote( - tb, remoteLogFetchInfo, replica.getLogHighWatermark()); + tb, + remoteLogFetchInfo, + replica.getLogHighWatermark(), + resolvedEarliestOffset); } // Remote log is expected to cover the offset, but segments/manifest may not be ready // yet. @@ -1940,7 +1942,7 @@ public Map readFromLog( LOG.warn( "Failed to fetch remote log first for replica {} at offset {}; falling back to local reads", tb, - normalizedFetchOffset, + effectiveFetchOffset, e); // Fall back to local reads on remote fetch failures. return null; @@ -1948,29 +1950,39 @@ public Map readFromLog( } private FetchLogResultForBucket handleFetchOutOfRangeException( - Replica replica, long fetchOffset, Exception e) { + Replica replica, long requestedFetchOffset, Exception e) { TableBucket tb = replica.getTableBucket(); - if (fetchOffset == FetchParams.FETCH_FROM_EARLIEST_OFFSET) { - fetchOffset = replica.getLogStartOffset(); - } - - if (canFetchFromLakeLog(replica, fetchOffset)) { + boolean resolvingEarliest = requestedFetchOffset == FetchParams.FETCH_FROM_EARLIEST_OFFSET; + long effectiveFetchOffset = + resolvingEarliest ? replica.getLogStartOffset() : requestedFetchOffset; + long resolvedEarliestOffset = + resolvingEarliest ? effectiveFetchOffset : NO_RESOLVED_EARLIEST_OFFSET; + if (canFetchFromLakeLog(replica, effectiveFetchOffset)) { // todo: currently, we just return empty records directly // need to return the info of datalake to make client can fetch // from datalake directly return FetchLogResultForBucket.records( - tb, MemoryLogRecords.EMPTY, replica.getLogHighWatermark(), -1L, -1L); + tb, + MemoryLogRecords.EMPTY, + replica.getLogHighWatermark(), + -1L, + -1L, + resolvedEarliestOffset); } // Once we get a fetch out of range exception from local storage, we need to check whether // the log segment already upload to the remote storage. If uploaded, we will return a list // of RemoteLogSegment. For client fetcher, it will fetch the log from remote in client. // For follower, it can update its local metadata to adjust the next fetch offset. - else if (canFetchFromRemoteLog(replica, fetchOffset)) { + else if (canFetchFromRemoteLog(replica, effectiveFetchOffset)) { try { - RemoteLogFetchInfo remoteLogFetchInfo = fetchLogFromRemote(replica, fetchOffset); + RemoteLogFetchInfo remoteLogFetchInfo = + fetchLogFromRemote(replica, effectiveFetchOffset); if (remoteLogFetchInfo != null) { return FetchLogResultForBucket.remote( - tb, remoteLogFetchInfo, replica.getLogHighWatermark()); + tb, + remoteLogFetchInfo, + replica.getLogHighWatermark(), + resolvedEarliestOffset); } } catch (Exception ex) { return FetchLogResultForBucket.error(tb, ApiError.fromThrowable(ex)); @@ -1981,7 +1993,7 @@ else if (canFetchFromRemoteLog(replica, fetchOffset)) { new LogOffsetOutOfRangeException( String.format( "The fetch offset %s is out of range for table bucket %s", - fetchOffset, tb)))); + effectiveFetchOffset, tb)))); } else { return FetchLogResultForBucket.error(tb, ApiError.fromThrowable(e)); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 242f92a0ea4..b5ae9eef67e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -1101,6 +1101,11 @@ public static FetchLogResponse makeFetchLogResponse( // TODO: set log start offset here if we support log clean. .setLogStartOffset(0L); + if (bucketResult.hasResolvedEarliestOffset()) { + fetchLogRespForBucket.setResolvedEarliestOffset( + bucketResult.getResolvedEarliestOffset()); + } + if (bucketResult.fetchFromRemote()) { // set remote log fetch info. RemoteLogFetchInfo rlfInfo = bucketResult.remoteLogFetchInfo(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index 50a9dc79ec2..c115261b21b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -100,6 +100,8 @@ import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH; import static org.apache.fluss.server.kv.KvTabletTestUtils.flushAndWait; +import static org.apache.fluss.server.log.FetchParams.FETCH_FROM_EARLIEST_OFFSET; +import static org.apache.fluss.server.log.LogReadInfo.NO_RESOLVED_EARLIEST_OFFSET; import static org.apache.fluss.server.zk.data.LeaderAndIsr.INITIAL_LEADER_EPOCH; import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals; import static org.apache.fluss.testutils.DataTestUtils.createBasicMemoryLogRecords; @@ -166,6 +168,55 @@ void testGetOffsetRequiresLeader() throws Exception { .isInstanceOf(NotLeaderOrFollowerException.class); } + @Test + void testEarliestFetchReportsResolvedEarliestOffset() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 1); + Replica replica = makeLogReplica(DATA1_PHYSICAL_TABLE_PATH, tableBucket); + makeLogReplicaAsLeader(replica); + + replica.truncateFullyAndStartAt(42L); + + assertThat(replica.getLogStartOffset()).isEqualTo(42L); + assertThat(replica.getLocalLogEndOffset()).isEqualTo(42L); + assertThat(replica.getLogHighWatermark()).isEqualTo(42L); + + FetchParams fetchParams = new FetchParams(-1, Integer.MAX_VALUE); + fetchParams.setCurrentFetch( + DATA1_TABLE_ID, + FETCH_FROM_EARLIEST_OFFSET, + Integer.MAX_VALUE, + replica.getSchemaGetter(), + DEFAULT_COMPRESSION, + null, + new ProjectionPushdownCache()); + + LogReadInfo readInfo = replica.fetchRecords(fetchParams); + + assertThat(readInfo.getFetchedData().getRecords().sizeInBytes()).isZero(); + assertThat(readInfo.getResolvedEarliestOffset()).isEqualTo(42L); + assertThat(readInfo.getHighWatermark()).isEqualTo(42L); + } + + @Test + void testExplicitFetchDoesNotReportResolvedEarliestOffset() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID, 1); + Replica replica = makeLogReplica(DATA1_PHYSICAL_TABLE_PATH, tableBucket); + makeLogReplicaAsLeader(replica); + + FetchParams fetchParams = new FetchParams(-1, Integer.MAX_VALUE); + fetchParams.setCurrentFetch( + DATA1_TABLE_ID, + 0L, + Integer.MAX_VALUE, + replica.getSchemaGetter(), + DEFAULT_COMPRESSION, + null, + new ProjectionPushdownCache()); + + LogReadInfo readInfo = replica.fetchRecords(fetchParams); + assertThat(readInfo.getResolvedEarliestOffset()).isEqualTo(NO_RESOLVED_EARLIEST_OFFSET); + } + @Test void testAppendRecordsToLeader() throws Exception { Replica logReplica = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java index 4ee5f121894..d29f80b0656 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -55,7 +55,6 @@ import org.apache.fluss.server.metadata.PartitionMetadata; import org.apache.fluss.server.metadata.TableMetadata; import org.apache.fluss.utils.FlussPaths; - import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -138,6 +137,59 @@ void testAlterTableDistributionChanges() { .containsExactly(TableChange.modifyBucketCount(8)); } + @Test + void testFetchLogResponseResolvedEarliestOffset() { + TableBucket tableBucket = new TableBucket(1L, 0); + for (long offset : new long[] {-1L, 0L, 42L}) { + FetchLogResultForBucket result = + offset < 0 + ? FetchLogResultForBucket.records( + tableBucket, MemoryLogRecords.EMPTY, 42L, -1L, -1L) + : FetchLogResultForBucket.records( + tableBucket, MemoryLogRecords.EMPTY, 42L, -1L, -1L, offset); + PbFetchLogRespForBucket response = + ServerRpcMessageUtils.makeFetchLogResponse( + Collections.singletonMap(tableBucket, result)) + .getTablesRespsList() + .get(0) + .getBucketsRespsList() + .get(0); + assertThat(response.hasResolvedEarliestOffset()).isEqualTo(offset >= 0); + FetchLogResultForBucket decoded = + CommonRpcMessageUtils.getFetchLogResultForBucket( + tableBucket, TablePath.of("db", "table"), response); + assertThat(decoded.hasResolvedEarliestOffset()).isEqualTo(offset >= 0); + assertThat(decoded.getResolvedEarliestOffset()).isEqualTo(offset); + } + } + + @Test + void testRemoteFetchLogResponseResolvedEarliestOffset() { + TableBucket tableBucket = new TableBucket(1L, 0); + RemoteLogFetchInfo remoteInfo = + new RemoteLogFetchInfo("remote/tablet", null, Collections.emptyList(), 0); + for (long offset : new long[] {-1L, 0L, 42L}) { + FetchLogResultForBucket result = + offset < 0 + ? FetchLogResultForBucket.remote(tableBucket, remoteInfo, 100L) + : FetchLogResultForBucket.remote(tableBucket, remoteInfo, 100L, offset); + PbFetchLogRespForBucket response = + ServerRpcMessageUtils.makeFetchLogResponse( + Collections.singletonMap(tableBucket, result)) + .getTablesRespsList() + .get(0) + .getBucketsRespsList() + .get(0); + assertThat(response.hasResolvedEarliestOffset()).isEqualTo(offset >= 0); + FetchLogResultForBucket decoded = + CommonRpcMessageUtils.getFetchLogResultForBucket( + tableBucket, TablePath.of("db", "table"), response); + assertThat(decoded.fetchFromRemote()).isTrue(); + assertThat(decoded.hasResolvedEarliestOffset()).isEqualTo(offset >= 0); + assertThat(decoded.getResolvedEarliestOffset()).isEqualTo(offset); + } + } + @Test void testFetchLogResponseContainsMinRetainOffset() { TableBucket tableBucket = new TableBucket(1L, 0); From 5f7bc61bfac1abad5673fe594f309491ba301393 Mon Sep 17 00:00:00 2001 From: Shangqing Yang Date: Tue, 6 Oct 2026 18:34:13 -0400 Subject: [PATCH 2/4] [Client] Improve bounded log scan completion and error handling --- .../log/AbstractLogFetchCollector.java | 35 ++- .../table/scanner/log/AbstractLogScanner.java | 3 + .../table/scanner/log/LogFetchBuffer.java | 20 ++ .../client/table/scanner/log/LogFetcher.java | 22 +- .../client/table/scanner/log/LogScanner.java | 25 ++- .../table/scanner/log/LogScannerImpl.java | 3 + .../table/scanner/log/LogScannerStatus.java | 9 + .../log/LogFetchCollectorErrorTest.java | 209 ++++++++++++++++-- .../scanner/log/LogFetchCollectorTest.java | 3 +- .../table/scanner/log/LogFetcherTest.java | 28 +++ .../scanner/log/LogScannerStatusTest.java | 43 ++++ .../table/scanner/log/LogScannerTest.java | 43 ++++ .../UnsupportedBoundedEarliestException.java | 34 +++ 13 files changed, 415 insertions(+), 62 deletions(-) create mode 100644 fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java index a4ff1c17936..dc243458e4c 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogFetchCollector.java @@ -23,6 +23,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.FetchException; +import org.apache.fluss.exception.UnsupportedBoundedEarliestException; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.protocol.ApiError; @@ -95,16 +96,19 @@ public R collectFetch(final LogFetchBuffer logFetchBuffer) { // when the failure policy says this response must not be retried. if (result.shouldPropagateImmediately(e) && result.shouldDiscardFailedFetch(e, completedFetch)) { - CompletedFetch removed = logFetchBuffer.poll(); - checkState( - removed == completedFetch, - "Expected failed fetch %s at the head of the buffer, but found %s.", - completedFetch, - removed); try { - completedFetch.drain(); + checkState( + logFetchBuffer.removeCompletedFetch(completedFetch), + "Expected failed fetch %s at the head of the buffer.", + completedFetch); } catch (RuntimeException cleanupException) { e.addSuppressed(cleanupException); + } finally { + try { + completedFetch.drain(); + } catch (RuntimeException cleanupException) { + e.addSuppressed(cleanupException); + } } } throw e; @@ -383,15 +387,14 @@ private final class PollAccumulator { private boolean hasBoundedAccumulatedResult; private boolean hasDeliverableResult() { - return !fetched.isEmpty() - || !consumedUpToOffsets.isEmpty() - || logScannerStatus.hasPendingFinishedBuckets(); + return !fetched.isEmpty() || !consumedUpToOffsets.isEmpty(); } private boolean shouldPropagateImmediately(Exception exception) { if (exception instanceof FetchException) { - // Preserve the existing FetchException behavior: return any already - // accumulated records, progress, or bounded completion first. + // Preserve the existing FetchException behavior: return records or offset progress + // already consumed by this poll before propagating the error. A completion that was + // already pending before this poll must not delay an unrelated bucket's failure. return !hasDeliverableResult(); } @@ -434,12 +437,4 @@ private void addRecords(TableBucket tableBucket, List records, boolean bounde hasBoundedAccumulatedResult |= bounded; } } - - private static final class UnsupportedBoundedEarliestException - extends UnsupportedOperationException { - - private UnsupportedBoundedEarliestException(String message) { - super(message); - } - } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogScanner.java index 60ac2acbb11..397321aa94b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/AbstractLogScanner.java @@ -94,6 +94,9 @@ public final R poll(Duration timeout) { do { ScanRecords scanRecords = pollForFetches(); if (!scanRecords.hasProgress()) { + if (!logScannerStatus.hasActiveSubscriptions()) { + return emptyResult(); + } try { if (!logFetcher.awaitNotEmpty(startNanos + timeoutNanos)) { // no data in buffer within the timeout diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java index 3a2f5d98e6a..d3b9e34f4e8 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetchBuffer.java @@ -308,6 +308,26 @@ void checkException() throws FetchException { } } + /** + * Removes the expected completed fetch from the head of the buffer without propagating + * asynchronous buffer failures. + * + *

This is used only for cleanup after the caller has already obtained the fetch from {@link + * #peek()} and decided to discard it. Any asynchronous failure remains recorded in the buffer + * and will be propagated by the next normal buffer operation. + */ + boolean removeCompletedFetch(CompletedFetch expectedFetch) { + return inLock( + lock, + () -> { + if (completedFetches.peek() != expectedFetch) { + return false; + } + completedFetches.poll(); + return true; + }); + } + @Override public void close() throws Exception { inLock(lock, () -> retainAll(Collections.emptySet())); diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java index ded99a56241..b3ae3acfc84 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java @@ -456,6 +456,17 @@ private synchronized void handleFetchLogResponse( : null, respForBucket.getBucketId()); + FetchLogResultForBucket fetchResultForBucket = + getFetchLogResultForBucket(tb, tablePathForResp, respForBucket); + + // If the fetch failed, handle metadata-related side effects first. + // The error response is still buffered below so that the collector + // can deliver the exception with the normal poll semantics. + if (fetchResultForBucket.getErrorCode() != Errors.NONE.code()) { + ApiError error = ApiError.fromErrorMessage(respForBucket); + handleFetchLogExceptionForBucket(tb, destination, error); + } + Long requestedFetchOffset = fetchRequestContext.requestedOffsets.get(tb); Long currentFetchOffset = logScannerStatus.getBucketOffset(tb); @@ -472,17 +483,6 @@ private synchronized void handleFetchLogResponse( continue; } - FetchLogResultForBucket fetchResultForBucket = - getFetchLogResultForBucket(tb, tablePathForResp, respForBucket); - - // If the fetch failed, handle metadata-related side effects first. - // The error response is still buffered below so that the collector - // can deliver the exception with the normal poll semantics. - if (fetchResultForBucket.getErrorCode() != Errors.NONE.code()) { - ApiError error = ApiError.fromErrorMessage(respForBucket); - handleFetchLogExceptionForBucket(tb, destination, error); - } - long stoppingOffset = logScannerStatus.getBucketStoppingOffset(tb); boolean boundedEarliest = diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java index a46caeb5c4f..42ce31a4db6 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java @@ -18,6 +18,7 @@ package org.apache.fluss.client.table.scanner.log; import org.apache.fluss.annotation.PublicEvolving; +import org.apache.fluss.exception.UnsupportedBoundedEarliestException; import java.time.Duration; @@ -104,10 +105,10 @@ default void subscribeFromBeginning(int bucket) { *

The stopping offset is exclusive, so the subscribed range is {@code [startingOffset, * stoppingOffset)}. The stopping offset must be non-negative. * - *

When {@link #EARLIEST_OFFSET} needs to be resolved by the server for a non-empty bounded - * range, the server must report the physical offset to which EARLIEST was resolved. If the - * connected server does not support this capability, polling fails with {@link - * UnsupportedOperationException}. + *

When scanning from{@link #EARLIEST_OFFSET}, the server must report the resolved starting + * offset unless the range can be completed locally, such as when the stopping offset is zero. + * If the connected server does not support this capability, a subsequent poll fails with {@link + * UnsupportedBoundedEarliestException}. * *

The starting offset must be non-negative or {@link #EARLIEST_OFFSET}. A resolved starting * offset at or beyond the stopping offset represents an empty range. A stopping offset of zero @@ -118,6 +119,10 @@ default void subscribeFromBeginning(int bucket) { * before completing the read. The exclusive boundary takes precedence over keeping * UPDATE_BEFORE and UPDATE_AFTER together. * + *

Once all currently subscribed buckets are bounded and have completed, subsequent polls + * return immediately without waiting for the requested timeout. A later subscription may make + * the scanner active again. + * *

Please use {@link #subscribeBounded(long, int, long, long)} to subscribe a partitioned * table. * @@ -140,10 +145,10 @@ default void subscribeBounded(int bucket, long startingOffset, long stoppingOffs *

The stopping offset is exclusive, so the subscribed range is {@code [startingOffset, * stoppingOffset)}. The stopping offset must be non-negative. * - *

When {@link #EARLIEST_OFFSET} needs to be resolved by the server for a non-empty bounded - * range, the server must report the physical offset to which EARLIEST was resolved. If the - * connected server does not support this capability, polling fails with {@link - * UnsupportedOperationException}. + *

When scanning from{@link #EARLIEST_OFFSET}, the server must report the resolved starting + * offset unless the range can be completed locally, such as when the stopping offset is zero. + * If the connected server does not support this capability, a subsequent poll fails with {@link + * UnsupportedBoundedEarliestException}. * *

The starting offset must be non-negative or {@link #EARLIEST_OFFSET}. A resolved starting * offset at or beyond the stopping offset represents an empty range. A stopping offset of zero @@ -154,6 +159,10 @@ default void subscribeBounded(int bucket, long startingOffset, long stoppingOffs * before completing the read. The exclusive boundary takes precedence over keeping * UPDATE_BEFORE and UPDATE_AFTER together. * + *

Once all currently subscribed buckets are bounded and have completed, subsequent polls + * return immediately without waiting for the requested timeout. A later subscription may make + * the scanner active again. + * *

Please use {@link #subscribeBounded(int, long, long)} to subscribe a non-partitioned * table. * diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java index d095dcfa846..1d2eae35048 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java @@ -198,6 +198,9 @@ public ArrowScanRecords pollRecordBatch(Duration timeout) { do { ArrowScanRecords scanRecords = pollForRecordBatches(); if (!scanRecords.hasProgress()) { + if (!logScannerStatus.hasActiveSubscriptions()) { + return scanRecords; + } try { if (!logFetcher.awaitNotEmpty(startNanos + timeoutNanos)) { return scanRecords; diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java index 15e3666d6df..821b48769ed 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java @@ -191,6 +191,15 @@ synchronized Set drainFinishedBuckets() { return finishedBuckets; } + synchronized boolean hasActiveSubscriptions() { + for (BucketScanStatus bucketScanStatus : bucketStatusMap.bucketStatusMap().values()) { + if (!bucketScanStatus.hasReachedStoppingOffset()) { + return true; + } + } + return false; + } + private BucketScanStatus bucketStatus(TableBucket tableBucket) { return bucketStatusMap.statusValue(tableBucket); } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java index bed360d4a5d..eea594ee48d 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java @@ -32,17 +32,21 @@ import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.rpc.protocol.Errors; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; @@ -56,6 +60,7 @@ import static org.apache.fluss.testutils.DataTestUtils.createBasicMemoryLogRecords; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; /** Tests error delivery alongside progress and completion for both collector formats. */ class LogFetchCollectorErrorTest { @@ -92,19 +97,22 @@ void testAuthorizationErrorDoesNotTrapPendingCompletion(boolean arrow) { @ParameterizedTest @ValueSource(booleans = {false, true}) - void testCompletionDefersFetchErrorUntilNextPoll(boolean arrow) { + void testPendingCompletionDoesNotDeferFetchError(boolean arrow) { AbstractLogFetchCollector collector = collector(arrow); + // Completion exists before this poll starts. status.assignScanBucket(finishedBucket, 0L, 0L); CompletedFetch failed = addError(Errors.LOG_OFFSET_OUT_OF_RANGE_EXCEPTION); - - ScanRecords result = collectProgressOnly(collector); - assertThat(result.finishedBuckets()).containsExactly(finishedBucket); - assertThat(result.isEmpty()).isTrue(); - assertThat(buffer.peek()).isSameAs(failed); - assertThat(failed.isConsumed()).isFalse(); assertThatThrownBy(() -> collector.collectFetch(buffer)).isInstanceOf(FetchException.class); + // The failed zero-byte response is discarded after immediate propagation. assertThat(failed.isConsumed()).isTrue(); assertThat(buffer.peek()).isNull(); + // Throwing the unrelated error must not consume the pending completion. + assertThat(status.hasPendingFinishedBuckets()).isTrue(); + ScanRecords result = collectProgressOnly(collector); + assertThat(result.finishedBuckets()).containsExactly(finishedBucket); + assertThat(result.isEmpty()).isTrue(); + // Completion is one-shot. + assertThat(status.hasPendingFinishedBuckets()).isFalse(); assertThat(collectProgressOnly(collector).hasProgress()).isFalse(); } @@ -128,16 +136,13 @@ void testProgressDefersFetchErrorUntilNextPoll(boolean arrow) { } @ParameterizedTest - @CsvSource({"false, false", "false, true", "true, false", "true, true"}) - void testRecordErrorDefersForProgressOrCompletion(boolean arrow, boolean completion) { + @ValueSource(booleans = {false, true}) + void testPendingCompletionDoesNotDeferRecordError(boolean arrow) { AbstractLogFetchCollector collector = collector(arrow); - if (completion) { - status.assignScanBucket(finishedBucket, 0L, 0L); - } else { - status.assignScanBucket(finishedBucket, 0L, LogScanner.NO_STOPPING_OFFSET); - buffer.add(makeFetch(FetchLogResultForBucket.empty(finishedBucket, 10L, 10L))); - } + // The completion already exists before this poll starts. + status.assignScanBucket(finishedBucket, 0L, 0L); status.assignScanBucket(errorBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + FetchException failure = new FetchException("record decoding failed"); CompletedFetch failed = new DefaultCompletedFetch( @@ -162,19 +167,60 @@ List fetchArrowBatches(int maxRecords) { buffer.add(failed); CompletedFetch queued = makeFetch(FetchLogResultForBucket.empty(errorBucket, 10L, -1L)); buffer.add(queued); + assertThatThrownBy(() -> collector.collectFetch(buffer)).isSameAs(failure); + // The unrelated completion must not delay the record decoding failure. + assertThat(status.hasPendingFinishedBuckets()).isTrue(); + // A decoding failure remains attached to the in-flight fetch and must not + // dequeue the following response. + assertThat(buffer.nextInLineFetch()).isSameAs(failed); + assertThat(buffer.peek()).isSameAs(queued); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testRecordErrorIsDeferredAfterProgress(boolean arrow) { + AbstractLogFetchCollector collector = collector(arrow); + status.assignScanBucket(finishedBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + // This fetch advances the scanner during this poll. + buffer.add(makeFetch(FetchLogResultForBucket.empty(finishedBucket, 10L, 10L))); + status.assignScanBucket(errorBucket, 0L, LogScanner.NO_STOPPING_OFFSET); + FetchException failure = new FetchException("record decoding failed"); + CompletedFetch failed = + new DefaultCompletedFetch( + errorBucket, + DATA1_TABLE_PATH, + FetchLogResultForBucket.empty(errorBucket, 10L, -1L), + readContext, + status, + true, + 0L, + null) { + @Override + public List fetchRecords(int maxRecords) { + throw failure; + } + @Override + List fetchArrowBatches(int maxRecords) { + throw failure; + } + }; + buffer.add(failed); + CompletedFetch queued = makeFetch(FetchLogResultForBucket.empty(errorBucket, 10L, -1L)); + buffer.add(queued); + // The first poll has already advanced another bucket, so that progress + // must be delivered before propagating the decoding failure. ScanRecords result = collectProgressOnly(collector); assertThat(result.hasProgress()).isTrue(); assertThat(result.isEmpty()).isTrue(); - if (completion) { - assertThat(result.finishedBuckets()).containsExactly(finishedBucket); - } else { - assertThat(result.consumedUpToOffset(finishedBucket)).isEqualTo(10L); - } + assertThat(result.consumedUpToOffset(finishedBucket)).isEqualTo(10L); + assertThat(result.finishedBuckets()).isEmpty(); assertThat(buffer.nextInLineFetch()).isSameAs(failed); assertThat(buffer.peek()).isSameAs(queued); + // With no newly accumulated result, the decoding failure is now propagated. assertThatThrownBy(() -> collector.collectFetch(buffer)).isSameAs(failure); - // Record errors stay attached to the in-flight fetch; do not dequeue another response. + // Record errors stay attached to the in-flight fetch; the next queued + // response must not be removed accidentally. assertThat(buffer.nextInLineFetch()).isSameAs(failed); assertThat(buffer.peek()).isSameAs(queued); } @@ -343,6 +389,102 @@ void testOffsetOutOfRangeReportsRequestedOffset(boolean arrow) { .hasMessageContaining("fetching offset 123"); } + @Test + void testAsyncBufferFailureDoesNotInterruptFailedFetchCleanup() throws Exception { + TableBucket failedBucket = new TableBucket(DATA1_TABLE_ID, 0); + TableBucket remoteBucket = new TableBucket(DATA1_TABLE_ID, 1); + // Bounded EARLIEST requires the server to report the resolved starting offset. + // The response below intentionally uses the old records(...) overload, so + // initialize() will fail with UnsupportedBoundedEarliestException. + status.assignScanBucket(failedBucket, EARLIEST_OFFSET, 100L); + + RuntimeException remoteFailure = new RuntimeException("remote download failed"); + + InjectingFailureLogFetchBuffer testBuffer = + new InjectingFailureLogFetchBuffer(remoteBucket, remoteFailure); + ByteBuf parsedByteBuf = Unpooled.buffer(1); + assertThat(parsedByteBuf.refCnt()).isEqualTo(1); + + FetchLogResultForBucket result = + FetchLogResultForBucket.records(failedBucket, data1Records(), 100L, -1L, -1L); + CompletedFetch failedFetch = + new DefaultCompletedFetch( + failedBucket, + DATA1_TABLE_PATH, + result, + readContext, + status, + true, + EARLIEST_OFFSET, + parsedByteBuf); + + testBuffer.add(failedFetch); + AbstractLogFetchCollector collector = collector(false); + // The initialization failure remains the primary failure even though an + // asynchronous remote-download failure is injected immediately before + // the failed fetch is removed from the buffer. + assertThatThrownBy(() -> collector.collectFetch(testBuffer)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("resolved_earliest_offset"); + + // The failed fetch must always be drained. + assertThat(failedFetch.isConsumed()).isTrue(); + + // DefaultCompletedFetch.drain() owns releasing the retained network buffer. + assertThat(parsedByteBuf.refCnt()).isZero(); + + // The failed fetch was removed rather than being left in the completed queue. + // bufferedBuckets() does not surface the pending asynchronous failure, so it + // can be used to inspect queue membership here. + assertThat(testBuffer.bufferedBuckets()).doesNotContain(failedBucket); + + // The unrelated asynchronous failure was not consumed or lost by cleanup. + // It remains in the buffer and is propagated by the next normal operation. + assertThatThrownBy(testBuffer::peek) + .isInstanceOf(FetchException.class) + .hasRootCauseMessage("remote download failed"); + } + + @Test + void testFailedFetchCleanupExceptionIsSuppressed() throws Exception { + TableBucket failedBucket = new TableBucket(DATA1_TABLE_ID, 0); + status.assignScanBucket(failedBucket, EARLIEST_OFFSET, 100L); + RuntimeException cleanupFailure = new RuntimeException("cleanup failed"); + + FetchLogResultForBucket result = + FetchLogResultForBucket.records(failedBucket, data1Records(), 100L, -1L, -1L); + + CompletedFetch failedFetch = + new DefaultCompletedFetch( + failedBucket, + DATA1_TABLE_PATH, + result, + readContext, + status, + true, + EARLIEST_OFFSET, + null) { + @Override + void drain() { + super.drain(); + throw cleanupFailure; + } + }; + + buffer.add(failedFetch); + + AbstractLogFetchCollector collector = collector(false); + Throwable thrown = catchThrowable(() -> collector.collectFetch(buffer)); + assertThat(thrown) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("resolved_earliest_offset"); + assertThat(thrown.getSuppressed()).containsExactly(cleanupFailure); + // super.drain() ran before the synthetic cleanup failure. + assertThat(failedFetch.isConsumed()).isTrue(); + // Removal happened before drain, so the failed response must not remain queued. + assertThat(buffer.peek()).isNull(); + } + private AbstractLogFetchCollector collector(boolean arrow) { TestingMetadataUpdater metadata = new TestingMetadataUpdater( @@ -410,4 +552,27 @@ private MemoryLogRecords data1Records() throws Exception { DEFAULT_COMPRESSION, true); } + + private static final class InjectingFailureLogFetchBuffer extends LogFetchBuffer { + private final TableBucket remoteBucket; + private final RuntimeException remoteFailure; + private final AtomicBoolean failureInjected = new AtomicBoolean(); + + private InjectingFailureLogFetchBuffer( + TableBucket remoteBucket, RuntimeException remoteFailure) { + this.remoteBucket = remoteBucket; + this.remoteFailure = remoteFailure; + } + + @Override + boolean removeCompletedFetch(CompletedFetch expectedFetch) { + if (failureInjected.compareAndSet(false, true)) { + // This is the same entry point used by the remote-download completion + // callback. Inject the failure after collector.peek() has succeeded but + // immediately before the failed response is removed. + tryComplete(remoteBucket, remoteFailure); + } + return super.removeCompletedFetch(expectedFetch); + } + } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java index d020bd7eccd..430fd1f1fe1 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java @@ -22,6 +22,7 @@ import org.apache.fluss.client.table.scanner.ScanRecord; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.UnsupportedBoundedEarliestException; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ArrowBatchData; @@ -116,7 +117,7 @@ void testBoundedEarliestRequiresResolvedOffset(boolean arrow) { : logFetchCollector; assertThatThrownBy(() -> collector.collectFetch(logFetchBuffer)) - .isInstanceOf(UnsupportedOperationException.class) + .isInstanceOf(UnsupportedBoundedEarliestException.class) .hasMessageContaining("resolved_earliest_offset"); assertThat(logFetchBuffer.peek()).isNull(); assertThat(logScannerStatus.getBucketOffset(tb)).isEqualTo(EARLIEST_OFFSET); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java index 97b0182ce41..1bc3d7ba9b6 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java @@ -154,6 +154,34 @@ void sendFetchRequestWithNotLeaderOrFollowerException() { assertThat(metadataUpdater.getBucketLocation(tb1)).isNotPresent(); } + @Test + void testStaleLeaderErrorInvalidatesMetadataAfterResubscription() { + IOUtils.closeQuietly(logFetcher); + DelayedTabletServerGateway delayedGateway = new DelayedTabletServerGateway(); + metadataUpdater = initializeMetadataUpdater(delayedGateway); + logFetcher = createLogFetcher(new Configuration()); + LogScannerStatus status = logFetcher.getLogScannerStatus(); + status.assignScanBuckets(Collections.singletonMap(tb1, 10L)); + + Map requestMap = + logFetcher.prepareFetchLogRequests(Collections.singletonList(tb1)); + assertThat(getFetchLogData(requestMap.get(1)).get(tb1).getFetchOffset()).isEqualTo(10L); + logFetcher.sendFetchRequest(1, requestMap.get(1)); + + // Change the subscription while the request for offset 10 is still in flight. + status.assignScanBuckets(Collections.singletonMap(tb1, 100L)); + assertThat(delayedGateway.getRequestCount()).isEqualTo(1); + assertThat(metadataUpdater.getBucketLocation(tb1)).isPresent(); + assertThat(logFetcher.getCompletedFetchesSize()).isZero(); + + // The stale NOT_LEADER_OR_FOLLOWER response must still invalidate the cached route. + delayedGateway.completeResponse(); + + assertThat(metadataUpdater.getBucketLocation(tb1)).isNotPresent(); + assertThat(logFetcher.getCompletedFetchesSize()).isZero(); + assertThat(status.getBucketOffset(tb1)).isEqualTo(100L); + } + @Test void testUnregisterTableDiscardsBufferedFetches() { Map requestMap = diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java index 96d1eab2d02..05eca652804 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerStatusTest.java @@ -222,4 +222,47 @@ void testResolveBoundedEarliestIgnoresStaleRequest() { assertThat(scannerStatus.hasReachedStoppingOffset(TABLE_BUCKET)).isFalse(); assertThat(scannerStatus.hasPendingFinishedBuckets()).isFalse(); } + + @Test + void testFinishedBoundedSubscriptionsAreNotActive() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + TableBucket first = new TableBucket(1L, 0); + TableBucket second = new TableBucket(1L, 1); + + scannerStatus.assignScanBucket(first, 0L, 5L); + scannerStatus.assignScanBucket(second, 10L, 20L); + assertThat(scannerStatus.hasActiveSubscriptions()).isTrue(); + + scannerStatus.updateOffset(first, 5L); + assertThat(scannerStatus.hasActiveSubscriptions()).isTrue(); + + scannerStatus.updateOffset(second, 20L); + assertThat(scannerStatus.hasActiveSubscriptions()).isFalse(); + } + + @Test + void testUnboundedSubscriptionRemainsActiveAfterBoundedSubscriptionFinishes() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + TableBucket bounded = new TableBucket(1L, 0); + TableBucket unbounded = new TableBucket(1L, 1); + + scannerStatus.assignScanBucket(bounded, 0L, 5L); + scannerStatus.assignScanBucket(unbounded, 0L, NO_STOPPING_OFFSET); + scannerStatus.updateOffset(bounded, 5L); + + assertThat(scannerStatus.hasReachedStoppingOffset(bounded)).isTrue(); + assertThat(scannerStatus.hasActiveSubscriptions()).isTrue(); + } + + @Test + void testResubscribeReactivatesFinishedBucket() { + LogScannerStatus scannerStatus = new LogScannerStatus(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 0L, 5L); + scannerStatus.updateOffset(TABLE_BUCKET, 5L); + assertThat(scannerStatus.hasActiveSubscriptions()).isFalse(); + + scannerStatus.assignScanBucket(TABLE_BUCKET, 5L, 10L); + assertThat(scannerStatus.hasActiveSubscriptions()).isTrue(); + } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java index e7563030ca4..ba99b533c28 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogScannerTest.java @@ -31,6 +31,10 @@ import java.time.Duration; import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO; @@ -170,6 +174,45 @@ void testSubscriptionRejectsWrongTableType(boolean bounded) throws Exception { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCompletedBoundedScanDoesNotWaitForPollTimeout(boolean arrow) throws Exception { + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogScannerImpl scanner = createScanner(downloader)) { + TableBucket bucket = new TableBucket(DATA1_TABLE_ID, 0); + scanner.subscribeBounded(0, 0L, 0L); + if (arrow) { + try (ArrowScanRecords completion = scanner.pollRecordBatch(Duration.ZERO)) { + assertThat(completion.finishedBuckets()).containsExactly(bucket); + } + } else { + assertThat(scanner.poll(Duration.ZERO).finishedBuckets()).containsExactly(bucket); + } + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + if (arrow) { + Future future = + executor.submit( + () -> { + try (ArrowScanRecords records = + scanner.pollRecordBatch(Duration.ofMinutes(1))) { + return records.hasProgress(); + } + }); + assertThat(future.get(3, TimeUnit.SECONDS)).isFalse(); + } else { + Future future = + executor.submit(() -> scanner.poll(Duration.ofMinutes(1))); + assertThat(future.get(3, TimeUnit.SECONDS).hasProgress()).isFalse(); + } + } finally { + scanner.wakeup(); + executor.shutdownNow(); + } + } + } + private static LogScannerImpl createScanner(RemoteFileDownloader downloader) { return createScanner(downloader, DATA1_TABLE_INFO); } diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java b/fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java new file mode 100644 index 00000000000..197485ed3ec --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.exception; + +import org.apache.fluss.annotation.PublicEvolving; + +/** + * Thrown when a bounded scan starts from EARLIEST_OFFSET but the server does not report the + * physical offset to which EARLIEST was resolved. + */ +@PublicEvolving +public class UnsupportedBoundedEarliestException extends UnsupportedOperationException { + private static final long serialVersionUID = 1L; + + public UnsupportedBoundedEarliestException(String message) { + super(message); + } +} From a80019caa0ee97ff305cc604c267d3c82ebd9562 Mon Sep 17 00:00:00 2001 From: Shangqing Yang Date: Tue, 6 Oct 2026 20:47:27 -0400 Subject: [PATCH 3/4] [Client] Migrate lake snapshot log reader to bounded scanning --- .../batch/LakeSnapshotAndLogSplitScanner.java | 32 +-- .../client/table/scanner/log/LogScanner.java | 4 +- .../LakeSnapshotAndLogSplitScannerTest.java | 198 ++++++++++++++++++ .../log/BoundedLogScannerRetentionITCase.java | 66 +++++- .../scanner/log/LogFetchCollectorTest.java | 8 +- .../fluss/server/replica/ReplicaManager.java | 2 + .../utils/ServerRpcMessageUtilsTest.java | 1 + 7 files changed, 288 insertions(+), 23 deletions(-) create mode 100644 fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java index d8292d11c8c..17e30ac0ea8 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java @@ -65,7 +65,7 @@ public class LakeSnapshotAndLogSplitScanner implements BatchScanner { private Map logRows; private final LogScanner logScanner; - private final long stoppingOffset; + private final TableBucket tableBucket; private boolean logScanFinished; private SortMergeReader currentSortMergeReader; @@ -81,7 +81,7 @@ public LakeSnapshotAndLogSplitScanner( this.pkIndexes = table.getTableInfo().getSchema().getPrimaryKeyIndexes(); this.lakeSplits = lakeSplits; this.lakeSource = lakeSource; - this.stoppingOffset = stoppingOffset; + this.tableBucket = tableBucket; ProjectionPlan projectionPlan = ProjectionPlan.create( table.getTableInfo().getRowType().getFieldCount(), @@ -97,14 +97,19 @@ public LakeSnapshotAndLogSplitScanner( .mapToObj(field -> new int[] {field}) .toArray(int[][]::new)); - if (tableBucket.getPartitionId() != null) { - this.logScanner.subscribe( - tableBucket.getPartitionId(), tableBucket.getBucket(), startingOffset); - } else { - this.logScanner.subscribe(tableBucket.getBucket(), startingOffset); - } - this.logScanFinished = startingOffset >= stoppingOffset || stoppingOffset <= 0; + if (!logScanFinished) { + if (tableBucket.getPartitionId() != null) { + this.logScanner.subscribeBounded( + tableBucket.getPartitionId(), + tableBucket.getBucket(), + startingOffset, + stoppingOffset); + } else { + this.logScanner.subscribeBounded( + tableBucket.getBucket(), startingOffset, stoppingOffset); + } + } } @Nullable @@ -187,11 +192,10 @@ private void pollLogRecords(Duration timeout) { InternalRow keyRow = keyValueRow.keyRow(); // upsert the key value row logRows.put(keyRow, keyValueRow); - if (scanRecord.logOffset() >= stoppingOffset - 1) { - // has reached to the end - logScanFinished = true; - break; - } + } + // Completion may accompany the final records or arrive without any records. + if (scanRecords.finishedBuckets().contains(tableBucket)) { + logScanFinished = true; } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java index 42ce31a4db6..6cfc2e860fa 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScanner.java @@ -105,7 +105,7 @@ default void subscribeFromBeginning(int bucket) { *

The stopping offset is exclusive, so the subscribed range is {@code [startingOffset, * stoppingOffset)}. The stopping offset must be non-negative. * - *

When scanning from{@link #EARLIEST_OFFSET}, the server must report the resolved starting + *

When scanning from {@link #EARLIEST_OFFSET}, the server must report the resolved starting * offset unless the range can be completed locally, such as when the stopping offset is zero. * If the connected server does not support this capability, a subsequent poll fails with {@link * UnsupportedBoundedEarliestException}. @@ -145,7 +145,7 @@ default void subscribeBounded(int bucket, long startingOffset, long stoppingOffs *

The stopping offset is exclusive, so the subscribed range is {@code [startingOffset, * stoppingOffset)}. The stopping offset must be non-negative. * - *

When scanning from{@link #EARLIEST_OFFSET}, the server must report the resolved starting + *

When scanning from {@link #EARLIEST_OFFSET}, the server must report the resolved starting * offset unless the range can be completed locally, such as when the stopping offset is zero. * If the connected server does not support this capability, a subsequent poll fails with {@link * UnsupportedBoundedEarliestException}. diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java new file mode 100644 index 00000000000..180d4e6d07b --- /dev/null +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.client.table.scanner.batch; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.Scan; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.exception.FetchException; +import org.apache.fluss.lake.source.LakeSource; +import org.apache.fluss.lake.source.LakeSplit; +import org.apache.fluss.lake.source.SortedRecordReader; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Set; + +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; +import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests bounded log consumption before merging a lake snapshot. */ +class LakeSnapshotAndLogSplitScannerTest { + private static final Duration TIMEOUT = Duration.ZERO; + private static final TableBucket BUCKET = new TableBucket(DATA1_TABLE_ID_PK, 0); + + private Table table; + private LogScanner logScanner; + private LakeSource lakeSource; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() throws Exception { + table = mock(Table.class); + Scan scan = mock(Scan.class); + logScanner = mock(LogScanner.class); + lakeSource = mock(LakeSource.class); + SortedRecordReader reader = mock(SortedRecordReader.class); + when(table.getTableInfo()).thenReturn(DATA1_TABLE_INFO_PK); + when(table.newScan()).thenReturn(scan); + when(scan.project(any(int[].class))).thenReturn(scan); + when(scan.createLogScanner()).thenReturn(logScanner); + when(lakeSource.createRecordReader(any())).thenReturn(reader); + when(reader.order()).thenReturn(Comparator.comparingInt(row -> row.getInt(0))); + when(reader.read()) + .thenReturn( + CloseableIterator.wrap( + Collections.singletonList( + new ScanRecord(GenericRow.of(1, null))) + .iterator())); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testConsumesFinalRecordsBeforeMergingSnapshot(boolean partitioned) throws Exception { + TableBucket bucket = partitioned ? new TableBucket(DATA1_TABLE_ID_PK, 7L, 0) : BUCKET; + when(logScanner.poll(TIMEOUT)) + .thenReturn( + finishedRecords( + bucket, + new ScanRecord(10, 0, ChangeType.DELETE, GenericRow.of(1, null)), + new ScanRecord(11, 0, ChangeType.INSERT, GenericRow.of(2, null)))); + + try (LakeSnapshotAndLogSplitScanner scanner = createScanner(bucket, 10L, 20L)) { + if (partitioned) { + verify(logScanner).subscribeBounded(7L, 0, 10L, 20L); + } else { + verify(logScanner).subscribeBounded(0, 10L, 20L); + } + // Completion can accompany records even when the final visible offset is below stop-1. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + try (CloseableIterator rows = scanner.pollBatch(TIMEOUT)) { + assertThat(rows).isNotNull(); + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getInt(0)).isEqualTo(2); + assertThat(rows.hasNext()).isFalse(); + } + assertThat(scanner.pollBatch(TIMEOUT)).isNull(); + verify(logScanner, times(1)).poll(TIMEOUT); + } + verify(logScanner).close(); + } + + @Test + void testCompletionWithoutRecords() throws Exception { + when(logScanner.poll(TIMEOUT)).thenReturn(finishedRecords(BUCKET)); + try (LakeSnapshotAndLogSplitScanner scanner = createScanner(BUCKET, 10L, 20L)) { + verify(logScanner).subscribeBounded(0, 10L, 20L); + // The bounded scan can finish without returning any records. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + // The next poll starts reading the merged lake snapshot. + try (CloseableIterator rows = scanner.pollBatch(TIMEOUT)) { + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getInt(0)).isEqualTo(1); + assertThat(rows.hasNext()).isFalse(); + } + assertThat(scanner.pollBatch(TIMEOUT)).isNull(); + verify(logScanner, times(1)).poll(TIMEOUT); + } + } + + @ParameterizedTest + @CsvSource({"10, 10", "10, 5", "-2, 0"}) + void testEmptyLogRangeDoesNotPollLog(long startingOffset, long stoppingOffset) + throws Exception { + try (LakeSnapshotAndLogSplitScanner scanner = + createScanner(BUCKET, startingOffset, stoppingOffset)) { + try (CloseableIterator rows = scanner.pollBatch(TIMEOUT)) { + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getInt(0)).isEqualTo(1); + } + verify(logScanner, times(0)).poll(any()); + } + } + + @Test + void testProgressWithoutCompletionKeepsReading() throws Exception { + ScanRecords progress = + new ScanRecords(Collections.emptyMap(), Collections.singletonMap(BUCKET, 15L)); + when(logScanner.poll(TIMEOUT)).thenReturn(progress, finishedRecords(BUCKET)); + try (LakeSnapshotAndLogSplitScanner scanner = createScanner(BUCKET, 10L, 20L)) { + for (int i = 0; i < 2; i++) { + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + } + try (CloseableIterator rows = scanner.pollBatch(TIMEOUT)) { + assertThat(rows.hasNext()).isTrue(); + } + verify(logScanner, times(2)).poll(TIMEOUT); + } + } + + @Test + void testMissingLogRangeErrorIsPropagated() throws Exception { + FetchException failure = + new FetchException("Requested snapshot log offset is out of range"); + when(logScanner.poll(TIMEOUT)).thenThrow(failure); + try (LakeSnapshotAndLogSplitScanner scanner = createScanner(BUCKET, 10L, 20L)) { + assertThatThrownBy(() -> scanner.pollBatch(TIMEOUT)).isSameAs(failure); + } + } + + private LakeSnapshotAndLogSplitScanner createScanner( + TableBucket bucket, long startingOffset, long stoppingOffset) { + return new LakeSnapshotAndLogSplitScanner( + table, lakeSource, null, bucket, startingOffset, stoppingOffset, null); + } + + private static ScanRecords finishedRecords(TableBucket bucket, ScanRecord... records) { + return new ScanRecords(Collections.singletonMap(bucket, Arrays.asList(records))) { + @Override + public Set finishedBuckets() { + return Collections.singleton(bucket); + } + }; + } +} diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java index 1096465f33d..5a726b15367 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/BoundedLogScannerRetentionITCase.java @@ -24,6 +24,7 @@ import org.apache.fluss.client.table.writer.AppendWriter; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FetchException; import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.TableBucket; @@ -46,6 +47,7 @@ import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests bounded subscriptions after retention advances the server's earliest offset. */ @Execution(ExecutionMode.SAME_THREAD) @@ -71,9 +73,7 @@ void testEarliestBeyondStoppingOffset(boolean retainRecords) throws Exception { try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); Admin admin = connection.getAdmin()) { - admin.createDatabase(tablePath.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); - admin.createTable( tablePath, TableDescriptor.builder() @@ -86,7 +86,6 @@ void testEarliestBeyondStoppingOffset(boolean retainRecords) throws Exception { .get(); long tableId = admin.getTableInfo(tablePath).get().getTableId(); - TableBucket bucket = new TableBucket(tableId, 0); LogTablet logTablet = CLUSTER.waitAndGetLeaderReplica(bucket).getLogTablet(); @@ -127,6 +126,67 @@ void testEarliestBeyondStoppingOffset(boolean retainRecords) throws Exception { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testExplicitStartingOffsetBeforeLogStartFails(boolean useArrowPoll) throws Exception { + TablePath tablePath = + TablePath.of("bounded_retention", "explicit_offset_out_of_range_" + useArrowPoll); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + admin.createDatabase(tablePath.getDatabaseName(), DatabaseDescriptor.EMPTY, true).get(); + admin.createTable( + tablePath, + TableDescriptor.builder() + .schema(DATA1_SCHEMA) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_LOG_TTL, Duration.ofHours(1)) + .build(), + false) + .get(); + + long tableId = admin.getTableInfo(tablePath).get().getTableId(); + TableBucket bucket = new TableBucket(tableId, 0); + LogTablet logTablet = CLUSTER.waitAndGetLeaderReplica(bucket).getLogTablet(); + + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + appendExpiredRecords(writer); + retry( + Duration.ofSeconds(10), + () -> + assertThat(logTablet.getHighWatermark()) + .isEqualTo((long) EXPIRED_RECORDS)); + + // Move offsets [0, EXPIRED_RECORDS) into a closed segment and expire them. + logTablet.roll(Optional.empty()); + CLOCK.advanceTime(Duration.ofHours(2)); + logTablet.deleteExpiredSegments(); + + assertThat(logTablet.logStartOffset()).isEqualTo((long) EXPIRED_RECORDS); + assertThat(logTablet.localLogEndOffset()).isEqualTo((long) EXPIRED_RECORDS); + + try (LogScannerImpl scanner = (LogScannerImpl) table.newScan().createLogScanner()) { + + // Unlike EARLIEST_OFFSET, an explicit historical offset must not + // be silently advanced to the current log start. + scanner.subscribeBounded(0, 0L, EXPIRED_RECORDS + 5L); + if (useArrowPoll) { + assertThatThrownBy(() -> scanner.pollRecordBatch(Duration.ofSeconds(1))) + .isInstanceOf(FetchException.class) + .hasMessageContaining("offset 0") + .hasMessageContaining("out of range"); + } else { + assertThatThrownBy(() -> scanner.poll(Duration.ofSeconds(1))) + .isInstanceOf(FetchException.class) + .hasMessageContaining("offset 0") + .hasMessageContaining("out of range"); + } + } + } + } + } + private static void appendExpiredRecords(AppendWriter writer) { for (int i = 0; i < EXPIRED_RECORDS; i++) { writer.append(row(i, "expired")); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java index 430fd1f1fe1..d2107fbc132 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java @@ -169,9 +169,9 @@ void testResolvedBoundedStartingOffset( assertThat(result.records(tb)).isEmpty(); assertThat(result.hasProgress()).isEqualTo(expectedProgress); if (finished) { - assertThat(result.finishedBuckets().contains(tb)); + assertThat(result.finishedBuckets().contains(tb)).isTrue(); } else { - assertThat(result.finishedBuckets().isEmpty()); + assertThat(result.finishedBuckets().isEmpty()).isTrue(); } if (expectedProgress) { assertThat(result.consumedUpToOffset(tb)).isEqualTo(expectedOffset); @@ -190,9 +190,9 @@ void testResolvedBoundedStartingOffset( assertThat(result.records(tb)).isEmpty(); assertThat(result.hasProgress()).isEqualTo(expectedProgress); if (finished) { - assertThat(result.finishedBuckets().contains(tb)); + assertThat(result.finishedBuckets().contains(tb)).isTrue(); } else { - assertThat(result.finishedBuckets().isEmpty()); + assertThat(result.finishedBuckets().isEmpty()).isTrue(); } if (expectedProgress) { assertThat(result.consumedUpToOffset(tb)).isEqualTo(expectedOffset); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index b5c041fdc2d..0481a0e4473 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -133,11 +133,13 @@ import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.fluss.utils.concurrent.Scheduler; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; + import java.io.File; import java.io.IOException; import java.nio.file.DirectoryNotEmptyException; diff --git a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java index d29f80b0656..a6f3ff9c5d9 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/utils/ServerRpcMessageUtilsTest.java @@ -55,6 +55,7 @@ import org.apache.fluss.server.metadata.PartitionMetadata; import org.apache.fluss.server.metadata.TableMetadata; import org.apache.fluss.utils.FlussPaths; + import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; From f054ef005ecb296d822f4d06200fa7ee78cc79ee Mon Sep 17 00:00:00 2001 From: Shangqing Yang Date: Fri, 16 Oct 2026 15:43:35 -0400 Subject: [PATCH 4/4] [client] Preserve compatibility for bounded EARLIEST scans --- .../batch/LakeSnapshotAndLogSplitScanner.java | 61 +++- .../table/scanner/log/ArrowScanRecords.java | 12 +- .../table/scanner/log/LogScannerStatus.java | 2 + .../client/table/scanner/log/ScanRecords.java | 13 +- .../LakeSnapshotAndLogSplitScannerTest.java | 293 +++++++++++++++++- .../log/LogFetchCollectorErrorTest.java | 5 +- .../scanner/log/LogFetchCollectorTest.java | 2 +- .../table/scanner/log/LogFetcherTest.java | 3 +- .../UnsupportedBoundedEarliestException.java | 2 +- fluss-rust/crates/fluss/proto/FlussApi.proto | 6 +- 10 files changed, 379 insertions(+), 20 deletions(-) diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java index 17e30ac0ea8..4d0ff6ebc0b 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScanner.java @@ -23,6 +23,7 @@ import org.apache.fluss.client.table.scanner.SortMergeReader; import org.apache.fluss.client.table.scanner.log.LogScanner; import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.exception.UnsupportedBoundedEarliestException; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.source.LakeSplit; import org.apache.fluss.lake.source.RecordReader; @@ -66,7 +67,11 @@ public class LakeSnapshotAndLogSplitScanner implements BatchScanner { private final LogScanner logScanner; private final TableBucket tableBucket; + private final long startingOffset; + private final long stoppingOffset; + private boolean logScanFinished; + private boolean legacyLogScan; private SortMergeReader currentSortMergeReader; @@ -82,6 +87,8 @@ public LakeSnapshotAndLogSplitScanner( this.lakeSplits = lakeSplits; this.lakeSource = lakeSource; this.tableBucket = tableBucket; + this.startingOffset = startingOffset; + this.stoppingOffset = stoppingOffset; ProjectionPlan projectionPlan = ProjectionPlan.create( table.getTableInfo().getRowType().getFieldCount(), @@ -182,8 +189,29 @@ public boolean requireSortedRecords() { } private void pollLogRecords(Duration timeout) { - ScanRecords scanRecords = logScanner.poll(timeout); + ScanRecords scanRecords; + try { + scanRecords = logScanner.poll(timeout); + } catch (UnsupportedBoundedEarliestException e) { + if (legacyLogScan || startingOffset != LogScanner.EARLIEST_OFFSET) { + throw e; + } + // An older server cannot report the resolved EARLIEST offset. + // Fall back to an unbounded subscription for older servers. + // The next poll will use the legacy stopping-offset checks below. + subscribeLegacy(); + legacyLogScan = true; + return; + } + for (ScanRecord scanRecord : scanRecords) { + long logOffset = scanRecord.logOffset(); + // The legacy scanner does not enforce the exclusive stopping + // offset, so records beyond the boundary must be discarded. + if (legacyLogScan && logOffset >= stoppingOffset) { + logScanFinished = true; + break; + } boolean isDelete = scanRecord.getChangeType() == ChangeType.DELETE || scanRecord.getChangeType() == ChangeType.UPDATE_BEFORE; @@ -192,10 +220,35 @@ private void pollLogRecords(Duration timeout) { InternalRow keyRow = keyValueRow.keyRow(); // upsert the key value row logRows.put(keyRow, keyValueRow); + // In legacy mode, reaching the last included record completes + // the scan. The stopping offset itself is exclusive. + if (legacyLogScan && logOffset >= stoppingOffset - 1) { + logScanFinished = true; + break; + } } - // Completion may accompany the final records or arrive without any records. - if (scanRecords.finishedBuckets().contains(tableBucket)) { - logScanFinished = true; + + if (legacyLogScan) { + // A fetch may advance its consumed position without returning + // the final record, for example after filtering. + Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); + if (consumedUpToOffset != null && consumedUpToOffset >= stoppingOffset) { + logScanFinished = true; + } + } else { + // Completion may accompany the final records or arrive without any records. + if (scanRecords.finishedBuckets().contains(tableBucket)) { + logScanFinished = true; + } + } + } + + private void subscribeLegacy() { + if (tableBucket.getPartitionId() != null) { + logScanner.subscribe( + tableBucket.getPartitionId(), tableBucket.getBucket(), startingOffset); + } else { + logScanner.subscribe(tableBucket.getBucket(), startingOffset); } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java index 3e82d0267c2..11c1bccd0f7 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ArrowScanRecords.java @@ -61,7 +61,17 @@ public ArrowScanRecords( this(records, consumedUpToOffsets, Collections.emptySet()); } - ArrowScanRecords( + /** + * Creates Arrow scan records with consumed offsets and bounded scan completions. + * + *

A completion event may accompany the final batches for the same bucket. + * + * @param records the Arrow batches grouped by table bucket + * @param consumedUpToOffsets the exclusive consumed offsets for each bucket + * @param finishedBuckets the buckets whose bounded subscriptions finished in this poll + */ + @Internal + public ArrowScanRecords( Map> records, Map consumedUpToOffsets, Set finishedBuckets) { diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java index 821b48769ed..0bd7e6e8008 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerStatus.java @@ -18,6 +18,7 @@ package org.apache.fluss.client.table.scanner.log; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.utils.log.FairBucketStatusMap; @@ -177,6 +178,7 @@ synchronized boolean hasReachedStoppingOffset(TableBucket tableBucket) { return bucketScanStatus != null && bucketScanStatus.hasReachedStoppingOffset(); } + @VisibleForTesting synchronized boolean hasPendingFinishedBuckets() { return !pendingFinishedBuckets.isEmpty(); } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java index 8870d5bdd00..2a161648fb0 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/ScanRecords.java @@ -17,6 +17,7 @@ package org.apache.fluss.client.table.scanner.log; +import org.apache.fluss.annotation.Internal; import org.apache.fluss.annotation.PublicEvolving; import org.apache.fluss.client.table.scanner.ScanRecord; import org.apache.fluss.metadata.TableBucket; @@ -60,7 +61,17 @@ public ScanRecords( this(records, consumedUpToOffsets, Collections.emptySet()); } - ScanRecords( + /** + * Creates scan records with consumed offsets and bounded scan completions. + * + *

A completion event may accompany the final records for the same bucket. + * + * @param records the records grouped by table bucket + * @param consumedUpToOffsets the exclusive consumed offsets for each bucket + * @param finishedBuckets the buckets whose bounded subscriptions finished in this poll + */ + @Internal + public ScanRecords( Map> records, Map consumedUpToOffsets, Set finishedBuckets) { diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java index 180d4e6d07b..14bd4f0716f 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/LakeSnapshotAndLogSplitScannerTest.java @@ -17,20 +17,32 @@ package org.apache.fluss.client.table.scanner.batch; +import org.apache.fluss.client.metadata.TestingMetadataUpdater; import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.RemoteFileDownloader; import org.apache.fluss.client.table.scanner.Scan; import org.apache.fluss.client.table.scanner.ScanRecord; import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.LogScannerImpl; import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FetchException; +import org.apache.fluss.exception.UnsupportedBoundedEarliestException; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.source.LakeSplit; import org.apache.fluss.lake.source.SortedRecordReader; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalRow; +import org.apache.fluss.rpc.entity.FetchLogResultForBucket; +import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.FetchLogResponse; +import org.apache.fluss.rpc.metrics.TestingClientMetricGroup; +import org.apache.fluss.server.entity.FetchReqInfo; +import org.apache.fluss.server.tablet.TestTabletServerGateway; import org.apache.fluss.utils.CloseableIterator; import org.junit.jupiter.api.BeforeEach; @@ -40,17 +52,35 @@ import org.junit.jupiter.params.provider.ValueSource; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.Set; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import static org.apache.fluss.client.metadata.TestingMetadataUpdater.NODE1; +import static org.apache.fluss.client.metadata.TestingMetadataUpdater.NODE2; +import static org.apache.fluss.client.metadata.TestingMetadataUpdater.NODE3; +import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; +import static org.apache.fluss.metadata.LogFormat.ARROW; +import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getFetchLogData; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeFetchLogResponse; +import static org.apache.fluss.testutils.DataTestUtils.createBasicMemoryLogRecords; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -181,6 +211,222 @@ void testMissingLogRangeErrorIsPropagated() throws Exception { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFallbackToLegacyScanForOldServer(boolean partitioned) throws Exception { + TableBucket bucket = partitioned ? new TableBucket(DATA1_TABLE_ID_PK, 7L, 0) : BUCKET; + UnsupportedBoundedEarliestException failure = + new UnsupportedBoundedEarliestException( + "Server does not report resolved_earliest_offset"); + ScanRecords legacyRecords = + new ScanRecords( + Collections.singletonMap( + bucket, + Arrays.asList( + new ScanRecord( + 10, 0, ChangeType.DELETE, GenericRow.of(1, null)), + new ScanRecord( + 11, 0, ChangeType.INSERT, GenericRow.of(2, null)), + new ScanRecord( + 12, 0, ChangeType.INSERT, GenericRow.of(3, null)))), + Collections.singletonMap(bucket, 13L)); + when(logScanner.poll(TIMEOUT)).thenThrow(failure).thenReturn(legacyRecords); + + try (LakeSnapshotAndLogSplitScanner scanner = + createScanner(bucket, LogScanner.EARLIEST_OFFSET, 12L)) { + if (partitioned) { + verify(logScanner).subscribeBounded(7L, 0, LogScanner.EARLIEST_OFFSET, 12L); + } else { + verify(logScanner).subscribeBounded(0, LogScanner.EARLIEST_OFFSET, 12L); + } + // The first poll detects the old server and switches subscription. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + if (partitioned) { + verify(logScanner).subscribe(7L, 0, LogScanner.EARLIEST_OFFSET); + } else { + verify(logScanner).subscribe(0, LogScanner.EARLIEST_OFFSET); + } + // The second poll consumes records from the legacy subscription. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + // The third poll returns the merged snapshot. + try (CloseableIterator rows = scanner.pollBatch(TIMEOUT)) { + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getInt(0)).isEqualTo(2); + assertThat(rows.hasNext()).isFalse(); + } + assertThat(scanner.pollBatch(TIMEOUT)).isNull(); + verify(logScanner, times(2)).poll(TIMEOUT); + } + } + + @Test + void testLegacyFallbackExcludesRecordsAtStoppingOffset() throws Exception { + ScanRecords legacyRecords = + new ScanRecords( + Collections.singletonMap( + BUCKET, + Arrays.asList( + new ScanRecord( + 10, 0, ChangeType.INSERT, GenericRow.of(2, null)), + new ScanRecord( + 12, 0, ChangeType.INSERT, GenericRow.of(3, null)))), + Collections.singletonMap(BUCKET, 13L)); + when(logScanner.poll(TIMEOUT)) + .thenThrow( + new UnsupportedBoundedEarliestException("Missing resolved_earliest_offset")) + .thenReturn(legacyRecords); + try (LakeSnapshotAndLogSplitScanner scanner = + createScanner(BUCKET, LogScanner.EARLIEST_OFFSET, 12L)) { + // Switch from bounded to legacy subscription. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + // Read log records. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + // Lake contains id=1, and the valid log contains id=2. + // Offset 12 / id=3 must be excluded. + try (CloseableIterator rows = scanner.pollBatch(TIMEOUT)) { + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getInt(0)).isEqualTo(1); + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getInt(0)).isEqualTo(2); + assertThat(rows.hasNext()).isFalse(); + } + assertThat(scanner.pollBatch(TIMEOUT)).isNull(); + } + } + + @Test + void testLegacyFallbackCompletesOnOffsetProgress() throws Exception { + ScanRecords progress = + new ScanRecords(Collections.emptyMap(), Collections.singletonMap(BUCKET, 25L)); + when(logScanner.poll(TIMEOUT)) + .thenThrow( + new UnsupportedBoundedEarliestException("Missing resolved_earliest_offset")) + .thenReturn(progress); + try (LakeSnapshotAndLogSplitScanner scanner = + createScanner(BUCKET, LogScanner.EARLIEST_OFFSET, 20L)) { + // First poll switches to the legacy subscription. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + verify(logScanner).subscribe(0, LogScanner.EARLIEST_OFFSET); + // Second poll sees progress beyond the stopping offset. + // There are no actual log records. + try (CloseableIterator pending = scanner.pollBatch(TIMEOUT)) { + assertThat(pending.hasNext()).isFalse(); + } + // The scanner must now start merging the lake snapshot. + try (CloseableIterator rows = scanner.pollBatch(TIMEOUT)) { + assertThat(rows.hasNext()).isTrue(); + assertThat(rows.next().getInt(0)).isEqualTo(1); + assertThat(rows.hasNext()).isFalse(); + } + assertThat(scanner.pollBatch(TIMEOUT)).isNull(); + verify(logScanner, times(2)).poll(TIMEOUT); + } + } + + @Test + void testOtherUnsupportedOperationDoesNotTriggerFallback() throws Exception { + UnsupportedOperationException failure = + new UnsupportedOperationException("Unrelated scanner error"); + when(logScanner.poll(TIMEOUT)).thenThrow(failure); + try (LakeSnapshotAndLogSplitScanner scanner = + createScanner(BUCKET, LogScanner.EARLIEST_OFFSET, 20L)) { + assertThatThrownBy(() -> scanner.pollBatch(TIMEOUT)).isSameAs(failure); + verify(logScanner, never()).subscribe(0, LogScanner.EARLIEST_OFFSET); + } + } + + @Test + void testOldServerFallbackWithRealLogScanner() throws Exception { + // Log records: + // 10: DELETE id=1 + // 11: INSERT id=2 + // 12: INSERT id=3 (exclusive boundary, must be ignored) + MemoryLogRecords logRecords = + createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 10L, + 0L, + CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + Arrays.asList(ChangeType.DELETE, ChangeType.INSERT, ChangeType.INSERT), + Arrays.asList( + new Object[] {1, null}, + new Object[] {2, null}, + new Object[] {3, null}), + ARROW, + DEFAULT_COMPRESSION); + + LegacyTabletServerGateway gateway = new LegacyTabletServerGateway(logRecords); + + Configuration conf = new Configuration(); + TestingMetadataUpdater metadataUpdater = + new TestingMetadataUpdater( + TestingMetadataUpdater.COORDINATOR, + Arrays.asList(NODE1, NODE2, NODE3), + Collections.singletonMap( + DATA1_TABLE_INFO_PK.getTablePath(), DATA1_TABLE_INFO_PK), + Collections.singletonMap(1, gateway), + conf); + + try (RemoteFileDownloader downloader = new RemoteFileDownloader(1); + LogScannerImpl realLogScanner = + new LogScannerImpl( + conf, + DATA1_TABLE_INFO_PK, + metadataUpdater, + TestingClientMetricGroup.newInstance(), + downloader, + null, + TEST_SCHEMA_GETTER, + null)) { + + // Replace only the mocked scanner with the real implementation. + Scan scan = table.newScan(); + when(scan.createLogScanner()).thenReturn(realLogScanner); + + try (LakeSnapshotAndLogSplitScanner scanner = + createScanner(BUCKET, LogScanner.EARLIEST_OFFSET, 12L)) { + List mergedKeys = new ArrayList<>(); + boolean completed = false; + for (int attempt = 0; attempt < 20; attempt++) { + CloseableIterator rows = scanner.pollBatch(Duration.ofMillis(200)); + if (rows == null) { + completed = true; + break; + } + try (CloseableIterator result = rows) { + while (result.hasNext()) { + mergedKeys.add(result.next().getInt(0)); + } + } + } + assertThat(completed).isTrue(); + // The lake snapshot contains id=1, but offset 10 deletes it. + // Offset 11 inserts id=2. + // Offset 12 is excluded by the exclusive stopping offset. + assertThat(mergedKeys).containsExactly(2); + // The first request uses bounded EARLIEST. + // After the missing field is detected, the legacy subscription + // issues another EARLIEST request. + assertThat(gateway.requestedOffsets).hasSizeGreaterThanOrEqualTo(2); + assertThat(gateway.requestedOffsets.subList(0, 2)) + .containsExactly(LogScanner.EARLIEST_OFFSET, LogScanner.EARLIEST_OFFSET); + } + } + } + private LakeSnapshotAndLogSplitScanner createScanner( TableBucket bucket, long startingOffset, long stoppingOffset) { return new LakeSnapshotAndLogSplitScanner( @@ -188,11 +434,46 @@ private LakeSnapshotAndLogSplitScanner createScanner( } private static ScanRecords finishedRecords(TableBucket bucket, ScanRecord... records) { - return new ScanRecords(Collections.singletonMap(bucket, Arrays.asList(records))) { - @Override - public Set finishedBuckets() { - return Collections.singleton(bucket); + return new ScanRecords( + Collections.singletonMap(bucket, Arrays.asList(records)), + Collections.emptyMap(), + Collections.singleton(bucket)); + } + + private static final class LegacyTabletServerGateway extends TestTabletServerGateway { + private final MemoryLogRecords logRecords; + private final List requestedOffsets = new ArrayList<>(); + + private LegacyTabletServerGateway(MemoryLogRecords logRecords) { + super(false, Collections.emptySet()); + this.logRecords = logRecords; + } + + @Override + public CompletableFuture fetchLog(FetchLogRequest request) { + Map requests = getFetchLogData(request); + assertThat(requests).hasSize(1); + Map.Entry entry = requests.entrySet().iterator().next(); + TableBucket bucket = entry.getKey(); + long offset = entry.getValue().getFetchOffset(); + requestedOffsets.add(offset); + FetchLogResultForBucket result; + if (requestedOffsets.size() == 1) { + // Simulate an old server response: + // no resolved_earliest_offset field. + result = + FetchLogResultForBucket.records( + bucket, MemoryLogRecords.EMPTY, 13L, -1L, -1L); + } else if (requestedOffsets.size() == 2) { + // The legacy unbounded subscription should resume fetching. + result = FetchLogResultForBucket.records(bucket, logRecords, 13L, -1L, -1L); + } else { + result = + FetchLogResultForBucket.records( + bucket, MemoryLogRecords.EMPTY, 13L, -1L, -1L); } - }; + return CompletableFuture.completedFuture( + makeFetchLogResponse(Collections.singletonMap(bucket, result))); + } } } diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java index eea594ee48d..3afd23c6bac 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorErrorTest.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.FetchException; +import org.apache.fluss.exception.UnsupportedBoundedEarliestException; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ArrowBatchData; @@ -424,7 +425,7 @@ void testAsyncBufferFailureDoesNotInterruptFailedFetchCleanup() throws Exception // asynchronous remote-download failure is injected immediately before // the failed fetch is removed from the buffer. assertThatThrownBy(() -> collector.collectFetch(testBuffer)) - .isInstanceOf(UnsupportedOperationException.class) + .isInstanceOf(UnsupportedBoundedEarliestException.class) .hasMessageContaining("resolved_earliest_offset"); // The failed fetch must always be drained. @@ -476,7 +477,7 @@ void drain() { AbstractLogFetchCollector collector = collector(false); Throwable thrown = catchThrowable(() -> collector.collectFetch(buffer)); assertThat(thrown) - .isInstanceOf(UnsupportedOperationException.class) + .isInstanceOf(UnsupportedBoundedEarliestException.class) .hasMessageContaining("resolved_earliest_offset"); assertThat(thrown.getSuppressed()).containsExactly(cleanupFailure); // super.drain() ran before the synthetic cleanup failure. diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java index d2107fbc132..9d11d04c4a2 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java @@ -391,7 +391,7 @@ void testBoundedEarliestWithoutResolutionDrainsNonEmptyFetch() throws Exception logFetchBuffer.add(fetch); assertThatThrownBy(() -> logFetchCollector.collectFetch(logFetchBuffer)) - .isInstanceOf(UnsupportedOperationException.class) + .isInstanceOf(UnsupportedBoundedEarliestException.class) .hasMessageContaining("resolved_earliest_offset"); assertThat(fetch.isConsumed()).isTrue(); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java index 1bc3d7ba9b6..cfb80771861 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java @@ -28,6 +28,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FetchException; import org.apache.fluss.exception.NotLeaderOrFollowerException; +import org.apache.fluss.exception.UnsupportedBoundedEarliestException; import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.SchemaInfo; @@ -218,7 +219,7 @@ void testBoundedEmptyEarliestResponse(long resolvedOffset) { if (resolvedOffset == FetchLogResultForBucket.NO_RESOLVED_EARLIEST_OFFSET) { assertThatThrownBy(logFetcher::collectFetch) - .isInstanceOf(UnsupportedOperationException.class) + .isInstanceOf(UnsupportedBoundedEarliestException.class) .hasMessageContaining("resolved_earliest_offset"); assertThat(logFetcher.hasAvailableFetches()).isFalse(); diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java b/fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java index 197485ed3ec..9ec01958500 100644 --- a/fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java +++ b/fluss-common/src/main/java/org/apache/fluss/exception/UnsupportedBoundedEarliestException.java @@ -25,7 +25,7 @@ * physical offset to which EARLIEST was resolved. */ @PublicEvolving -public class UnsupportedBoundedEarliestException extends UnsupportedOperationException { +public class UnsupportedBoundedEarliestException extends FlussRuntimeException { private static final long serialVersionUID = 1L; public UnsupportedBoundedEarliestException(String message) { diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto index 74f4664240d..f9c19ea32a8 100644 --- a/fluss-rust/crates/fluss/proto/FlussApi.proto +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -1002,10 +1002,10 @@ message PbFetchLogRespForBucket { // The safe local log retention boundary confirmed by a committed KV snapshot. This is only // returned for KV follower fetches and is distinct from the physical log_start_offset. optional int64 min_retain_offset = 10; - // The actual offset used for this fetch after resolving an EARLIEST request. + // The log start offset to which an EARLIEST_OFFSET request was resolved. // This field is present only when the request used EARLIEST_OFFSET. - // Its presence also allows a new client to distinguish a server that supports - // EARLIEST resolution from an older server that does not expose the resolved offset. + // Its presence also allows a client to distinguish a server that reports the + // resolved offset from an older server that does not. optional int64 resolved_earliest_offset = 11; }