diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java index 3438f08b02d1..27d6e52ad72a 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java @@ -70,6 +70,7 @@ final class ResumableUploadChunkCoordinator { private final String uploadUrl; private final RewindableStreamBuffer buffer; private final ApiCallContext callContext; + private final ResumableUploadProgressTracker progressTracker; private final SettableApiFuture result = SettableApiFuture.create(); private volatile @Nullable ApiFuture currentChunkFuture; @@ -79,7 +80,8 @@ final class ResumableUploadChunkCoordinator { String uploadUrl, InputStream payload, int chunkSize, - ApiCallContext callContext) { + ApiCallContext callContext, + ResumableUploadProgressTracker progressTracker) { this.uploadChunkCallable = checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null"); this.queryStatusCallable = @@ -87,6 +89,7 @@ final class ResumableUploadChunkCoordinator { this.uploadUrl = checkNotNull(uploadUrl, "uploadUrl must not be null"); checkNotNull(payload, "payload must not be null"); this.callContext = checkNotNull(callContext, "callContext must not be null"); + this.progressTracker = checkNotNull(progressTracker, "progressTracker must not be null"); this.buffer = new RewindableStreamBuffer(payload, chunkSize, uploadUrl); } @@ -136,7 +139,7 @@ public void onSuccess(ChunkUploadResponse response) { return; } if (response.getUploadStatus() == ResumableUploadStatus.UNKNOWN) { - recover(); + recover(null); } else { onChunkUploaded(response); } @@ -150,7 +153,7 @@ public void onFailure(Throwable t) { Category category = ResumableUploadErrorClassifier.classify(t, ResumableUploadCommand.UPLOAD); if (category == Category.RECOVERABLE) { - recover(); + recover(t); } else { // Category.TRANSIENT errors reaching here have already exhausted their retry budget // in the underlying RetryingCallable and become fatal per protocol specification. @@ -164,7 +167,14 @@ public void onFailure(Throwable t) { } } - private void recover() { + /** + * Queries the session for the server's committed offset and resumes transmission from it. + * + * @param cause the error that triggered recovery, or null if the server acknowledged the chunk + * without an upload status header + */ + private void recover(@Nullable Throwable cause) { + progressTracker.onRecovering(); try { // Dispatch the query status call and register the in-flight future for cancellation. ApiFuture> queryFuture = @@ -228,22 +238,29 @@ private void handleQueryResponse(QueryStatusResponse queryResponse) "Incomplete query status response did not include a committed offset for upload URL: " + uploadUrl); } + progressTracker.onOffsetReceived(committedOffset); buffer.realignTo(committedOffset); dispatchCurrentChunk(); } private void onChunkUploaded(ChunkUploadResponse response) { - long nextOffset = buffer.getBufferBaseOffset() + buffer.getPayloadLength(); - if (response.getUploadStatus() == ResumableUploadStatus.FINAL) { - result.set(response.getResponse()); - } else if (buffer.isFinal()) { - result.setException( - new IllegalStateException( - "Upload stream ended and final chunk was transmitted, but server returned" - + " incomplete status for upload URL: " - + uploadUrl)); - } else { - chunkExecutor.execute(() -> transmitChunk(nextOffset)); + try { + long nextOffset = buffer.getBufferBaseOffset() + buffer.getPayloadLength(); + if (response.getUploadStatus() == ResumableUploadStatus.FINAL) { + progressTracker.onChunkUploaded(nextOffset); + result.set(response.getResponse()); + } else if (buffer.isFinal()) { + result.setException( + new IllegalStateException( + "Upload stream ended and final chunk was transmitted, but server returned" + + " incomplete status for upload URL: " + + uploadUrl)); + } else { + progressTracker.onChunkUploaded(nextOffset); + chunkExecutor.execute(() -> transmitChunk(nextOffset)); + } + } catch (Throwable t) { + result.setException(t); } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFuture.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFuture.java index 4282ef89fa33..5dfbedd95aaa 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFuture.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFuture.java @@ -31,6 +31,9 @@ import com.google.api.core.ApiFuture; import com.google.api.core.BetaApi; +import com.google.api.gax.resumable.ResumableUploadProgress; +import com.google.api.gax.resumable.ResumableUploadProgressListener; +import java.util.concurrent.Executor; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -48,4 +51,18 @@ public interface ResumableUploadFuture extends ApiFuture { /** Returns the upload session URL, or {@code null} if session initiation is in progress. */ @Nullable String getUploadSessionUrl(); + + /** + * Registers a listener to receive progress and state transition notifications for this upload. + * + *

A snapshot of the current upload status is dispatched to the listener immediately upon + * subscription on the provided executor. Subsequent status updates are delivered in order. + * + * @param listener callback listener to receive progress notifications + * @param executor executor on which the listener callbacks are dispatched + */ + void addProgressListener(ResumableUploadProgressListener listener, Executor executor); + + /** Returns the current progress snapshot of the upload session. */ + ResumableUploadProgress getStatus(); } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java index 0d046edff2c7..0a33de08d88b 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java @@ -41,6 +41,8 @@ import com.google.api.gax.resumable.ChunkUploadResponse; import com.google.api.gax.resumable.QueryStatusRequest; import com.google.api.gax.resumable.QueryStatusResponse; +import com.google.api.gax.resumable.ResumableUploadProgress; +import com.google.api.gax.resumable.ResumableUploadProgressListener; import com.google.api.gax.resumable.ResumableUploadSession; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.concurrent.GuardedBy; @@ -58,10 +60,11 @@ import org.jspecify.annotations.Nullable; /** - * Implementation of {@link ResumableUploadFuture} responsible for the end-to-end management of a - * resumable upload session. + * Baseline implementation of {@link ResumableUploadFuture}. * - * @param the type of the final response message returned once the upload completes + *

Coordinates session initiation, delegates chunk streaming to {@link + * ResumableUploadChunkCoordinator}, and manages the lifecycle of the user-provided payload {@link + * InputStream}. */ @NullMarked final class ResumableUploadFutureImpl implements ResumableUploadFuture { @@ -79,10 +82,15 @@ final class ResumableUploadFutureImpl implements ResumableUploadFutur private final ResumableUploadCallSettings settings; private final ApiCallContext callContext; private final ScheduledExecutorService executor; + private final ResumableUploadProgressTracker progressTracker = + new ResumableUploadProgressTracker(); private final SettableApiFuture resultFuture = SettableApiFuture.create(); private volatile @Nullable String uploadSessionUrl; + @GuardedBy("lock") + private boolean done; + @GuardedBy("lock") private @Nullable ApiFuture inFlightFuture; @@ -147,10 +155,16 @@ private void start() { new ApiFutureCallback() { @Override public void onSuccess(ResumableUploadSession session) { + synchronized (lock) { + if (done) { + return; + } + uploadSessionUrl = session.getUploadUrl(); + } + progressTracker.onStarted(uploadSessionUrl); if (resultFuture.isDone()) { return; } - uploadSessionUrl = session.getUploadUrl(); ResumableUploadChunkCoordinator coordinator = new ResumableUploadChunkCoordinator<>( uploadChunkCallable, @@ -158,7 +172,8 @@ public void onSuccess(ResumableUploadSession session) { uploadSessionUrl, payload, settings.getChunkSize(), - callContext); + callContext, + progressTracker); ApiFuture uploadFuture; try { uploadFuture = coordinator.start(); @@ -168,7 +183,7 @@ public void onSuccess(ResumableUploadSession session) { } boolean alreadyDone = false; synchronized (lock) { - if (resultFuture.isDone()) { + if (done) { alreadyDone = true; } else { inFlightFuture = uploadFuture; @@ -221,8 +236,13 @@ private void onTimeout() { private void succeed(@Nullable ResponseT result) { synchronized (lock) { + if (done) { + return; + } + done = true; inFlightFuture = null; } + progressTracker.onFinalized(progressTracker.getStatus().getBytesUploaded()); closePayload(); resultFuture.set(result); } @@ -230,12 +250,17 @@ private void succeed(@Nullable ResponseT result) { private void fail(Throwable t) { ApiFuture inFlight; synchronized (lock) { + if (done) { + return; + } + done = true; inFlight = this.inFlightFuture; this.inFlightFuture = null; } if (inFlight != null) { inFlight.cancel(true); } + progressTracker.onFailed(); closePayload(); resultFuture.setException(t); } @@ -253,6 +278,16 @@ private void closePayload() { return uploadSessionUrl; } + @Override + public void addProgressListener(ResumableUploadProgressListener listener, Executor executor) { + progressTracker.addListener(listener, executor); + } + + @Override + public ResumableUploadProgress getStatus() { + return progressTracker.getStatus(); + } + @Override public void addListener(Runnable listener, Executor executor) { resultFuture.addListener(listener, executor); @@ -263,6 +298,10 @@ public boolean cancel(boolean mayInterruptIfRunning) { boolean cancelled; ApiFuture inFlight; synchronized (lock) { + if (done) { + return false; + } + done = true; cancelled = resultFuture.cancel(mayInterruptIfRunning); inFlight = this.inFlightFuture; this.inFlightFuture = null; @@ -270,6 +309,7 @@ public boolean cancel(boolean mayInterruptIfRunning) { if (inFlight != null) { inFlight.cancel(mayInterruptIfRunning); } + progressTracker.onFailed(); closePayload(); return cancelled; } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java index c0f6f1b0f4ba..7b5d9bce619c 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java @@ -49,6 +49,7 @@ import com.google.api.gax.resumable.QueryStatusRequest; import com.google.api.gax.resumable.QueryStatusResponse; import com.google.api.gax.resumable.ResumableUploadClient; +import com.google.api.gax.resumable.ResumableUploadProgress; import com.google.api.gax.resumable.ResumableUploadSession; import com.google.api.gax.resumable.ResumableUploadStatus; import com.google.api.gax.rpc.testing.FakeCallContext; @@ -57,15 +58,22 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.CancellationException; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -83,6 +91,7 @@ class ResumableUploadCallableImplTest { private ResumableUploadCallSettings defaultSettings; private FakeCallContext callContext; private ClientContext clientContext; + private ExecutorService executor; private ResumableUploadCallableImpl callable; @BeforeEach @@ -100,9 +109,15 @@ void setUp() { defaultSettings = ResumableUploadCallSettings.newBuilder().setChunkSize(8).build(); callContext = FakeCallContext.createDefault(); clientContext = ClientContext.newBuilder().setDefaultCallContext(callContext).build(); + executor = Executors.newSingleThreadExecutor(); callable = new ResumableUploadCallableImpl<>(mockClient, defaultSettings, clientContext); } + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + @Test void testUploadCallable_singleChunk_happyPath() throws Exception { stubStartSession("https://upload.url/single"); @@ -951,6 +966,294 @@ void testGlobalTimeout_coversStartSessionTimeout() throws Exception { assertThat(hungStartFuture.isCancelled()).isTrue(); } + @Test + void testProgressListener_snapshotOnSubscribe_postsExactlyOneImmediateUpdate() throws Exception { + SettableApiFuture hungStartFuture = SettableApiFuture.create(); + when(mockStartCallable.futureCall(any(), any())).thenReturn(hungStartFuture); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + List statuses = new CopyOnWriteArrayList<>(); + CountDownLatch latch = new CountDownLatch(1); + future.addProgressListener( + status -> { + statuses.add(status); + latch.countDown(); + }, + executor); + + assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(statuses).hasSize(1); + ResumableUploadProgress snapshot = statuses.get(0); + assertThat(snapshot.getState()).isEqualTo(ResumableUploadProgress.State.STARTING); + assertThat(snapshot.getBytesUploaded()).isEqualTo(0L); + assertThat(snapshot.getUploadUrl()).isNull(); + } + + @Test + void testProgressListener_prescribedStateTransitions() throws Exception { + SettableApiFuture startFuture = SettableApiFuture.create(); + when(mockStartCallable.futureCall(any(), any())).thenReturn(startFuture); + + // 20 bytes with chunkSize = 8 -> 3 chunks: [0..8), [8..16), [16..20) + // Chunk 1 succeeds -> [0..8) + // Chunk 2 fails with Cat-2 400 + // Query succeeds -> committed offset = 8 + // Chunk 2 resend succeeds -> [8..16) + // Chunk 3 succeeds and finalizes -> [16..20) + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(ResumableUploadStatus.ACTIVE, null))) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(ResumableUploadStatus.ACTIVE, null))) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "done"))); + + when(mockQueryCallable.futureCall(any(QueryStatusRequest.class), any())) + .thenReturn( + ApiFutures.immediateFuture( + QueryStatusResponse.newBuilder() + .setCommittedOffset(8L) + .setUploadStatus(ResumableUploadStatus.ACTIVE) + .build())); + + List receivedStatuses = new CopyOnWriteArrayList<>(); + CountDownLatch finalizedLatch = new CountDownLatch(1); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("01234567890123456789"), null); + future.addProgressListener( + status -> { + receivedStatuses.add(status); + if (status.getState() == ResumableUploadProgress.State.FINALIZED) { + finalizedLatch.countDown(); + } + }, + executor); + + startFuture.set( + ResumableUploadSession.newBuilder() + .setUploadUrl("https://upload.url/progress-transitions") + .build()); + + assertThat(future.get()).isEqualTo("done"); + assertThat(finalizedLatch.await(5, TimeUnit.SECONDS)).isTrue(); + + List states = new ArrayList<>(); + for (ResumableUploadProgress s : receivedStatuses) { + states.add(s.getState()); + } + + assertThat(states) + .containsAtLeast( + ResumableUploadProgress.State.STARTED, + ResumableUploadProgress.State.UPLOADING, + ResumableUploadProgress.State.RECOVERING, + ResumableUploadProgress.State.OFFSET_RECEIVED, + ResumableUploadProgress.State.FINALIZED) + .inOrder(); + + long lastBytes = 0; + for (ResumableUploadProgress s : receivedStatuses) { + assertThat(s.getBytesUploaded()).isAtLeast(lastBytes); + lastBytes = s.getBytesUploaded(); + if (s.getState() != ResumableUploadProgress.State.STARTING) { + assertThat(s.getUploadUrl()).isEqualTo("https://upload.url/progress-transitions"); + } + } + assertThat(lastBytes).isEqualTo(20L); + } + + @Test + void testProgressListener_throwingListener_doesNotBreakUpload() throws Exception { + stubStartSession("https://upload.url/throwing-listener"); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "ok"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + future.addProgressListener( + status -> { + throw new RuntimeException("boom from listener"); + }, + executor); + + assertThat(future.get()).isEqualTo("ok"); + assertThat(future.isDone()).isTrue(); + } + + @Test + void testProgressListener_subscribingAfterCompletion_yieldsOneTerminalSnapshot() + throws Exception { + stubStartSession("https://upload.url/post-completion"); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "completed-ok"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + assertThat(future.get()).isEqualTo("completed-ok"); + + List postStatuses = new CopyOnWriteArrayList<>(); + CountDownLatch latch = new CountDownLatch(1); + future.addProgressListener( + status -> { + postStatuses.add(status); + latch.countDown(); + }, + executor); + + assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); + executor.submit(() -> {}).get(5, TimeUnit.SECONDS); + assertThat(postStatuses).hasSize(1); + ResumableUploadProgress snapshot = postStatuses.get(0); + assertThat(snapshot.getState()).isEqualTo(ResumableUploadProgress.State.FINALIZED); + assertThat(snapshot.getUploadUrl()).isEqualTo("https://upload.url/post-completion"); + assertThat(snapshot.getBytesUploaded()).isEqualTo(5L); + } + + @Test + void testProgressListener_subscribingAfterFailure_yieldsOneTerminalFailedSnapshot() + throws Exception { + when(mockStartCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(401, StatusCode.Code.UNAUTHENTICATED))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + assertThrows(ExecutionException.class, future::get); + + List postStatuses = new CopyOnWriteArrayList<>(); + CountDownLatch latch = new CountDownLatch(1); + future.addProgressListener( + status -> { + postStatuses.add(status); + latch.countDown(); + }, + executor); + + assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue(); + executor.submit(() -> {}).get(5, TimeUnit.SECONDS); + assertThat(postStatuses).hasSize(1); + ResumableUploadProgress snapshot = postStatuses.get(0); + assertThat(snapshot.getState()).isEqualTo(ResumableUploadProgress.State.FAILED); + } + + @Test + void testProgressListener_futureCancelFromInsideListenerBody_worksWithoutDeadlock() + throws Exception { + stubStartSession("https://upload.url/cancel-inside-listener"); + SettableApiFuture> hungChunk = SettableApiFuture.create(); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())).thenReturn(hungChunk); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + CountDownLatch cancelAttemptedLatch = new CountDownLatch(1); + future.addProgressListener( + status -> { + if (status.getState() == ResumableUploadProgress.State.STARTED) { + future.cancel(true); + cancelAttemptedLatch.countDown(); + } + }, + executor); + + assertThat(cancelAttemptedLatch.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(future.isCancelled()).isTrue(); + assertThat(hungChunk.isCancelled()).isTrue(); + } + + @Test + void testProgressListener_getStatus_reflectsCurrentState() throws Exception { + stubStartSession("https://upload.url/get-status"); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "ok"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + assertThat(future.get()).isEqualTo("ok"); + ResumableUploadProgress status = future.getStatus(); + assertThat(status.getState()).isEqualTo(ResumableUploadProgress.State.FINALIZED); + assertThat(status.getUploadUrl()).isEqualTo("https://upload.url/get-status"); + assertThat(status.getBytesUploaded()).isEqualTo(5L); + } + + @Test + void testProgressListener_orderingUnderConcurrency_pinsSequentialExecutor() throws Exception { + ExecutorService multiThreadedExecutor = Executors.newFixedThreadPool(8); + try { + stubStartSession("https://upload.url/concurrency-order"); + ChunkUploadResponse activeResponse = + ChunkUploadResponse.create(ResumableUploadStatus.ACTIVE, null); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn(ApiFutures.immediateFuture(activeResponse)) + .thenReturn(ApiFutures.immediateFuture(activeResponse)) + .thenReturn(ApiFutures.immediateFuture(activeResponse)) + .thenReturn(ApiFutures.immediateFuture(activeResponse)) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "finished"))); + + List events = new CopyOnWriteArrayList<>(); + AtomicInteger concurrentExecutions = new AtomicInteger(0); + AtomicBoolean concurrencyDetected = new AtomicBoolean(false); + CountDownLatch finalizedLatch = new CountDownLatch(1); + + ResumableUploadFuture future = + callable.futureCall( + "resource-path", streamOf("0123456789012345678901234567890123456789"), null); + + future.addProgressListener( + status -> { + int inProgress = concurrentExecutions.incrementAndGet(); + if (inProgress > 1) { + concurrencyDetected.set(true); + } + try { + Thread.sleep(10); + events.add(status); + if (status.getState() == ResumableUploadProgress.State.FINALIZED) { + finalizedLatch.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrentExecutions.decrementAndGet(); + } + }, + multiThreadedExecutor); + + assertThat(future.get(10, TimeUnit.SECONDS)).isEqualTo("finished"); + assertThat(finalizedLatch.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(concurrencyDetected.get()).isFalse(); + + long lastBytes = 0; + for (ResumableUploadProgress s : events) { + assertThat(s.getBytesUploaded()).isAtLeast(lastBytes); + lastBytes = s.getBytesUploaded(); + } + assertThat(lastBytes).isEqualTo(40L); + } finally { + multiThreadedExecutor.shutdownNow(); + } + } + private static class HttpStatusStatusCode implements StatusCode { private final int httpStatus; private final StatusCode.Code code;