From 1930517c958f71e9e66b5b46a31e8a76669a9bd7 Mon Sep 17 00:00:00 2001 From: sagnghos Date: Fri, 17 Jul 2026 12:11:47 +0000 Subject: [PATCH] chore(spanner): introduce commitRpcTimeout to connection properties This explicitly sets an operation timeout bound for Spanner transactions so that severe network blockholes do not lead to an infinite connection timeout at the YCSB or application layer when proxy mechanisms (like PGAdapter) are routing via EKS Load Balancers. --- .../google/cloud/spanner/SpannerOptions.java | 22 +++++++++++++++++++ .../spanner/connection/ConnectionOptions.java | 9 ++++++++ .../connection/ConnectionProperties.java | 8 +++++++ .../cloud/spanner/connection/SpannerPool.java | 7 ++++++ 4 files changed, 46 insertions(+) diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index 0a9c619b763d..d58eb406fb3f 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -269,6 +269,7 @@ static GcpChannelPoolOptions mergeWithDefaultChannelPoolOptions( private final DatabaseAdminStubSettings databaseAdminStubSettings; private final Duration partitionedDmlTimeout; private final Duration grpcKeepAliveTime; + private final Duration commitRpcTimeout; private final Duration grpcKeepAliveTimeout; private final boolean grpcGcpExtensionEnabled; private final GcpManagedChannelOptions grpcGcpOptions; @@ -925,6 +926,15 @@ protected SpannerOptions(Builder builder) { sessionLabels = builder.sessionLabels; try { String resolvedUniversalDomain = getResolvedUniverseDomain(); + if (builder.commitRpcTimeout != null) { + com.google.api.gax.retrying.RetrySettings currentCommitRetrySettings = builder.spannerStubSettingsBuilder.commitSettings().getRetrySettings(); + builder.spannerStubSettingsBuilder.commitSettings().setRetrySettings( + currentCommitRetrySettings.toBuilder() + .setInitialRpcTimeoutDuration(builder.commitRpcTimeout) + .setMaxRpcTimeoutDuration(builder.commitRpcTimeout) + .setTotalTimeoutDuration(builder.commitRpcTimeout) + .build()); + } spannerStubSettings = builder.spannerStubSettingsBuilder.setUniverseDomain(resolvedUniversalDomain).build(); instanceAdminStubSettings = @@ -942,6 +952,7 @@ protected SpannerOptions(Builder builder) { } partitionedDmlTimeout = builder.partitionedDmlTimeout; grpcKeepAliveTime = builder.grpcKeepAliveTime; + commitRpcTimeout = builder.commitRpcTimeout; grpcKeepAliveTimeout = builder.grpcKeepAliveTimeout; grpcGcpExtensionEnabled = builder.grpcGcpExtensionEnabled; grpcGcpOptions = builder.grpcGcpOptions; @@ -1322,6 +1333,7 @@ private static Builder prepareBuilder(Builder builder) { private boolean enableGrpcGcpOtelMetrics = SpannerOptions.environment.isEnableGrpcGcpOtelMetrics(); private Duration grpcKeepAliveTime = Duration.ofSeconds(120); + private Duration commitRpcTimeout = Duration.ofMinutes(60); private Duration grpcKeepAliveTimeout = Duration.ofSeconds(20); private CallCredentialsProvider callCredentialsProvider; private CloseableExecutorProvider asyncExecutorProvider; @@ -1437,6 +1449,7 @@ protected Builder() { this.defaultQueryOptions = options.defaultQueryOptions; this.callCredentialsProvider = options.callCredentialsProvider; this.grpcKeepAliveTime = options.grpcKeepAliveTime; + this.commitRpcTimeout = options.commitRpcTimeout; this.grpcKeepAliveTimeout = options.grpcKeepAliveTimeout; this.asyncExecutorProvider = options.asyncExecutorProvider; this.compressorName = options.compressorName; @@ -1704,6 +1717,11 @@ public Builder setPartitionedDmlTimeoutDuration(Duration timeout) { * Sets the keep-alive time for gRPC connections. The default is 120 seconds. Note that the * client-side keepalive time is clamped to a minimum of 10 seconds by gRPC. */ + public Builder setCommitRpcTimeout(Duration commitRpcTimeout) { + this.commitRpcTimeout = commitRpcTimeout; + return this; + } + public Builder setGrpcKeepAliveTime(Duration grpcKeepAliveTime) { Preconditions.checkNotNull(grpcKeepAliveTime, "grpcKeepAliveTime cannot be null"); Preconditions.checkArgument( @@ -2527,6 +2545,10 @@ public Duration getPartitionedDmlTimeoutDuration() { return partitionedDmlTimeout; } + public Duration getCommitRpcTimeout() { + return commitRpcTimeout; + } + public Duration getGrpcKeepAliveTime() { return grpcKeepAliveTime; } diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java index 00d616c53a6b..9b121a0241f7 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java @@ -43,6 +43,7 @@ import static com.google.cloud.spanner.connection.ConnectionProperties.ENDPOINT; import static com.google.cloud.spanner.connection.ConnectionProperties.GRPC_INTERCEPTOR_PROVIDER; import static com.google.cloud.spanner.connection.ConnectionProperties.GRPC_KEEPALIVE_TIME; +import static com.google.cloud.spanner.connection.ConnectionProperties.COMMIT_RPC_TIMEOUT; import static com.google.cloud.spanner.connection.ConnectionProperties.GRPC_KEEPALIVE_TIMEOUT; import static com.google.cloud.spanner.connection.ConnectionProperties.IS_EXPERIMENTAL_HOST; import static com.google.cloud.spanner.connection.ConnectionProperties.LENIENT; @@ -284,6 +285,9 @@ public class ConnectionOptions { /** Name of the 'grpcKeepAliveTime' connection property. */ public static final String GRPC_KEEPALIVE_TIME_PROPERTY_NAME = "grpcKeepAliveTime"; + /** Name of the 'commitRpcTimeout' connection property. */ + public static final String COMMIT_RPC_TIMEOUT_PROPERTY_NAME = "commitRpcTimeout"; + /** Name of the 'grpcKeepAliveTimeout' connection property. */ public static final String GRPC_KEEPALIVE_TIMEOUT_PROPERTY_NAME = "grpcKeepAliveTimeout"; @@ -1092,6 +1096,11 @@ public Integer getNumChannels() { } /** The gRPC keepalive time for this connection. */ + /** The commit RPC timeout for this connection. */ + public Duration getCommitRpcTimeout() { + return getInitialConnectionPropertyValue(COMMIT_RPC_TIMEOUT); + } + public Duration getGrpcKeepAliveTime() { return getInitialConnectionPropertyValue(GRPC_KEEPALIVE_TIME); } diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java index d501ab11b138..2f16ccdbc268 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java @@ -98,6 +98,7 @@ import static com.google.cloud.spanner.connection.ConnectionOptions.ENCODED_CREDENTIALS_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.ENDPOINT_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.GRPC_KEEPALIVE_TIMEOUT_PROPERTY_NAME; +import static com.google.cloud.spanner.connection.ConnectionOptions.COMMIT_RPC_TIMEOUT_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.GRPC_KEEPALIVE_TIME_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.IS_EXPERIMENTAL_HOST_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.KEEP_TRANSACTION_ALIVE_PROPERTY_NAME; @@ -255,6 +256,13 @@ public class ConnectionProperties { BOOLEANS, BooleanConverter.INSTANCE, Context.STARTUP); + static final ConnectionProperty COMMIT_RPC_TIMEOUT = + create( + COMMIT_RPC_TIMEOUT_PROPERTY_NAME, + "The default RPC timeout applied strictly to commit operations (e.g. '30s', '10000ms').", + null, + DurationConverter.INSTANCE, + Context.STARTUP); static final ConnectionProperty GRPC_KEEPALIVE_TIME = create( GRPC_KEEPALIVE_TIME_PROPERTY_NAME, diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java index 785d2c80cd3e..627362651ebf 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java @@ -153,6 +153,7 @@ static class SpannerPoolKey { private final String host; private final String projectId; private final Duration grpcKeepAliveTime; + final Duration commitRpcTimeout; private final Duration grpcKeepAliveTimeout; private final CredentialsKey credentialsKey; private final SessionPoolOptions sessionPoolOptions; @@ -226,6 +227,7 @@ private SpannerPoolKey(ConnectionOptions options) throws IOException { this.universeDomain = options.getUniverseDomain(); this.grpcInterceptorProvider = options.getGrpcInterceptorProviderName(); this.grpcKeepAliveTime = options.getGrpcKeepAliveTime(); + this.commitRpcTimeout = options.getCommitRpcTimeout(); this.grpcKeepAliveTimeout = options.getGrpcKeepAliveTimeout(); } @@ -266,6 +268,7 @@ public boolean equals(Object o) { && Objects.equals(this.universeDomain, other.universeDomain) && Objects.equals(this.grpcInterceptorProvider, other.grpcInterceptorProvider) && Objects.equals(this.grpcKeepAliveTime, other.grpcKeepAliveTime) + && Objects.equals(this.commitRpcTimeout, other.commitRpcTimeout) && Objects.equals(this.grpcKeepAliveTimeout, other.grpcKeepAliveTimeout); } @@ -301,6 +304,7 @@ public int hashCode() { this.universeDomain, this.grpcInterceptorProvider, this.grpcKeepAliveTime, + commitRpcTimeout, this.grpcKeepAliveTimeout); } } @@ -522,6 +526,9 @@ Spanner createSpanner(SpannerPoolKey key, ConnectionOptions options) { if (key.grpcKeepAliveTime != null) { builder.setGrpcKeepAliveTime(key.grpcKeepAliveTime); } + if (key.commitRpcTimeout != null) { + builder.setCommitRpcTimeout(key.commitRpcTimeout); + } if (key.grpcKeepAliveTimeout != null) { builder.setGrpcKeepAliveTimeout(key.grpcKeepAliveTimeout); }