From c64b9c7b9ab6b7ecc30510cbe4a49d6830f46f06 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 23 Jul 2026 23:23:36 +0200 Subject: [PATCH 01/33] refactor: remove dead local-DC contact-point compatibility check (DRIVER-201) OptionalLocalDcHelper.checkLocalDatacenterCompatibility() warned when a contact point reported a datacenter different from the configured local DC. This has been dead code on scylla-4.x since 12e6acb90b: refresh matches nodes by hostId only, so contact-point nodes never get a datacenter assigned and the warning could never fire. Remove it. The separate "configured local DC matches no node" warning is retained. Co-Authored-By: Claude Opus 4.8 --- .../helper/OptionalLocalDcHelper.java | 74 +++++-------------- 1 file changed, 17 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java index b93a16a6525..97aab92fff7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java @@ -26,7 +26,6 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -68,73 +67,34 @@ public OptionalLocalDcHelper( @Override @NonNull public Optional discoverLocalDc(@NonNull Map nodes) { - String localDcStr = context.getLocalDatacenter(profile.getName()); - Optional localDc; - if (localDcStr != null) { - LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); + String localDc = context.getLocalDatacenter(profile.getName()); + if (localDc != null) { + LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDc); } else if (profile.isDefined(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) { - localDcStr = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); - LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDcStr); - localDc = Optional.of(localDcStr); - } else { - localDc = Optional.empty(); - } - if (localDc.isPresent()) { - checkLocalDatacenterCompatibility( - localDc.get(), context.getMetadataManager().getContactPoints()); - // Also warn if the configured DC doesn't match any node in the cluster - if (!nodes.isEmpty()) { - boolean found = false; - for (Node node : nodes.values()) { - if (localDc.get().equals(node.getDatacenter())) { - found = true; - break; - } - } - if (!found) { - LOG.warn( - "[{}] Configured local DC '{}' does not match any node's datacenter" - + " (available DCs: {}); please verify your configuration", - logPrefix, - localDc.get(), - formatDcs(nodes.values())); - } - } + localDc = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER); + LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDc); } else { LOG.debug("[{}] Local DC not set, DC awareness will be disabled", logPrefix); + return Optional.empty(); } - return localDc; - } - - /** - * Checks if the contact points are compatible with the local datacenter specified either through - * configuration, or programmatically. - * - *

The default implementation logs a warning when a contact point reports a datacenter - * different from the local one, and only for the default profile. - * - * @param localDc The local datacenter, as specified in the config, or programmatically. - * @param contactPoints The contact points provided when creating the session. - */ - protected void checkLocalDatacenterCompatibility( - @NonNull String localDc, Set contactPoints) { - if (profile.getName().equals(DriverExecutionProfile.DEFAULT_NAME)) { - Set badContactPoints = new LinkedHashSet<>(); - for (Node node : contactPoints) { - if (!Objects.equals(localDc, node.getDatacenter())) { - badContactPoints.add(node); + if (!nodes.isEmpty()) { + boolean found = false; + for (Node node : nodes.values()) { + if (localDc.equals(node.getDatacenter())) { + found = true; + break; } } - if (!badContactPoints.isEmpty()) { + if (!found) { LOG.warn( - "[{}] You specified {} as the local DC, but some contact points are from a different DC: {}; " - + "please provide the correct local DC, or check your contact points", + "[{}] Configured local DC '{}' does not match any node's datacenter" + + " (available DCs: {}); please verify your configuration", logPrefix, localDc, - formatNodesAndDcs(badContactPoints)); + formatDcs(nodes.values())); } } + return Optional.of(localDc); } /** From 43e7a1603f97e650727fee9f9185a1cb70a50e6b Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 23 Jul 2026 23:23:59 +0200 Subject: [PATCH 02/33] feat: keep contact points unresolved; deprecate RESOLVE_CONTACT_POINTS (DRIVER-201) Contact points backed by a hostname are now always kept unresolved, so they can be expanded to all their DNS-mapped IPs later, at connection time (see the follow-up EndPoint.resolveAll() commits). SessionBuilder no longer reads RESOLVE_CONTACT_POINTS when merging contact points; the option is deprecated and has no effect. An already-resolved InetSocketAddress passed programmatically is still used as provided, with no further expansion. Co-Authored-By: Claude Opus 4.8 --- .../api/core/config/DefaultDriverOption.java | 4 ++++ .../api/core/config/TypedDriverOption.java | 8 ++++++- .../api/core/session/SessionBuilder.java | 21 +++++++++++-------- core/src/main/resources/reference.conf | 4 ++++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index 3a6e4ed69bb..c2d723a00e7 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -837,7 +837,11 @@ public enum DefaultDriverOption implements DriverOption { * Whether to resolve the addresses passed to `basic.contact-points`. * *

Value-type: boolean + * + * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all + * their DNS-mapped IPs lazily at connection time. Setting this option has no effect. */ + @Deprecated RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"), /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index e412b99b404..cfe22540b4d 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -664,7 +664,13 @@ public String toString() { /** The coalescer reschedule interval. */ public static final TypedDriverOption COALESCER_INTERVAL = new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION); - /** Whether to resolve the addresses passed to `basic.contact-points`. */ + /** + * Whether to resolve the addresses passed to `basic.contact-points`. + * + * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all + * their DNS-mapped IPs lazily at connection time. Setting this option has no effect. + */ + @Deprecated public static final TypedDriverOption RESOLVE_CONTACT_POINTS = new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN); /** diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index 8375f0ef30b..85a2e0488b3 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -166,11 +166,15 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad *

Contact points can also be provided statically in the configuration. If both are specified, * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042. * - *

Contrary to the configuration, DNS names with multiple A-records will not be handled here. - * If you need that, extract them manually with {@link java.net.InetAddress#getAllByName(String)} - * before calling this method. Similarly, if you need connect addresses to stay unresolved, make - * sure you pass unresolved instances here (see {@code advanced.resolve-contact-points} in the - * configuration for more explanations). + *

The driver automatically expands any contact point backed by an unresolved hostname to all + * its DNS-mapped IPs at connection time (via {@code EndPoint.resolveAll()}), so passing a single + * hostname is sufficient to try all its IPs on initial connect. This applies equally to hostnames + * provided here programmatically (build an unresolved {@link InetSocketAddress} with {@link + * InetSocketAddress#createUnresolved(String, int)} to opt in) and to hostnames specified in the + * configuration. An already-resolved address passed here (the common case when constructing an + * {@code InetSocketAddress} directly from a hostname, which resolves eagerly) is used as + * provided, with no further expansion. The {@code advanced.resolve-contact-points} option is + * deprecated and has no effect. */ @NonNull public SelfT addContactPoints(@NonNull Collection contactPoints) { @@ -957,11 +961,10 @@ protected final CompletionStage buildDefaultSessionAsync() { programmaticArguments = programmaticArgumentsBuilder.build(); } - boolean resolveAddresses = - defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false); - + // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved + // hostnames and expanded to all their DNS IPs at connection time via EndPoint.resolveAll(). Set contactPoints = - ContactPoints.merge(programmaticContactPoints, configContactPoints, resolveAddresses); + ContactPoints.merge(programmaticContactPoints, configContactPoints, false); if (keyspace == null && defaultConfig.isDefined(DefaultDriverOption.SESSION_KEYSPACE)) { keyspace = diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 8a3a444319e..54f18e8eddf 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1247,6 +1247,10 @@ datastax-java-driver { # Required: no (defaults to false) # Modifiable at runtime: no # Overridable in a profile: no + # + # DEPRECATED: this option no longer has any effect. Contact points are always kept as unresolved + # hostnames and expanded to all their DNS-mapped IPs at connection time (see + # EndPoint.resolveAll()). It will be removed in a future release. advanced.resolve-contact-points = false advanced.protocol { From 767674fc0d6bca04f6cf853306ee32271317effd Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 23 Jul 2026 23:24:33 +0200 Subject: [PATCH 03/33] feat: add EndPoint.resolveAll() for multi-address DNS expansion (DRIVER-201) Add EndPoint.resolveAll(Executor), a default method returning a CompletionStage of all socket addresses an endpoint maps to, so a hostname with multiple A-records can be expanded to every IP. The default offloads resolve() to the executor and wraps it in a single-element array, keeping existing third-party implementations working (source- and binary-compatible; resolve() is not deprecated yet). Overrides: - DefaultEndPoint: InetAddress.getAllByName() expansion for unresolved addresses, falling back to the single stored address on DNS failure. - SniEndPoint: returns the complete A-record set, rotated with the same round-robin OFFSET as resolve() so healthy connections spread across proxy IPs. - ClientRoutesEndPoint: single-address-by-design, delegates to resolve(). Co-Authored-By: Claude Opus 4.8 --- .../driver/api/core/metadata/EndPoint.java | 45 +++++++- .../core/metadata/ClientRoutesEndPoint.java | 18 ++++ .../core/metadata/DefaultEndPoint.java | 50 +++++++++ .../internal/core/metadata/SniEndPoint.java | 53 +++++++++ .../core/metadata/DefaultEndPointTest.java | 37 +++++++ .../core/metadata/SniEndPointTest.java | 101 ++++++++++++++++++ 6 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java index 530f2ad38ac..c4b131d9e6f 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java @@ -18,14 +18,16 @@ package com.datastax.oss.driver.api.core.metadata; import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; /** * Encapsulates the information needed to open connections to a node. * *

By default, the driver assumes plain TCP connections, and this is just a wrapper around an - * {@link InetSocketAddress}. However, more complex deployment scenarios might use a custom + * {@link java.net.InetSocketAddress}. However, more complex deployment scenarios might use a custom * implementation that contains additional information; for example, if the nodes are accessed * through a proxy with SNI routing, an SNI server name is needed in addition to the proxy address. */ @@ -40,6 +42,45 @@ public interface EndPoint { @NonNull SocketAddress resolve(); + /** + * Resolves this instance to all known socket addresses, asynchronously. + * + *

This is called each time the driver opens a new connection to the node. For endpoints backed + * by a plain IP address the returned array contains exactly one element. For endpoints whose + * hostname resolves to multiple IPs (e.g. a DNS round-robin entry) all addresses are returned so + * that the driver can try each one in sequence and fall back gracefully when individual IPs are + * unreachable. + * + *

Resolution is asynchronous on purpose: name resolution can block (e.g. {@link + * java.net.InetAddress#getAllByName(String)}), and the driver calls this from its admin event + * loop, which must never block. Implementations whose resolution may block must run it on + * the supplied {@code executor} rather than on the calling thread (see the default + * implementation). Implementations that resolve from memory (e.g. an already-resolved address or + * an in-memory lookup) may return an {@linkplain CompletableFuture#completedFuture(Object) + * already completed stage} and ignore the executor. + * + *

The default implementation offloads {@link #resolve()} to {@code executor} and wraps the + * result in a single-element array. Implementations that can supply multiple addresses should + * override this method. + * + *

The returned stage must not be null and must complete with a non-null, non-empty array. + * + * @param executor the executor to run potentially-blocking resolution on; must not be null. The + * driver supplies a dedicated resolver executor so that blocking name resolution never runs + * on an event loop. + * @apiNote Timeout note: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} tries each address in + * sequence. If a hostname resolves to N addresses and each attempt times out, the worst-case + * connection time before declaring a node unreachable is {@code N × + * advanced.connection.connect-timeout}. In practice DNS round-robin entries have only a small + * number of records, so this is rarely a concern, but callers should be aware of this when + * configuring connect timeouts. + */ + @NonNull + default CompletionStage resolveAll(@NonNull Executor executor) { + return CompletableFuture.supplyAsync(() -> new SocketAddress[] {resolve()}, executor); + } + /** * Returns an alternate string representation for use in node-level metric names. * diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java index 15d825b2efc..62976f08cb3 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -27,6 +27,9 @@ import java.net.SocketAddress; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; public class ClientRoutesEndPoint implements EndPoint { private final UUID hostId; @@ -76,6 +79,21 @@ public SocketAddress resolve() { return fallbackEndPoint.resolve(); } + /** + * Returns all socket addresses for this endpoint. + * + *

Delegates to {@link #resolve()} to obtain the single address provided by the topology + * monitor (or the fallback endpoint), then returns it as a one-element array in an already + * completed stage. The topology monitor resolves each node to exactly one address by design (via + * an in-memory per-host-id lookup), so multi-address expansion is not applicable here and the + * {@code executor} is not used. + */ + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + return CompletableFuture.completedFuture(new SocketAddress[] {resolve()}); + } + @Override public boolean equals(Object other) { if (other == this) { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index 7ffbee8e4bb..a7bec899ccd 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -20,8 +20,14 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.Serializable; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; public class DefaultEndPoint implements EndPoint, Serializable { @@ -41,6 +47,50 @@ public InetSocketAddress resolve() { return address; } + /** + * Returns all socket addresses for this endpoint. + * + *

If the stored address is already resolved, the returned stage completes immediately (on the + * calling thread) with a single-element array; no executor hop occurs. + * + *

If the stored address is unresolved (i.e. the driver was configured with {@code + * RESOLVE_CONTACT_POINTS=false} and the hostname has not been looked up yet), the blocking {@link + * InetAddress#getAllByName(String)} lookup is run on {@code executor} to expand the hostname to + * every IP it resolves to. Each resolved IP is returned as an {@link InetSocketAddress} with the + * same port as the original. + * + *

If DNS resolution fails, falls back to a single-element array containing {@link #resolve()}. + * + *

Note on resolver: DNS lookup is performed via {@link + * InetAddress#getAllByName(String)}, bypassing any custom Netty {@code AddressResolverGroup} + * configured on the driver. This is consistent with how {@link SniEndPoint} performs DNS + * resolution elsewhere in the driver. Users who rely on a custom Netty resolver should supply + * pre-resolved {@link java.net.InetSocketAddress} instances instead of hostnames. + */ + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + if (!address.isUnresolved()) { + return CompletableFuture.completedFuture(new SocketAddress[] {address}); + } + return CompletableFuture.supplyAsync( + () -> { + try { + InetAddress[] all = InetAddress.getAllByName(address.getHostString()); + SocketAddress[] result = new SocketAddress[all.length]; + for (int i = 0; i < all.length; i++) { + result[i] = new InetSocketAddress(all[i], address.getPort()); + } + return result; + } catch (UnknownHostException e) { + // Fallback: return the single unresolved address; the connect attempt will fail with a + // descriptive error rather than silently returning an empty array. + return new SocketAddress[] {address}; + } + }, + executor); + } + @Override public boolean equals(Object other) { if (other == this) { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index d1ab8eec98d..acd3431580e 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -22,10 +22,14 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.net.UnknownHostException; import java.util.Arrays; import java.util.Comparator; import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; public class SniEndPoint implements EndPoint { @@ -75,6 +79,55 @@ public InetSocketAddress resolve() { } } + /** + * Returns all socket addresses for this SNI proxy endpoint. + * + *

Re-resolves the proxy hostname on each call and returns one {@link InetSocketAddress} per + * A-record, so that the driver can try every proxy IP in sequence if one is unreachable. + * + *

All A-records are returned so a single connection attempt can fall back across every proxy + * IP, but the candidate order is rotated on each call using the same round-robin {@link #OFFSET} + * counter as {@link #resolve()}. This spreads healthy connections across proxy IPs instead of + * always starting at the first one, preserving the previous load-balancing behavior. + * + *

The blocking DNS lookup is run on {@code executor} so it never blocks the calling (event + * loop) thread; the returned stage completes with the rotated candidate array, or completes + * exceptionally with an {@link IllegalArgumentException} if the proxy hostname cannot be + * resolved. + */ + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + return CompletableFuture.supplyAsync( + () -> { + try { + InetAddress[] aRecords = InetAddress.getAllByName(proxyAddress.getHostName()); + if (aRecords.length == 0) { + throw new IllegalArgumentException( + "Could not resolve proxy address " + proxyAddress.getHostName()); + } + // The order of the returned addresses is unspecified. Sort by IP so the round-robin + // rotation below is deterministic across calls. + Arrays.sort(aRecords, IP_COMPARATOR); + int start = + (aRecords.length == 1) + ? 0 + : OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) + % aRecords.length; + SocketAddress[] result = new SocketAddress[aRecords.length]; + for (int i = 0; i < aRecords.length; i++) { + InetAddress aRecord = aRecords[(start + i) % aRecords.length]; + result[i] = new InetSocketAddress(aRecord, proxyAddress.getPort()); + } + return result; + } catch (UnknownHostException e) { + throw new IllegalArgumentException( + "Could not resolve proxy address " + proxyAddress.getHostName(), e); + } + }, + executor); + } + @Override public boolean equals(Object other) { if (other == this) { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index 7da8fb39415..66b794cdcf8 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.net.InetSocketAddress; +import java.net.SocketAddress; import org.junit.Test; public class DefaultEndPointTest { @@ -57,4 +58,40 @@ public void should_reject_null_address() { .isInstanceOf(NullPointerException.class) .hasMessage("address can't be null"); } + + @Test + public void resolve_all_returns_single_element_for_already_resolved_address() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + assertThat(all).hasSize(1); + assertThat(((InetSocketAddress) all[0]).isUnresolved()).isFalse(); + assertThat(((InetSocketAddress) all[0]).getHostString()).isEqualTo("127.0.0.1"); + } + + @Test + public void resolve_all_expands_unresolved_hostname_to_at_least_one_address() { + // localhost reliably resolves to at least 127.0.0.1 + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + assertThat(all).isNotEmpty(); + for (SocketAddress addr : all) { + InetSocketAddress inet = (InetSocketAddress) addr; + assertThat(inet.isUnresolved()).isFalse(); + assertThat(inet.getPort()).isEqualTo(9042); + } + } + + @Test + public void resolve_all_falls_back_to_single_element_when_hostname_is_unresolvable() { + // Unresolvable hostname: resolveAll() must not throw; it returns the unresolved address. + DefaultEndPoint endPoint = + new DefaultEndPoint( + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042)); + SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + assertThat(all).hasSize(1); + // The fallback address is the original unresolved one. + assertThat(((InetSocketAddress) all[0]).getHostString()) + .isEqualTo("this-host-does-not-exist.invalid"); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java new file mode 100644 index 00000000000..c7f5bb695e8 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.junit.Test; + +public class SniEndPointTest { + + @Test + public void resolve_all_returns_all_proxy_addresses_for_resolvable_hostname() { + // localhost reliably resolves to at least one address + SniEndPoint endPoint = + new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); + SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + assertThat(all).isNotEmpty(); + for (SocketAddress addr : all) { + InetSocketAddress inet = (InetSocketAddress) addr; + assertThat(inet.isUnresolved()).isFalse(); + assertThat(inet.getPort()).isEqualTo(9042); + } + } + + @Test + public void resolve_all_fails_for_unresolvable_hostname() { + SniEndPoint endPoint = + new SniEndPoint( + new InetSocketAddress("this-host-does-not-exist.invalid", 9042), "test-server-name"); + // Resolution is async now: the failure surfaces as an exceptionally-completed stage whose cause + // is the IllegalArgumentException. + assertThatThrownBy(() -> endPoint.resolveAll(Runnable::run).toCompletableFuture().join()) + .hasCauseInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Could not resolve proxy address"); + } + + @Test + public void resolve_returns_single_address_from_round_robin() { + // Sanity check: resolve() still works and returns a single address + SniEndPoint endPoint = + new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); + InetSocketAddress addr = endPoint.resolve(); + assertThat(addr.isUnresolved()).isFalse(); + assertThat(addr.getPort()).isEqualTo(9042); + } + + @Test + public void resolve_all_returns_complete_and_rotated_candidate_order() { + // resolveAll() must always return the full set of A-records (so a single connection attempt can + // fall back across every proxy IP), while rotating the starting element on each call to + // preserve the round-robin behavior of resolve(). We assert both invariants without depending + // on how many addresses "localhost" resolves to in a given environment. + SniEndPoint endPoint = + new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); + + SocketAddress[] first = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + int size = first.length; + assertThat(size).isGreaterThanOrEqualTo(1); + Set expected = new HashSet<>(Arrays.asList(first)); + + // Every call returns the same complete set of candidates, regardless of rotation. + boolean sawRotation = false; + SocketAddress firstElementSeed = first[0]; + for (int call = 0; call < size * 2; call++) { + SocketAddress[] candidates = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + assertThat(candidates).hasSize(size); + assertThat(new HashSet<>(Arrays.asList(candidates))).isEqualTo(expected); + if (!candidates[0].equals(firstElementSeed)) { + sawRotation = true; + } + } + + // When there is more than one A-record the starting candidate must rotate across calls. + if (size > 1) { + assertThat(sawRotation) + .as("resolveAll() should rotate the starting candidate when multiple IPs exist") + .isTrue(); + } + } +} From 023f791f379a3ed8ae288be7cb864334de9e37e5 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 23 Jul 2026 23:25:04 +0200 Subject: [PATCH 04/33] feat: resolve endpoints off the event loop with multi-IP fallback; deprecate resolve() (DRIVER-201) ChannelFactory.connect() now resolves via EndPoint.resolveAll() on a dedicated daemon resolver executor, so blocking DNS never runs on the admin event loop. The returned candidates are tried serially (tryNextCandidate): each address is attempted via connectToAddress(), and on a per-address failure the next candidate is tried; only when all are exhausted does the overall future fail. Protocol-version negotiation (downgrade retries) stays scoped to a single address. A null/empty resolveAll() result fails the future rather than NPE/AIOOBE. DefaultSession shuts the resolver executor down on close. resolve() is now @Deprecated in favour of resolveAll(). The five internal callers that legitimately need a single canonical address (DefaultTopologyMonitor, InsightsClient, DseGssApiAuthProviderBase, DefaultSslEngineFactory, SniSslEngineFactory) are annotated @SuppressWarnings("deprecation") so the -Werror build stays green. Co-Authored-By: Claude Opus 4.8 --- .../core/auth/DseGssApiAuthProviderBase.java | 2 + .../core/insights/InsightsClient.java | 6 + .../driver/api/core/metadata/EndPoint.java | 8 +- .../internal/core/channel/ChannelFactory.java | 171 +++++++++++++++++- .../core/metadata/DefaultTopologyMonitor.java | 4 + .../internal/core/session/DefaultSession.java | 8 + .../core/ssl/DefaultSslEngineFactory.java | 2 + .../core/ssl/SniSslEngineFactory.java | 2 + .../ChannelFactoryAsyncResolveTest.java | 146 +++++++++++++++ .../ChannelFactoryResolveAllGuardTest.java | 136 ++++++++++++++ .../internal/core/channel/LocalEndPoint.java | 10 + .../driver/core/resolver/MockResolverIT.java | 39 ++++ 12 files changed, 525 insertions(+), 9 deletions(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java diff --git a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java index 48a0e5b0ef3..593552d77c1 100644 --- a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java +++ b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java @@ -291,6 +291,8 @@ protected static class GssApiAuthenticator extends BaseDseAuthenticator { private SaslClient saslClient; private EndPoint endPoint; + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); + // Kerberos authentication needs a single canonical hostname for SASL service name resolution. protected GssApiAuthenticator( GssApiOptions options, EndPoint endPoint, String serverAuthenticator) { super(serverAuthenticator); diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java b/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java index 168477894ed..54d723152a4 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java @@ -288,6 +288,8 @@ private InsightsStatusData createStatusData() { .build(); } + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); + // address reporting needs a single canonical address per node. private Map getConnectedNodes() { Map pools = driverContext.getPoolManager().getPools(); return pools.entrySet().stream() @@ -302,6 +304,8 @@ private SessionStateForNode constructSessionStateForNode(Map.Entry startupOptions = driverContext.getStartupOptions(); return InsightsStartupData.builder() @@ -454,6 +458,8 @@ private PoolSizeByHostDistance getPoolSizeByHostDistance() { 0); } + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); + // address reporting needs a single canonical address for the control connection. private String getControlConnectionSocketAddress() { SocketAddress controlConnectionAddress = controlConnection.channel().getEndPoint().resolve(); return AddressFormatter.nullSafeToString(controlConnectionAddress); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java index c4b131d9e6f..b4c6a03286b 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java @@ -34,11 +34,17 @@ public interface EndPoint { /** - * Resolves this instance to a socket address. + * Resolves this instance to a single socket address. * *

This will be called each time the driver opens a new connection to the node. The returned * address cannot be null. + * + * @deprecated Use {@link #resolveAll(Executor)} instead. When a hostname maps to multiple IPs + * (e.g. in dynamic DNS environments) only one address is returned here, causing the driver to + * miss fallback IPs when the first one is unreachable. {@code resolveAll(Executor)} returns + * the full set, resolved asynchronously off the calling thread. */ + @Deprecated @NonNull SocketAddress resolve(); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 6bfc355f910..d2e05ebef85 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -62,8 +62,11 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import net.jcip.annotations.ThreadSafe; @@ -107,6 +110,12 @@ public class ChannelFactory { private final String logPrefix; protected final InternalDriverContext context; + // Runs potentially-blocking EndPoint name resolution (EndPoint.resolveAll()) off the caller + // thread: connect() is invoked from the admin event loop, which must never block on DNS. A cached + // pool gives each concurrent resolution its own daemon thread, so a single blackholed lookup + // cannot starve other nodes' reconnections; idle threads are reclaimed after ~60s. + private final ExecutorService resolverExecutor; + /** either set from the configuration, or null and will be negotiated */ @VisibleForTesting volatile ProtocolVersion protocolVersion; @@ -124,6 +133,18 @@ public ChannelFactory(InternalDriverContext context) { this.logPrefix = context.getSessionName(); this.context = context; + AtomicInteger resolverThreadCount = new AtomicInteger(); + this.resolverExecutor = + Executors.newCachedThreadPool( + runnable -> { + Thread thread = + new Thread( + runnable, + logPrefix + "-connection-resolver-" + resolverThreadCount.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); + DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); if (defaultConfig.isDefined(DefaultDriverOption.PROTOCOL_VERSION)) { String versionName = defaultConfig.getString(DefaultDriverOption.PROTOCOL_VERSION); @@ -154,6 +175,14 @@ public String getClusterName() { return clusterName; } + /** + * Shuts down the internal DNS-resolver executor. Invoked during session shutdown. The executor's + * threads are daemon threads, so a missed call cannot prevent JVM exit. + */ + public void close() { + resolverExecutor.shutdownNow(); + } + public CompletionStage connect(Node node, DriverChannelOptions options) { NodeMetricUpdater nodeMetricUpdater; if (node instanceof DefaultNode) { @@ -219,14 +248,137 @@ private void connect( List attemptedVersions, CompletableFuture resultFuture) { - SocketAddress resolvedAddress; + // Resolution may block (DNS); it is offloaded to resolverExecutor so the calling thread (the + // admin event loop, for control-connection reconnects) never waits on it. The continuation + // below runs off the admin loop -- tryNextCandidate()/Bootstrap.connect() are safe from any + // thread and per-candidate retries already run on Netty I/O threads. + CompletionStage resolveStage; try { - resolvedAddress = endPoint.resolve(); + resolveStage = endPoint.resolveAll(resolverExecutor); } catch (Exception e) { + // An implementation may throw synchronously while building the stage. resultFuture.completeExceptionally(e); return; } + resolveStage.whenComplete( + (candidates, error) -> { + if (error != null) { + // supplyAsync wraps supplier throwables in a CompletionException; unwrap so callers + // (and the guard test's isSameAs assertion) see the original cause. + Throwable cause = + (error instanceof CompletionException && error.getCause() != null) + ? error.getCause() + : error; + resultFuture.completeExceptionally(cause); + return; + } + if (candidates == null || candidates.length == 0) { + resultFuture.completeExceptionally( + new IllegalArgumentException( + "EndPoint.resolveAll() must return a non-null, non-empty array: " + endPoint)); + return; + } + tryNextCandidate( + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + resultFuture, + candidates, + 0); + }); + } + + /** + * Iterates through the candidate addresses from {@link + * EndPoint#resolveAll(java.util.concurrent.Executor)}. Tries each one in sequence; if an address + * fails for a reason other than protocol-version negotiation exhaustion, the next candidate is + * tried. Only when all candidates are exhausted is the overall {@code resultFuture} failed. + * + *

Timeout note: addresses are tried serially, so the worst-case time before failure is + * {@code N × connect-timeout} where N is the number of candidates. This is an intentional + * tradeoff: failing immediately on the first unreachable IP would prevent fallback to healthy + * ones. In practice DNS entries have only a small number of records. + */ + private void tryNextCandidate( + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + List attemptedVersions, + CompletableFuture resultFuture, + SocketAddress[] candidates, + int index) { + + SocketAddress candidate = candidates[index]; + CompletableFuture perAddressFuture = new CompletableFuture<>(); + connectToAddress( + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + perAddressFuture, + candidate); + + perAddressFuture.whenComplete( + (channel, error) -> { + if (error == null) { + resultFuture.complete(channel); + } else if (index + 1 < candidates.length) { + LOG.debug( + "[{}] Failed to connect to {} ({}), trying next address", + logPrefix, + candidate, + error.getMessage()); + tryNextCandidate( + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + resultFuture, + candidates, + index + 1); + } else { + // Note: might be completed already if the failure happened in initializer() + resultFuture.completeExceptionally(error); + } + }); + } + + /** + * Performs a Netty bootstrap connect to a single, already-resolved address. Handles + * protocol-version negotiation (downgrade retries) internally, staying on the same address. Uses + * {@code perAddressFuture} so {@link #tryNextCandidate} can distinguish a per-address TCP failure + * (try the next IP) from a successful protocol handshake. + */ + private void connectToAddress( + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + ProtocolVersion currentVersion, + boolean isNegotiating, + List attemptedVersions, + CompletableFuture perAddressFuture, + SocketAddress resolvedAddress) { + NettyOptions nettyOptions = context.getNettyOptions(); Bootstrap bootstrap = @@ -235,7 +387,8 @@ private void connect( .channel(nettyOptions.channelClass()) .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()) .handler( - initializer(endPoint, currentVersion, options, nodeMetricUpdater, resultFuture)); + initializer( + endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); nettyOptions.afterBootstrapInitialized(bootstrap); @@ -294,7 +447,7 @@ private void connect( ConsistencyLevel.LOCAL_QUORUM.name())); } } - resultFuture.complete(driverChannel); + perAddressFuture.complete(driverChannel); } else { Throwable error = connectFuture.cause(); if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { @@ -307,7 +460,8 @@ private void connect( logPrefix, currentVersion, downgraded.get()); - connect( + // Stay on the same address for protocol-version downgrade retries. + connectToAddress( endPoint, shardingInfo, shardId, @@ -316,16 +470,17 @@ private void connect( downgraded.get(), true, attemptedVersions, - resultFuture); + perAddressFuture, + resolvedAddress); } else { - resultFuture.completeExceptionally( + perAddressFuture.completeExceptionally( UnsupportedProtocolVersionException.forNegotiation( endPoint, attemptedVersions)); } } else { // Note: might be completed already if the failure happened in initializer(), this is // fine - resultFuture.completeExceptionally(error); + perAddressFuture.completeExceptionally(error); } } }); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java index 5a82bfe2c86..2098a3e6553 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java @@ -701,6 +701,8 @@ private Optional findInPeers( // Current versions of Cassandra (3.11 at the time of writing), require the same port for all // nodes. As a consequence, the port is not stored in system tables. // We save it the first time we get a control connection channel. + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); a single + // canonical address is all that is needed here to extract the port. protected void savePort(DriverChannel channel) { if (port < 0) { SocketAddress address = channel.getEndPoint().resolve(); @@ -723,6 +725,8 @@ protected void savePort(DriverChannel channel) { * otherwise. */ @Nullable + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); a single + // canonical address is all that is needed here for the peer-vs-local comparison. protected InetSocketAddress getBroadcastRpcAddress( @NonNull AdminRow row, @NonNull EndPoint localEndPoint) { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java index c9fee86f2c1..b2ed5111077 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java @@ -630,6 +630,14 @@ private void onChildrenClosed(List> childrenCloseStages) { for (CompletionStage stage : childrenCloseStages) { warnIfFailed(stage); } + // The channel factory owns a DNS-resolver executor that is not an AsyncAutoCloseable child; + // shut it down here, after all pools/control-connection that used it are closed. Guarded like + // the other context-component accesses below (the factory may have failed to initialize). + try { + context.getChannelFactory().close(); + } catch (Throwable t) { + // ignore: the factory may have failed to initialize, nothing to close + } context .getNettyOptions() .onClose() diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java index 343d3f9e4e7..c64a334b1fe 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java @@ -116,6 +116,8 @@ protected String hostNoLookup(InetSocketAddress addr) { @NonNull @Override + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); SSL + // factories legitimately need a single address for hostname verification. public SSLEngine newSslEngine(@NonNull EndPoint remoteEndpoint) { SSLEngine engine; SocketAddress remoteAddress = remoteEndpoint.resolve(); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java index 4d2cb69fbfc..424120a99f3 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java @@ -51,6 +51,8 @@ public SniSslEngineFactory(SSLContext sslContext, boolean allowDnsReverseLookupS @NonNull @Override + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); SSL + // factories legitimately need a single address for SNI hostname verification. public SSLEngine newSslEngine(@NonNull EndPoint remoteEndpoint) { if (!(remoteEndpoint instanceof SniEndPoint)) { throw new IllegalArgumentException( diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java new file mode 100644 index 00000000000..d604e4709fb --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThat; +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +/** + * Verifies that {@link ChannelFactory#connect} resolves endpoint addresses asynchronously, off the + * calling (admin event loop) thread, so DNS never blocks the caller. + */ +public class ChannelFactoryAsyncResolveTest extends ChannelFactoryTestBase { + + @Test + public void should_resolve_on_dedicated_resolver_thread() throws Exception { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletableFuture resolverThreadName = new CompletableFuture<>(); + EndPoint endPoint = + new EndPoint() { + @NonNull + @Override + public SocketAddress resolve() { + return SERVER_ADDRESS.resolve(); + } + + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + return CompletableFuture.supplyAsync( + () -> { + resolverThreadName.complete(Thread.currentThread().getName()); + return new SocketAddress[] {SERVER_ADDRESS.resolve()}; + }, + executor); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + }; + + String callerThreadName = Thread.currentThread().getName(); + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – resolution ran on the factory's dedicated resolver executor, not the caller thread + String actualThreadName = resolverThreadName.get(2, TimeUnit.SECONDS); + assertThat(actualThreadName).isNotEqualTo(callerThreadName); + assertThat(actualThreadName).contains("-connection-resolver-"); + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_not_block_caller_while_resolving() throws Exception { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CountDownLatch resolutionEntered = new CountDownLatch(1); + CountDownLatch releaseResolution = new CountDownLatch(1); + EndPoint endPoint = + new EndPoint() { + @NonNull + @Override + public SocketAddress resolve() { + return SERVER_ADDRESS.resolve(); + } + + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + return CompletableFuture.supplyAsync( + () -> { + resolutionEntered.countDown(); + try { + releaseResolution.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return new SocketAddress[] {SERVER_ADDRESS.resolve()}; + }, + executor); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + }; + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – connect() returned control while resolution is still blocked (caller not blocked) + assertThat(resolutionEntered.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(channelFuture.toCompletableFuture().isDone()).isFalse(); + + // Once resolution is unblocked the connection proceeds to completion. + releaseResolution.countDown(); + completeSimpleChannelInit(); + assertThatStage(channelFuture).isSuccess(); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java new file mode 100644 index 00000000000..99fbd156f7a --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies that {@link ChannelFactory#connect} completes the result future exceptionally (rather + * than throwing or hanging) when {@link EndPoint#resolveAll(java.util.concurrent.Executor)} + * completes with {@code null}, an empty array, throws synchronously, or completes exceptionally. + * Resolution is now asynchronous, so these assertions exercise the {@code whenComplete} callback. + */ +public class ChannelFactoryResolveAllGuardTest extends ChannelFactoryTestBase { + + @Test + public void should_fail_future_when_resolve_all_returns_null() { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + EndPoint badEndPoint = mock(EndPoint.class); + when(badEndPoint.resolveAll(any())).thenReturn(CompletableFuture.completedFuture(null)); + + // When + CompletionStage channelFuture = + factory.connect( + badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – future must complete exceptionally without hanging + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("resolveAll() must return a non-null, non-empty array")); + } + + @Test + public void should_fail_future_when_resolve_all_returns_empty_array() { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + EndPoint badEndPoint = mock(EndPoint.class); + when(badEndPoint.resolveAll(any())) + .thenReturn(CompletableFuture.completedFuture(new SocketAddress[0])); + + // When + CompletionStage channelFuture = + factory.connect( + badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – future must complete exceptionally without hanging + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("resolveAll() must return a non-null, non-empty array")); + } + + @Test + public void should_fail_future_when_resolve_all_throws_synchronously() { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + EndPoint badEndPoint = mock(EndPoint.class); + RuntimeException testException = new RuntimeException("DNS lookup failed"); + when(badEndPoint.resolveAll(any())).thenThrow(testException); + + // When + CompletionStage channelFuture = + factory.connect( + badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – future must complete exceptionally with the thrown exception + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(testException)); + } + + @Test + public void should_fail_future_with_unwrapped_cause_when_resolve_all_stage_fails() { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + EndPoint badEndPoint = mock(EndPoint.class); + RuntimeException testException = new RuntimeException("DNS lookup failed"); + // A real impl offloads via supplyAsync, which wraps the supplier's throwable in a + // CompletionException; connect() must unwrap it so callers see the original cause. + CompletableFuture failedStage = new CompletableFuture<>(); + failedStage.completeExceptionally(new CompletionException(testException)); + when(badEndPoint.resolveAll(any())).thenReturn(failedStage); + + // When + CompletionStage channelFuture = + factory.connect( + badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – future must complete exceptionally with the unwrapped original exception + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(testException)); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java index c90731eece9..e9c8f836de4 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java @@ -21,6 +21,9 @@ import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.channel.local.LocalAddress; import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; /** Endpoint implementation for unit tests that use the local Netty transport. */ public class LocalEndPoint implements EndPoint { @@ -37,6 +40,13 @@ public SocketAddress resolve() { return localAddress; } + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + // In-memory local address; resolve synchronously without hopping to the executor. + return CompletableFuture.completedFuture(new SocketAddress[] {localAddress}); + } + @NonNull @Override public String asMetricPrefix() { diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java index 4e9eefebf63..730827b2537 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java @@ -114,6 +114,45 @@ public void should_connect_with_mocked_hostname() { } } + @Test + public void should_connect_when_first_dns_entry_is_non_responsive() { + final int numberOfNodes = 2; + DriverConfigLoader loader = + new DefaultProgrammaticDriverConfigLoaderBuilder() + .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) + .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) + .withStringList( + TypedDriverOption.CONTACT_POINTS.getRawOption(), + Collections.singletonList("test.cluster.fake:9042")) + .build(); + + CqlSessionBuilder builder = new CqlSessionBuilder().withConfigLoader(loader); + try (CcmBridge ccmBridge = + CcmBridge.builder().withNodes(numberOfNodes).withIpPrefix("127.0.1.").build()) { + MultimapHostResolverProvider.removeResolverEntries("test.cluster.fake"); + // Register the dead IP first so it's the first entry InetAddress.getAllByName() returns for + // this hostname (MultimapHostResolver preserves insertion order). Node 11 is never started, + // so nothing listens on 127.0.1.11 in this subnet. + MultimapHostResolverProvider.addResolverEntry("test.cluster.fake", "127.0.1.11"); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(1)); + MultimapHostResolverProvider.addResolverEntry( + "test.cluster.fake", ccmBridge.getNodeIpAddress(2)); + ccmBridge.create(); + ccmBridge.start(); + + try (CqlSession session = builder.build()) { + waitForAllNodesUp(session, numberOfNodes); + ResultSet rs = session.execute("select * from system.local where key='local'"); + assertThat(rs).isNotNull(); + List rows = rs.all(); + assertThat(rows).hasSize(1); + Collection nodes = session.getMetadata().getNodes().values(); + assertThat(nodes).hasSize(numberOfNodes); + } + } + } + @Test public void replace_cluster_test() { final int numberOfNodes = 3; From 27704808b516c01a99044c2bcb93dbd768e29faf Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 23 Jul 2026 23:25:32 +0200 Subject: [PATCH 05/33] fix: gate contact-point reconnection fallback by topology monitor and enable it by default (DRIVER-201) fallback-to-original-contact-points now defaults to true. On a control-connection reconnect the original (unresolved) contact points are appended after the live-node plan; EndPoint.resolveAll() re-expands each hostname to its current DNS IPs, which is the driver's DNS re-resolution path (metadata nodes hold an already-resolved endpoint that is never re-resolved). The append is gated: - only in the RUNNING state (before that newQueryPlan already builds the plan from the contact points, so appending would duplicate them); - skipped when the topology monitor re-resolves node addresses itself (TopologyMonitor.reresolvesNodeAddresses(), overridden true by CloudTopologyMonitor and route-aware in ClientRoutesTopologyMonitor), unless the regular plan is empty, to avoid resurrecting nodes those proxy-based monitors authoritatively removed. The plan is now composed (CompositeQueryPlan + SimpleQueryPlan) rather than mutated, since a RUNNING-state built-in query plan rejects add()/addAll(). HeartbeatIT pins the option to false so the extra OPTIONS message does not skew heartbeat counts. Co-Authored-By: Claude Opus 4.8 --- .../api/core/config/DefaultDriverOption.java | 9 +- .../driver/api/core/config/OptionsMap.java | 2 +- .../api/core/config/TypedDriverOption.java | 2 +- .../metadata/ClientRoutesTopologyMonitor.java | 17 +++ .../core/metadata/CloudTopologyMonitor.java | 8 ++ .../metadata/LoadBalancingPolicyWrapper.java | 44 +++++-- .../core/metadata/MetadataManager.java | 7 ++ .../core/metadata/TopologyMonitor.java | 20 ++++ core/src/main/resources/reference.conf | 14 ++- .../ClientRoutesTopologyMonitorTest.java | 55 +++++++++ .../LoadBalancingPolicyWrapperTest.java | 108 ++++++++++++++++-- .../driver/core/heartbeat/HeartbeatIT.java | 4 + 12 files changed, 261 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index c2d723a00e7..0949da79443 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -701,8 +701,13 @@ public enum DefaultDriverOption implements DriverOption { CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"), /** - * Whether to forcibly add original contact points held by MetadataManager to the reconnection - * plan, in case there is no live nodes available according to LBP. Experimental. + * Whether to append the original contact points held by MetadataManager to the reconnection plan, + * after the live nodes reported by the load balancing policy. Defaults to {@code true}. + * + *

This is also the driver's DNS re-resolution path: contact points are expanded to their + * current DNS IPs at query-plan time, whereas metadata nodes hold an already-resolved endpoint + * that is never re-resolved. Keeping this enabled lets control-connection reconnects re-resolve + * the original hostnames and pick up new IPs once the live-node plan is exhausted. * *

Value-type: boolean */ diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index 1d906caa985..5ea5039ae34 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -369,7 +369,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10)); map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true); - map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false); + map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true); map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true); map.put(TypedDriverOption.REPREPARE_ENABLED, true); map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index cfe22540b4d..b01ff43aa16 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -600,7 +600,7 @@ public String toString() { public static final TypedDriverOption CONTROL_CONNECTION_AGREEMENT_WARN = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN); - /** Whether to forcibly try original contacts if no live nodes are available */ + /** Whether to append the original contact points to the reconnection plan (defaults to true) */ public static final TypedDriverOption CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java index 1ffc35fd9f4..a93c34463bd 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java @@ -21,6 +21,7 @@ import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; @@ -480,6 +481,22 @@ protected EndPoint buildNodeEndPoint( return new ClientRoutesEndPoint(this, hostId, broadcastInetAddress, fallback); } + @Override + public boolean reresolvesNodeAddresses() { + // ClientRoutesEndPoint re-resolves via the client route hostname on every connection attempt, + // but only when a route exists for that host_id (see ClientRoutesEndPoint#resolve()) -- for + // mixed/incomplete route sets it falls back to a static, non-re-resolving endpoint instead. + // Only report true when every currently-known node actually has a live route; otherwise the + // contact-point reconnection fallback must stay available for the nodes stuck on the fallback. + Map routes = resolvedRoutesCache.get(); + for (Node node : context.getMetadataManager().getMetadata().getNodes().values()) { + if (!routes.containsKey(node.getHostId())) { + return false; + } + } + return true; + } + /** * Builds the CQL query to fetch client routes. * diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java index 021824a9b16..013027e139d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java @@ -44,4 +44,12 @@ protected EndPoint buildNodeEndPoint( UUID hostId = Objects.requireNonNull(row.getUuid("host_id")); return new SniEndPoint(cloudProxyAddress, hostId.toString()); } + + @Override + public boolean reresolvesNodeAddresses() { + // Nodes are reached through the cloud SNI proxy via SniEndPoint, which re-resolves the proxy + // hostname on every connection attempt. Appending the original contact points as a DNS + // re-resolution fallback is therefore unnecessary and could resurrect removed nodes. + return true; + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index f3f3e4fe346..fc1390f4ac2 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -27,6 +27,8 @@ import com.datastax.oss.driver.api.core.session.Request; import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.util.collection.CompositeQueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.internal.core.util.concurrent.ReplayingEventFilter; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; @@ -147,7 +149,9 @@ public Queue newQueryPlan( switch (stateRef.get()) { case BEFORE_INIT: case DURING_INIT: - // The contact points are not stored in the metadata yet: + // The contact points are not stored in the metadata yet. Each unresolved hostname is + // expanded to all its DNS IPs at connection time by EndPoint.resolveAll(), so one entry per + // contact point is enough here. List nodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(nodes); return new ConcurrentLinkedQueue<>(nodes); @@ -164,20 +168,42 @@ public Queue newQueryPlan( @NonNull public Queue newControlReconnectionQueryPlan() { + // Read the state once, before building the regular plan. State transitions are monotonic + // (BEFORE_INIT -> DURING_INIT -> RUNNING -> ...), so this captured value is <= the value + // newQueryPlan() reads internally; that guarantees we never both build the plan from the + // contact points (pre-RUNNING branch of newQueryPlan) and append them again below. + State state = stateRef.get(); Queue regularQueryPlan = newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - if (context - .getConfig() - .getDefaultProfile() - .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) { - Set originalNodes = context.getMetadataManager().getContactPoints(); + // Only append the contact points as an explicit fallback once the LBP is RUNNING: before that + // (BEFORE_INIT/DURING_INIT), newQueryPlan() above already built regularQueryPlan directly from + // the contact points, so appending them again here would just duplicate every entry. + // + // Skipped when the topology monitor re-resolves node addresses on its own (e.g. proxy-based + // monitors such as client routes or the cloud SNI proxy): those keep addresses fresh without + // this fallback, and appending raw contact points could resurrect nodes the monitor has + // authoritatively removed. The exception is an empty regular plan: with no live node to try, + // reconnection cannot recover on its own, so the contact-point fallback is kept even for those + // monitors. + if (state == State.RUNNING + && context + .getConfig() + .getDefaultProfile() + .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS) + && (!context.getTopologyMonitor().reresolvesNodeAddresses() + || regularQueryPlan.isEmpty())) { + // Append the original (unresolved) contact points so every IP their hostname resolves to is + // tried as a fallback: EndPoint.resolveAll() expands each one at connection time, instead of + // the driver being stuck with whatever single IP a metadata node happens to hold. List contactNodes = new ArrayList<>(); - for (DefaultNode node : originalNodes) { + for (DefaultNode node : context.getMetadataManager().getContactPoints()) { contactNodes.add(DefaultNode.newContactPoint(node.getEndPoint(), context)); } Collections.shuffle(contactNodes); - // Append contact points to the end of the regular query plan so they serve as a fallback - regularQueryPlan.addAll(contactNodes); + // Concatenate rather than mutate: the RUNNING-state regularQueryPlan is a built-in QueryPlan + // whose add()/addAll() throw UnsupportedOperationException (poll() is its only mutator). + // CompositeQueryPlan drains the regular plan first, then the contact-point fallback. + return new CompositeQueryPlan(regularQueryPlan, new SimpleQueryPlan(contactNodes.toArray())); } return regularQueryPlan; diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java index cd765c818e6..2ca9b6f1012 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java @@ -188,6 +188,13 @@ public boolean wasImplicitContactPoint() { * they are never added to metadata and never exposed to user-facing APIs (events, {@link * com.datastax.oss.driver.api.core.metadata.Metadata#getNodes()}, or {@link * com.datastax.oss.driver.api.core.metadata.NodeStateListener} callbacks). + * + *

The metadata node stores {@code nodeInfo.getEndPoint()} as-is and never re-resolves it on + * its own. Re-resolving the original contact-point hostname to pick up current DNS only happens + * through the original-contact-point reconnection fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}), which re-enters + * the contact points and lets {@link EndPoint#resolveAll(java.util.concurrent.Executor)} expand + * each hostname at connection time. */ public CompletionStage registerNode(NodeInfo nodeInfo) { Preconditions.checkNotNull(nodeInfo.getHostId(), "Cannot register node without hostId"); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java index 1bb8e343d96..420cd8e4f92 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java @@ -141,4 +141,24 @@ public interface TopologyMonitor extends AsyncAutoCloseable { * {@link DefaultTopologyMonitor}) should override this method. */ default void resetColumnCaches() {} + + /** + * Whether this monitor re-resolves node addresses dynamically on every connection attempt (for + * example by re-resolving a proxy hostname each time), rather than relying on an endpoint address + * captured once at node-registration time. + * + *

When this returns {@code true}, the control connection's reconnection query plan must not + * append the original contact points as a DNS re-resolution fallback (see {@code + * advanced.control-connection.reconnection.fallback-to-original-contact-points}): the monitor + * already keeps addresses fresh, and appending raw contact points could resurrect nodes that the + * monitor has authoritatively removed. + * + *

The default implementation returns {@code false}, which is correct for {@link + * DefaultTopologyMonitor}, whose {@code DefaultEndPoint}s cache their resolved address and never + * re-resolve. Proxy-based monitors that re-resolve per call should override this to return {@code + * true}. + */ + default boolean reresolvesNodeAddresses() { + return false; + } } diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 54f18e8eddf..fbde1277fc6 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -2342,14 +2342,20 @@ datastax-java-driver { } reconnection { - # Whether to forcibly add original contact points held by MetadataManager to the reconnection plan, - # in case there is no live nodes available according to LBP. - # Experimental. + # Whether to append the original contact points held by MetadataManager to the reconnection + # plan, after the live nodes reported by the load balancing policy. + # + # This is also the driver's DNS re-resolution path. Contact points are kept as unresolved + # hostnames and expanded to their current DNS IPs at connection time (see + # EndPoint.resolveAll()). Metadata nodes, in contrast, store an already-resolved endpoint that + # is never re-resolved, so once DNS records change they would otherwise become stale. Keeping + # this enabled lets control-connection reconnects re-resolve the original hostnames and pick up + # the new IPs once the live-node plan is exhausted. # # Required: yes # Modifiable at runtime: yes, the new value will be used for checks issued after the change. # Overridable in a profile: no - fallback-to-original-contact-points = false + fallback-to-original-contact-points = true } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java index a1ba4617ef5..0a6268f3378 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java @@ -28,6 +28,8 @@ import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.api.core.metadata.Metadata; +import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.adminrequest.AdminResult; import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; import com.datastax.oss.driver.internal.core.channel.DriverChannel; @@ -66,6 +68,8 @@ public class ClientRoutesTopologyMonitorTest { @Mock private ControlConnection controlConnection; @Mock private DriverConfig driverConfig; @Mock private DriverExecutionProfile defaultProfile; + @Mock private MetadataManager metadataManager; + @Mock private Metadata metadata; private TestableClientRoutesTopologyMonitor handler; @@ -220,6 +224,57 @@ public void should_refresh_updates_routes() throws UnknownHostException { assertThat(handler.resolve(hostId2)).isNotNull(); } + // ---- reresolvesNodeAddresses() ------------------------------------------- + + @Test + public void should_reresolve_when_all_known_nodes_have_client_routes() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + handler.setRoutes( + ImmutableMap.of( + hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042), + hostId2, new ClientRouteRecord(hostId2, "127.0.0.2", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isTrue(); + } + + @Test + public void should_not_reresolve_when_a_known_node_has_no_client_route() { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + Node node1 = Mockito.mock(Node.class); + when(node1.getHostId()).thenReturn(hostId1); + Node node2 = Mockito.mock(Node.class); + when(node2.getHostId()).thenReturn(hostId2); + + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(ImmutableMap.of(hostId1, node1, hostId2, node2)); + + // Only node1 has a live client route; node2 would fall back to a static endpoint. + handler.setRoutes(ImmutableMap.of(hostId1, new ClientRouteRecord(hostId1, "127.0.0.1", 9042))); + + assertThat(handler.reresolvesNodeAddresses()).isFalse(); + } + + @Test + public void should_reresolve_when_no_nodes_known_yet() { + when(context.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetadata()).thenReturn(metadata); + when(metadata.getNodes()).thenReturn(Collections.emptyMap()); + + assertThat(handler.reresolvesNodeAddresses()).isTrue(); + } + // ---- Merge behavior tests ----------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java index 89b36b9ee09..8c635982eea 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapperTest.java @@ -40,10 +40,11 @@ import com.datastax.oss.driver.internal.core.context.EventBus; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; +import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; +import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; -import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.Map; import java.util.Objects; import java.util.Queue; @@ -79,6 +80,7 @@ public class LoadBalancingPolicyWrapperTest { private EventBus eventBus; @Mock private MetadataManager metadataManager; @Mock private Metadata metadata; + @Mock private TopologyMonitor topologyMonitor; @Mock protected MetricsFactory metricsFactory; @Captor private ArgumentCaptor> initNodesCaptor; @@ -102,13 +104,16 @@ public void setup() { when(metadata.getNodes()).thenReturn(allNodes); when(metadataManager.getContactPoints()).thenReturn(contactPoints); when(context.getMetadataManager()).thenReturn(metadataManager); + when(context.getTopologyMonitor()).thenReturn(topologyMonitor); when(context.getConfig()).thenReturn(config); when(config.getDefaultProfile()).thenReturn(defaultProfile); when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(false); - defaultPolicyQueryPlan = Lists.newLinkedList(ImmutableList.of(node3, node2, node1)); + // Use a real built-in QueryPlan (not a mutable LinkedList): its add()/addAll() throw, so the + // control-reconnection plan must compose rather than mutate it (see CompositeQueryPlan usage). + defaultPolicyQueryPlan = new SimpleQueryPlan(node3, node2, node1); when(policy1.newQueryPlan(null, null)).thenReturn(defaultPolicyQueryPlan); eventBus = spy(new EventBus("test")); @@ -130,26 +135,28 @@ public void setup() { @Test public void should_build_control_connection_query_plan_from_contact_points_before_init() { - // When + // When — before init, the control-reconnection plan is built straight from the contact points + // (bypassing the load balancing policies), so each hostname can be tried on the first connect. Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test public void should_build_query_plan_from_contact_points_before_init() { - // When + // When — before init, the query plan is built straight from the contact points (bypassing the + // load balancing policies) Queue queryPlan = wrapper.newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); - // Then + // Then — query plan contains the contact points, and no policy was consulted for (LoadBalancingPolicy policy : ImmutableList.of(policy1, policy2, policy3)) { verify(policy, never()).newQueryPlan(null, null); } - assertThat(queryPlan).hasSameElementsAs(contactPoints); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); } @Test @@ -204,8 +211,7 @@ public void should_fetch_control_connection_query_plan_from_policy_after_init() assertThat(queryPlan.poll()).isEqualTo(node3); assertThat(queryPlan.poll()).isEqualTo(node2); assertThat(queryPlan.poll()).isEqualTo(node1); - // Remaining nodes are contact points appended at the end. - // They are new DefaultNode instances created via newContactPoint, so compare by endpoint. + // Remaining nodes are the original contact points appended at the end. Set remainingEndpoints = new java.util.HashSet<>(); for (Node n : queryPlan) { remainingEndpoints.add(n.getEndPoint()); @@ -217,14 +223,92 @@ public void should_fetch_control_connection_query_plan_from_policy_after_init() assertThat(remainingEndpoints).isEqualTo(contactEndpoints); } + @Test + public void should_not_duplicate_contact_points_before_init() { + // Given — the wrapper hasn't been init()-ed yet (state=BEFORE_INIT), so newQueryPlan() already + // builds the regular plan directly from the contact points. The reconnect-contact-points flag + // doesn't matter here: newControlReconnectionQueryPlan() short-circuits on state before even + // reading it, since appending contact points again pre-init would just duplicate every entry in + // the plan. + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the contact points are read only once (not once for the regular plan and again for a + // redundant "fallback" append), and the plan has no duplicate entries. + verify(metadataManager, times(1)).getContactPoints(); + assertThat(queryPlan).containsExactlyInAnyOrder(node1, node2); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_reconnect_contact_points_is_disabled() { + // Given — the flag defaults to false in the test setup (see @Before) + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Only the policy query plan is returned; no contact points are appended. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_not_append_contact_points_to_query_plan_when_topology_monitor_reresolves_addresses() { + // Given — the flag is enabled, but the topology monitor re-resolves node addresses on its own + // (e.g. a proxy-based monitor such as client routes or the cloud SNI proxy). + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then + // Contact points must not be appended: the monitor keeps addresses fresh, and appending raw + // contact points could resurrect nodes it has authoritatively removed. + assertThat(queryPlan).isEqualTo(defaultPolicyQueryPlan); + } + + @Test + public void + should_append_contact_points_when_query_plan_empty_even_if_topology_monitor_reresolves() { + // Given — the flag is enabled and the topology monitor re-resolves node addresses on its own, + // but the live-node query plan is empty. With no node to try, reconnection can only recover + // through the contact-point fallback, so it must be appended despite the re-resolving monitor. + when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) + .thenReturn(true); + when(topologyMonitor.reresolvesNodeAddresses()).thenReturn(true); + wrapper.init(); + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); + + // When + Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); + + // Then — the contact points are appended (compare by endpoint since they are new instances) + assertThat(queryPlan.size()).isEqualTo(contactPoints.size()); + Set resultEndpoints = new java.util.HashSet<>(); + for (Node n : queryPlan) { + resultEndpoints.add(n.getEndPoint()); + } + Set contactEndpoints = new java.util.HashSet<>(); + for (DefaultNode n : contactPoints) { + contactEndpoints.add(n.getEndPoint()); + } + assertThat(resultEndpoints).isEqualTo(contactEndpoints); + } + @Test public void should_return_contact_points_when_query_plan_empty_and_flag_enabled() { // Given when(defaultProfile.getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) .thenReturn(true); wrapper.init(); - // Make the policy return an empty query plan - when(policy1.newQueryPlan(null, null)).thenReturn(Lists.newLinkedList(ImmutableList.of())); + // Make the policy return an empty query plan (QueryPlan.EMPTY, as the real policies do) + when(policy1.newQueryPlan(null, null)).thenReturn(QueryPlan.EMPTY); // When Queue queryPlan = wrapper.newControlReconnectionQueryPlan(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java index 26658bd76d1..b8a32c68450 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/heartbeat/HeartbeatIT.java @@ -235,6 +235,10 @@ private CqlSession newSession(ProgrammaticDriverConfigLoaderBuilder loaderBuilde .withDuration(DefaultDriverOption.HEARTBEAT_TIMEOUT, Duration.ofMillis(500)) .withDuration(DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT, Duration.ofSeconds(2)) .withDuration(DefaultDriverOption.RECONNECTION_MAX_DELAY, Duration.ofSeconds(1)) + // These tests exercise heartbeat behavior only. Disable the contact-point + // reconnection fallback, which would otherwise send an extra OPTIONS message on + // init/reconnect and skew the heartbeat counts. + .withBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false) .build(); return SessionUtils.newSession(SIMULACRON_RULE, loader); } From 19359f958c3870a577042a4ab5dc6c8255a70a42 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 23 Jul 2026 23:25:38 +0200 Subject: [PATCH 06/33] docs: document contact-point DNS expansion in the upgrade guide (DRIVER-201) Add a 4.19.2.1 upgrade-guide entry covering the multi-address contact-point expansion, the deprecation of advanced.resolve-contact-points (now a no-op), and the fallback-to-original-contact-points default flip (false -> true) with how to restore the previous behavior. Co-Authored-By: Claude Opus 4.8 --- upgrade_guide/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index 214399dacc7..f1dfdceb82e 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -19,6 +19,27 @@ under the License. ## Upgrade guide +### 4.19.2.1 + +#### Contact points are expanded to all their DNS addresses at connection time + +Contact points backed by a hostname are now kept unresolved and expanded to **all** the IP +addresses the hostname maps to, at connection time (via `EndPoint.resolveAll()`). Previously only +the first address returned by DNS was tried, so a single non-responsive IP behind a multi-record +hostname could fail the initial connection (or a control-connection reconnect) even when the other +addresses were healthy. No configuration change is required to benefit from this. + +As part of this change: + +- `advanced.resolve-contact-points` is deprecated and now has **no effect**. Contact points are + always kept as unresolved hostnames and expanded at connection time. An already-resolved + `InetSocketAddress` passed programmatically is still used as provided, with no further expansion. +- `advanced.control-connection.reconnection.fallback-to-original-contact-points` now defaults to + `true` (previously `false`). This is also the driver's DNS re-resolution path: metadata nodes + hold an already-resolved endpoint that is never re-resolved, so on control-connection reconnect + the driver falls back to the original contact points to pick up current DNS records once the + live-node plan is exhausted. Set it to `false` to restore the previous behavior. + ### 4.19.0.7 #### Cloud private-endpoint support via client routes From ecd9497e49648a09be2bde25b0b59ac464877272 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 24 Jul 2026 00:05:13 +0200 Subject: [PATCH 07/33] fix: rotate SNI resolveAll() with a dedicated counter (DRIVER-201) resolveAll() shared the round-robin OFFSET counter with the deprecated resolve(). SniSslEngineFactory.newSslEngine() calls resolve() on every TLS connection, so a normal SNI-over-TLS connect advanced the counter twice and resolveAll()'s start index moved in steps of two -- collapsing rotation to a single proxy IP whenever the hostname resolved to an even number of A-records (index 0 in the common two-record case). Give resolveAll() its own RESOLVE_ALL_OFFSET so SSL engine setup no longer perturbs its rotation, and add a test that interleaves resolve() calls to lock this in. Co-Authored-By: Claude Opus 4.8 --- .../internal/core/metadata/SniEndPoint.java | 17 +++++++--- .../core/metadata/SniEndPointTest.java | 32 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index acd3431580e..a141ef1576b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -33,7 +33,14 @@ import java.util.concurrent.atomic.AtomicInteger; public class SniEndPoint implements EndPoint { + // Rotates the single address returned by resolve() (still used for SSL engine setup). private static final AtomicInteger OFFSET = new AtomicInteger(); + // Rotates the starting candidate of resolveAll(). Kept separate from OFFSET so that SSL engine + // creation (which calls the deprecated resolve() once per connection) does not advance the + // resolveAll() counter a second time -- otherwise the start index would move in steps of 2 and + // rotation would collapse to a single IP whenever the proxy resolves to an even number of + // A-records (DRIVER-201). + private static final AtomicInteger RESOLVE_ALL_OFFSET = new AtomicInteger(); private final InetSocketAddress proxyAddress; private final String serverName; @@ -86,9 +93,11 @@ public InetSocketAddress resolve() { * A-record, so that the driver can try every proxy IP in sequence if one is unreachable. * *

All A-records are returned so a single connection attempt can fall back across every proxy - * IP, but the candidate order is rotated on each call using the same round-robin {@link #OFFSET} - * counter as {@link #resolve()}. This spreads healthy connections across proxy IPs instead of - * always starting at the first one, preserving the previous load-balancing behavior. + * IP, but the candidate order is rotated on each call using a dedicated round-robin counter + * ({@link #RESOLVE_ALL_OFFSET}). This spreads healthy connections across proxy IPs instead of + * always starting at the first one, preserving the previous load-balancing behavior. The counter + * is intentionally separate from the one used by {@link #resolve()} so that SSL engine setup + * (which calls {@code resolve()} once per connection) does not perturb this rotation. * *

The blocking DNS lookup is run on {@code executor} so it never blocks the calling (event * loop) thread; the returned stage completes with the rotated candidate array, or completes @@ -112,7 +121,7 @@ public CompletionStage resolveAll(@NonNull Executor executor) { int start = (aRecords.length == 1) ? 0 - : OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) + : RESOLVE_ALL_OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) % aRecords.length; SocketAddress[] result = new SocketAddress[aRecords.length]; for (int i = 0; i < aRecords.length; i++) { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java index c7f5bb695e8..99672006747 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -98,4 +98,36 @@ public void resolve_all_returns_complete_and_rotated_candidate_order() { .isTrue(); } } + + @Test + public void resolve_all_rotation_is_not_disturbed_by_interleaved_resolve() { + // resolveAll() rotates using a counter independent from resolve(). SSL engine setup calls the + // deprecated resolve() once per connection; if the two shared a counter, that extra advance + // would pin resolveAll()'s start index to 0 whenever the proxy resolves to an even number of + // A-records (DRIVER-201). Interleaving resolve() calls here must not stop resolveAll() from + // rotating. On single-address environments (size == 1) rotation is a no-op, matching the + // conditional assertion in resolve_all_returns_complete_and_rotated_candidate_order(). + SniEndPoint endPoint = + new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); + + SocketAddress[] first = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + int size = first.length; + SocketAddress firstStart = first[0]; + + boolean sawRotation = false; + for (int call = 0; call < size * 2; call++) { + SocketAddress[] candidates = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + if (!candidates[0].equals(firstStart)) { + sawRotation = true; + } + // Interleave a resolve() call, as the SSL engine does on every connection. + endPoint.resolve(); + } + + if (size > 1) { + assertThat(sawRotation) + .as("resolveAll() rotation must survive interleaved resolve() calls") + .isTrue(); + } + } } From 4bf038c5bf7e30cc1db75fcdf565d3f3ee827234 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 24 Jul 2026 00:05:22 +0200 Subject: [PATCH 08/33] fix: aggregate multi-address connect failures; cover fallback with tests (DRIVER-201) When every candidate address from resolveAll() failed, ChannelFactory propagated only the last candidate's error and discarded the earlier ones (they were logged at DEBUG only). Attach the earlier failures as suppressed exceptions on the propagated error so the full picture is available for diagnosis. Add ChannelFactoryMultiAddressTest, which the existing single-address tests did not cover: first-candidate-fails/second-succeeds fallback, and the all-candidates-exhausted path (asserting the suppressed cause). Strengthen DefaultEndPointTest to assert resolveAll() expands to the complete DNS result set (compared against an independent InetAddress.getAllByName lookup), not just a non-empty subset. Co-Authored-By: Claude Opus 4.8 --- .../internal/core/channel/ChannelFactory.java | 23 +++- .../ChannelFactoryMultiAddressTest.java | 117 ++++++++++++++++++ .../core/metadata/DefaultEndPointTest.java | 23 +++- 3 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index d2e05ebef85..c0e6276343b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -58,6 +58,7 @@ import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketAddress; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -290,7 +291,8 @@ private void connect( attemptedVersions, resultFuture, candidates, - 0); + 0, + new ArrayList<>()); }); } @@ -304,6 +306,11 @@ private void connect( * {@code N × connect-timeout} where N is the number of candidates. This is an intentional * tradeoff: failing immediately on the first unreachable IP would prevent fallback to healthy * ones. In practice DNS entries have only a small number of records. + * + *

When every candidate fails, the last candidate's error is propagated with each earlier + * candidate's failure attached as a {@linkplain Throwable#addSuppressed(Throwable) suppressed} + * exception, so the full set of per-address causes is visible for diagnosis (they are otherwise + * only logged at DEBUG). */ private void tryNextCandidate( EndPoint endPoint, @@ -316,7 +323,8 @@ private void tryNextCandidate( List attemptedVersions, CompletableFuture resultFuture, SocketAddress[] candidates, - int index) { + int index, + List priorErrors) { SocketAddress candidate = candidates[index]; CompletableFuture perAddressFuture = new CompletableFuture<>(); @@ -342,6 +350,7 @@ private void tryNextCandidate( logPrefix, candidate, error.getMessage()); + priorErrors.add(error); tryNextCandidate( endPoint, shardingInfo, @@ -353,8 +362,16 @@ private void tryNextCandidate( attemptedVersions, resultFuture, candidates, - index + 1); + index + 1, + priorErrors); } else { + // All candidates exhausted. Surface the last error, carrying the earlier failures as + // suppressed exceptions so they are not lost (they were only logged at DEBUG above). + for (Throwable priorError : priorErrors) { + if (priorError != error) { + error.addSuppressed(priorError); + } + } // Note: might be completed already if the failure happened in initializer() resultFuture.completeExceptionally(error); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java new file mode 100644 index 00000000000..5e56c7c07c9 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.channel.local.LocalAddress; +import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import org.junit.Test; + +/** + * Verifies that {@link ChannelFactory#connect} tries every candidate returned by {@link + * EndPoint#resolveAll(java.util.concurrent.Executor)} in sequence: it falls back to the next + * address when one is unreachable, and only fails the overall future once all candidates are + * exhausted, carrying the earlier failures as suppressed exceptions. + */ +public class ChannelFactoryMultiAddressTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE_1 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-1"); + private static final SocketAddress UNREACHABLE_2 = + new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-2"); + + @Test + public void should_fall_back_to_next_candidate_when_first_is_unreachable() { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + // First candidate is unreachable, second is the running local server. + EndPoint endPoint = endPointReturning(UNREACHABLE_1, SERVER_ADDRESS.resolve()); + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + // The handshake only happens once we fall back to the reachable second candidate. + completeSimpleChannelInit(); + + // Then – the connection succeeds via the second candidate. + assertThatStage(channelFuture).isSuccess(); + } + + @Test + public void should_fail_with_suppressed_causes_when_all_candidates_are_unreachable() { + // Given + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + EndPoint endPoint = endPointReturning(UNREACHABLE_1, UNREACHABLE_2); + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + // Then – the future fails, and the earlier candidate's failure is preserved as a suppressed + // exception on the last candidate's error (rather than being silently dropped). + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e.getSuppressed()) + .as("earlier candidate failures should be attached as suppressed exceptions") + .isNotEmpty()); + } + + /** An endpoint whose {@link EndPoint#resolveAll} yields the given candidates, in order. */ + private static EndPoint endPointReturning(SocketAddress... candidates) { + return new EndPoint() { + @NonNull + @Override + public SocketAddress resolve() { + return candidates[candidates.length - 1]; + } + + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + return CompletableFuture.completedFuture(candidates.clone()); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + }; + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index 66b794cdcf8..404de8f7f4c 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -20,8 +20,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.util.HashSet; +import java.util.Set; import org.junit.Test; public class DefaultEndPointTest { @@ -69,17 +73,30 @@ public void resolve_all_returns_single_element_for_already_resolved_address() { } @Test - public void resolve_all_expands_unresolved_hostname_to_at_least_one_address() { - // localhost reliably resolves to at least 127.0.0.1 + public void resolve_all_expands_unresolved_hostname_to_all_dns_ips() throws UnknownHostException { + // localhost reliably resolves to at least 127.0.0.1 (and possibly ::1). DefaultEndPoint endPoint = new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - assertThat(all).isNotEmpty(); + + // The complete DNS result set must be expanded -- one resolved InetSocketAddress per record -- + // so a connection attempt can fall back across every IP, not just the first. Compare against an + // independent getAllByName() lookup so the test fails if production returned only a subset. + InetAddress[] expectedIps = InetAddress.getAllByName("localhost"); + Set expected = new HashSet<>(); + for (InetAddress ip : expectedIps) { + expected.add(new InetSocketAddress(ip, 9042)); + } + Set actual = new HashSet<>(); for (SocketAddress addr : all) { InetSocketAddress inet = (InetSocketAddress) addr; assertThat(inet.isUnresolved()).isFalse(); assertThat(inet.getPort()).isEqualTo(9042); + actual.add(inet); } + assertThat(all).hasSize(expectedIps.length); + assertThat(actual).isEqualTo(expected); } @Test From 33be4762ce93dac402fdadb9b4ec0c4bfdab2866 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 24 Jul 2026 00:05:29 +0200 Subject: [PATCH 09/33] docs: fix contradictory resolve-contact-points doc; clarify reconnect option (DRIVER-201) The reference.conf entry for advanced.resolve-contact-points still described the old resolve-once/resolve-every-connection semantics as active above a "DEPRECATED: no effect" footer, contradicting itself. Lead with the deprecation and drop the stale active-voice prose. Clarify the CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS Javadoc in TypedDriverOption: it appends the original (unresolved) contact points, expands them to all DNS IPs at connection time via EndPoint.resolveAll(), and is skipped for topology monitors that re-resolve node addresses themselves. Co-Authored-By: Claude Opus 4.8 --- .../api/core/config/TypedDriverOption.java | 10 +++++- core/src/main/resources/reference.conf | 33 +++++++------------ 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index b01ff43aa16..21c5f68d3d1 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -600,7 +600,15 @@ public String toString() { public static final TypedDriverOption CONTROL_CONNECTION_AGREEMENT_WARN = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN); - /** Whether to append the original contact points to the reconnection plan (defaults to true) */ + /** + * Whether to append the original contact points to the control-connection reconnection plan, + * after the live nodes reported by the load balancing policy (defaults to {@code true}). + * + *

Contact points are appended as-is (unresolved hostnames); each is expanded to all of its + * current DNS IPs at connection time via {@code EndPoint.resolveAll()}, which is also the + * driver's DNS re-resolution mechanism. The append is skipped for topology monitors that + * re-resolve node addresses themselves (such as the cloud/proxy monitors). + */ public static final TypedDriverOption CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS = new TypedDriverOption<>( DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN); diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index fbde1277fc6..2ce07eb48c3 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1224,33 +1224,22 @@ datastax-java-driver { } - # Whether to resolve the addresses passed to `basic.contact-points`. + # DEPRECATED: this option no longer has any effect and will be removed in a future release. # - # If this is true, addresses are created with `InetSocketAddress(String, int)`: the host name will - # be resolved the first time, and the driver will use the resolved IP address for all subsequent - # connection attempts. + # Contact points are now always kept as unresolved hostnames and expanded to all of their + # DNS-mapped IPs lazily at connection time (see EndPoint.resolveAll()). This means the driver + # tries every IP a hostname resolves to, and re-resolves the hostname on each new connection so + # DNS changes are picked up automatically. Previously this option selected between resolving a + # contact-point hostname once (true) and re-resolving it on every connection (false); that + # distinction no longer applies. # - # If this is false, addresses are created with `InetSocketAddress.createUnresolved()`: the host - # name will be resolved again every time the driver opens a new connection. This is useful for - # containerized environments where DNS records are more likely to change over time (note that the - # JVM and OS have their own DNS caching mechanisms, so you might need additional configuration - # beyond the driver). + # This option only ever applied to the contact points specified in the configuration -- never to + # programmatic contact points passed to SessionBuilder.addContactPoints, nor to dynamically + # discovered peers. # - # This option only applies to the contact points specified in the configuration. It has no effect - # on: - # - programmatic contact points passed to SessionBuilder.addContactPoints: these addresses are - # built outside of the driver, so it is your responsibility to provide unresolved instances. - # - dynamically discovered peers: the driver relies on Cassandra system tables, which expose raw - # IP addresses. Use a custom address translator to convert them to unresolved addresses (if - # you're in a containerized environment, you probably already need address translation anyway). - # - # Required: no (defaults to false) + # Required: no # Modifiable at runtime: no # Overridable in a profile: no - # - # DEPRECATED: this option no longer has any effect. Contact points are always kept as unresolved - # hostnames and expanded to all their DNS-mapped IPs at connection time (see - # EndPoint.resolveAll()). It will be removed in a future release. advanced.resolve-contact-points = false advanced.protocol { From d3b39786377cd5fb517f095cd92b1fb4d6e88ccd Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Mon, 27 Jul 2026 15:02:31 +0200 Subject: [PATCH 10/33] fix: bound the DNS-resolver thread pool in ChannelFactory (DRIVER-201) resolverExecutor was an unbounded Executors.newCachedThreadPool(), so a large simultaneous reconnect burst across many nodes could spin up an unbounded number of resolver threads. Replace it with a fixed-size ThreadPoolExecutor (RESOLVER_MAX_THREADS = 16) backed by an unbounded LinkedBlockingQueue with allowCoreThreadTimeOut(true): submissions are never rejected and never run inline on the caller (admin-loop) thread, excess resolutions simply queue until a pool thread frees up, and idle threads still reclaim after ~60s as before. --- .../internal/core/channel/ChannelFactory.java | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index c0e6276343b..3414fa13ad6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -67,7 +67,9 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import net.jcip.annotations.ThreadSafe; @@ -111,10 +113,20 @@ public class ChannelFactory { private final String logPrefix; protected final InternalDriverContext context; + /** + * Maximum number of threads used to run {@link + * EndPoint#resolveAll(java.util.concurrent.Executor)} calls concurrently (see {@link + * #resolverExecutor}). DNS resolution is I/O-bound, and a fixed bound avoids unbounded thread + * creation during a large simultaneous reconnect burst while still giving plenty of headroom (DNS + * lookups are fast; contention here should be rare). + */ + private static final int RESOLVER_MAX_THREADS = 16; + // Runs potentially-blocking EndPoint name resolution (EndPoint.resolveAll()) off the caller - // thread: connect() is invoked from the admin event loop, which must never block on DNS. A cached - // pool gives each concurrent resolution its own daemon thread, so a single blackholed lookup - // cannot starve other nodes' reconnections; idle threads are reclaimed after ~60s. + // thread: connect() is invoked from the admin event loop, which must never block on DNS. Bounded + // at RESOLVER_MAX_THREADS threads with an unbounded queue, so a submission is never rejected and + // never runs inline on the caller -- excess resolutions simply queue until a thread frees up. + // Idle threads are reclaimed after ~60s. private final ExecutorService resolverExecutor; /** either set from the configuration, or null and will be negotiated */ @@ -135,8 +147,13 @@ public ChannelFactory(InternalDriverContext context) { this.context = context; AtomicInteger resolverThreadCount = new AtomicInteger(); - this.resolverExecutor = - Executors.newCachedThreadPool( + ThreadPoolExecutor resolverThreadPool = + new ThreadPoolExecutor( + RESOLVER_MAX_THREADS, + RESOLVER_MAX_THREADS, + 60L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), runnable -> { Thread thread = new Thread( @@ -145,6 +162,8 @@ public ChannelFactory(InternalDriverContext context) { thread.setDaemon(true); return thread; }); + resolverThreadPool.allowCoreThreadTimeOut(true); + this.resolverExecutor = resolverThreadPool; DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); if (defaultConfig.isDefined(DefaultDriverOption.PROTOCOL_VERSION)) { From a6c554d913ddf38af9129e318ad110b290154d07 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Mon, 27 Jul 2026 15:02:48 +0200 Subject: [PATCH 11/33] fix: scope protocol-version negotiation history per candidate address (DRIVER-201) attemptedVersions was created once per connect() call and threaded unchanged through every candidate address tried by tryNextCandidate(). If candidate #1 exhausted protocol-downgrade negotiation before falling back to candidate #2, and #2 also exhausted negotiation, the final UnsupportedProtocolVersionException reported a version-history conflated from two different IPs. Construct a fresh list per candidate inside tryNextCandidate() instead of threading one down from connect(); connectToAddress()'s own downgrade-retry recursion (correctly scoped to a single address) is unaffected. --- .../driver/internal/core/channel/ChannelFactory.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 3414fa13ad6..bb58a5f54fe 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -235,7 +235,6 @@ CompletionStage connect( ProtocolVersion currentVersion; boolean isNegotiating; - List attemptedVersions = new CopyOnWriteArrayList<>(); if (this.protocolVersion != null) { currentVersion = protocolVersion; isNegotiating = false; @@ -252,7 +251,6 @@ CompletionStage connect( nodeMetricUpdater, currentVersion, isNegotiating, - attemptedVersions, resultFuture); return resultFuture; } @@ -265,7 +263,6 @@ private void connect( NodeMetricUpdater nodeMetricUpdater, ProtocolVersion currentVersion, boolean isNegotiating, - List attemptedVersions, CompletableFuture resultFuture) { // Resolution may block (DNS); it is offloaded to resolverExecutor so the calling thread (the @@ -307,7 +304,6 @@ private void connect( nodeMetricUpdater, currentVersion, isNegotiating, - attemptedVersions, resultFuture, candidates, 0, @@ -339,7 +335,6 @@ private void tryNextCandidate( NodeMetricUpdater nodeMetricUpdater, ProtocolVersion currentVersion, boolean isNegotiating, - List attemptedVersions, CompletableFuture resultFuture, SocketAddress[] candidates, int index, @@ -347,6 +342,10 @@ private void tryNextCandidate( SocketAddress candidate = candidates[index]; CompletableFuture perAddressFuture = new CompletableFuture<>(); + // Fresh per candidate address: connectToAddress()'s downgrade retries stay on this one + // address, so the final UnsupportedProtocolVersionException (if negotiation is what dooms + // this candidate) only reports versions actually tried against it, not earlier candidates'. + List attemptedVersions = new CopyOnWriteArrayList<>(); connectToAddress( endPoint, shardingInfo, @@ -378,7 +377,6 @@ private void tryNextCandidate( nodeMetricUpdater, currentVersion, isNegotiating, - attemptedVersions, resultFuture, candidates, index + 1, From dc40c041696e7ec82ad30f03e20372cbc64e1470 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Mon, 27 Jul 2026 15:03:02 +0200 Subject: [PATCH 12/33] docs: clarify control-reconnection race window and reresolvesNodeAddresses javadoc (DRIVER-201) LoadBalancingPolicyWrapper.newControlReconnectionQueryPlan(): document the narrow window between the outer captured `state` read and newQueryPlan()'s own internal stateRef read -- a transition landing between them can skip the contact-point fallback for one reconnection attempt. Benign (no crash, no duplicate entries, self-corrects on the next attempt), but worth spelling out next to the existing "state is monotonic" reasoning. TopologyMonitor.reresolvesNodeAddresses(): tighten the javadoc claim that DefaultEndPoints "cache their resolved address and never re-resolve" -- true for a peer node's already-resolved physical IP, but a node whose EndPoint originated from an unresolved hostname does re-resolve via EndPoint.resolveAll() on every connect() call, independent of this flag. --- .../core/metadata/LoadBalancingPolicyWrapper.java | 8 ++++++++ .../driver/internal/core/metadata/TopologyMonitor.java | 9 ++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index fc1390f4ac2..1bee460d389 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -172,6 +172,14 @@ public Queue newControlReconnectionQueryPlan() { // (BEFORE_INIT -> DURING_INIT -> RUNNING -> ...), so this captured value is <= the value // newQueryPlan() reads internally; that guarantees we never both build the plan from the // contact points (pre-RUNNING branch of newQueryPlan) and append them again below. + // + // Note: this is still two separate reads of stateRef (this one, and newQueryPlan()'s own + // internal read a moment later), so a transition landing exactly between them is possible: if + // state flips BEFORE_INIT/DURING_INIT -> RUNNING in that window, newQueryPlan() takes the + // RUNNING branch (a real LBP-built plan) while the state captured here is still pre-RUNNING, + // so the contact-point fallback below is skipped for this one call even though + // regularQueryPlan didn't come from the contact-point branch. This is benign: no crash, no + // duplicate entries, and it self-corrects on the very next reconnection attempt. State state = stateRef.get(); Queue regularQueryPlan = newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java index 420cd8e4f92..cf00fd39b32 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java @@ -154,9 +154,12 @@ default void resetColumnCaches() {} * monitor has authoritatively removed. * *

The default implementation returns {@code false}, which is correct for {@link - * DefaultTopologyMonitor}, whose {@code DefaultEndPoint}s cache their resolved address and never - * re-resolve. Proxy-based monitors that re-resolve per call should override this to return {@code - * true}. + * DefaultTopologyMonitor}: the peer nodes it registers hold a {@code DefaultEndPoint} built from + * an already-resolved physical IP (from {@code system.peers}), which never needs re-resolving. + * This is separate from the connected node's own {@code EndPoint}, which may originate from an + * unresolved contact-point hostname -- that one does re-resolve via {@code EndPoint.resolveAll()} + * on every connection attempt, independently of this flag. Proxy-based monitors that re-resolve + * per call should override this to return {@code true}. */ default boolean reresolvesNodeAddresses() { return false; From 46ba69b889387a8f425ac1d7ee19274acc6a7907 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Mon, 27 Jul 2026 15:03:18 +0200 Subject: [PATCH 13/33] test: remove vestigial RESOLVE_CONTACT_POINTS config from MockResolverIT (DRIVER-201) advanced.resolve-contact-points is now a documented no-op (contact points are always kept unresolved and expanded via EndPoint.resolveAll() at connection time). Remove the now-dead .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS..., false) line from all four MockResolverIT test methods that set it, so a future reader isn't misled into thinking it's load-bearing. --- .../com/datastax/oss/driver/core/resolver/MockResolverIT.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java index 730827b2537..cb4f1abfa5c 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java @@ -84,7 +84,6 @@ public void should_connect_with_mocked_hostname() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -119,7 +118,6 @@ public void should_connect_when_first_dns_entry_is_non_responsive() { final int numberOfNodes = 2; DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -158,7 +156,6 @@ public void replace_cluster_test() { final int numberOfNodes = 3; DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), @@ -245,7 +242,6 @@ public void run_replace_test_20_times() { public void cannot_reconnect_with_resolved_socket() { DriverConfigLoader loader = new DefaultProgrammaticDriverConfigLoaderBuilder() - .withBoolean(TypedDriverOption.RESOLVE_CONTACT_POINTS.getRawOption(), false) .withBoolean(TypedDriverOption.RECONNECT_ON_INIT.getRawOption(), true) .withStringList( TypedDriverOption.CONTACT_POINTS.getRawOption(), From 648c824aac04c326c110517adef853cbd94f367d Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Wed, 29 Jul 2026 17:10:53 +0200 Subject: [PATCH 14/33] fix: resolve candidates via Netty's resolver and pin the connected address (DRIVER-201) Addresses dkropachev's second review round. Five of the six comments trace back to two root causes, fixed here together because they meet in ChannelFactory: once it performs the address expansion itself, it also knows which concrete address a connection landed on. Resolve through Netty's AddressResolverGroup -------------------------------------------- DefaultEndPoint.resolveAll() no longer calls InetAddress.getAllByName(); it returns its address -- resolved or not -- as a single candidate. ChannelFactory expands unresolved candidates through the bootstrap's AddressResolverGroup, so a custom resolver installed via NettyOptions.afterBootstrapInitialized() is honoured again. That is the resolver an unresolved address already reached when it was handed straight to Bootstrap.connect(), so resolving anywhere else silently bypassed the user's configuration. Mirrors Bootstrap#doResolveAndConnect0: candidates the resolver does not support (LocalAddress) or that are already resolved (metadata nodes, whose endpoints hold addresses from the peers rows) pass through untouched, and a null group -- Bootstrap.disableResolver() -- is respected. A candidate that fails to resolve is skipped rather than failing the whole attempt; only an all-candidates failure fails the connect. The bootstrap is now built once per connect() instead of once per candidate, since it is the only handle on the resolver group; each attempt uses a clone() with its own handler. As a side effect the afterBootstrapInitialized() hook runs once per logical connection rather than once per address attempt. With Netty's default resolver the lookup blocks the I/O event loop it runs on, because DefaultNameResolver performs the JDK lookup inline. That is the pre-existing behaviour of handing an unresolved address to Bootstrap.connect(); the admin event loop -- the one control-connection reconnects run on, and the reason resolution was made async in the first place -- is still never blocked. Deployments needing non-blocking resolution can install DnsAddressResolverGroup and now have it take effect. Pin the connected address onto the channel ------------------------------------------ New internal PinnableEndPoint: a copy of an endpoint bound to one address. DefaultEndPoint, SniEndPoint and ClientRoutesEndPoint implement it with a nullable pinnedAddress excluded from equals/hashCode/asMetricPrefix, so a pinned copy denotes the same node and metric names do not change with the IP a connection happened to use. Equality stays symmetric, which a delegating wrapper could not offer -- endpoints are set and map keys. ChannelFactory hands the pinned copy to the channel initializer and the DriverChannel. Three consequences: - Node identity: once a node is known by host id it keeps reconnecting to the IP it was identified at. Previously DefaultTopologyMonitor#buildNodeEndPoint could store a shared multi-address endpoint for system.local, and since ControlConnection skips identity re-resolution for nodes that already have a host id, a later reconnect could reach a different node while still being treated as the original. - SniSslEngineFactory#newSslEngine() runs inside Netty's channel initializer. resolve() is now a field read there instead of a blocking getAllByName() on an event loop, and it returns the very proxy IP the channel is connected to. - GSSAPI: the authenticator receives a resolved endpoint, so getAddress().getCanonicalHostName() no longer NPEs on a contact point that is kept unresolved. A null-safe fallback to getHostString() is added anyway, for third-party endpoints that cannot be pinned. Endpoints that do not implement PinnableEndPoint are passed through unchanged, so third-party implementations behave exactly as before. Also in this change ------------------- - ClientRoutesEndPoint.resolveAll() runs topologyMonitor.resolve() on the supplied executor instead of the caller path -- it can reach InetAddress.getByName() -- and delegates to fallbackEndPoint.resolveAll() when there is no route, rather than flattening it to resolve(). - The resolver thread pool follows advanced.netty.daemon like every other driver thread, instead of hardcoding daemon threads. close() is what lets the JVM exit under the default non-daemon setting; its javadoc no longer claims otherwise. - Docs updated where they described expansion as happening inside the endpoint via JVM DNS: reference.conf, the upgrade guide, SessionBuilder and EndPoint.resolveAll()'s contract, which now states that returning a hostname is expected. The client-routes manual no longer says resolution blocks Netty I/O threads. Tests: DefaultEndPointTest covers the no-lookup contract and pinning identity in both directions; ChannelFactoryNettyResolverTest asserts a custom resolver is consulted, that all the addresses it returns are tried, that already-resolved candidates are left alone and that disableResolver() is respected; ChannelFactoryPinnedEndPointTest asserts the channel carries the address that connected while still equalling the original, and that non-pinnable endpoints are untouched; SniEndPointTest and ClientRoutesEndPointTest cover pinning and the executor hop. Co-Authored-By: Claude Opus 5 (1M context) --- .../core/auth/DseGssApiAuthProviderBase.java | 24 +- .../driver/api/core/metadata/EndPoint.java | 16 +- .../api/core/session/SessionBuilder.java | 16 +- .../internal/core/channel/ChannelFactory.java | 290 ++++++++++++++---- .../core/metadata/ClientRoutesEndPoint.java | 76 ++++- .../core/metadata/DefaultEndPoint.java | 81 ++--- .../core/metadata/PinnableEndPoint.java | 69 +++++ .../internal/core/metadata/SniEndPoint.java | 58 +++- core/src/main/resources/reference.conf | 19 +- .../ChannelFactoryNettyResolverTest.java | 269 ++++++++++++++++ .../ChannelFactoryPinnedEndPointTest.java | 166 ++++++++++ .../core/channel/ChannelFactoryTestBase.java | 3 + .../metadata/ClientRoutesEndPointTest.java | 99 ++++++ .../core/metadata/DefaultEndPointTest.java | 118 +++++-- .../core/metadata/SniEndPointTest.java | 26 ++ manual/core/address_resolution/README.md | 5 +- upgrade_guide/README.md | 15 +- 17 files changed, 1192 insertions(+), 158 deletions(-) create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java diff --git a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java index 593552d77c1..8d913127355 100644 --- a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java +++ b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java @@ -27,6 +27,7 @@ import com.datastax.oss.protocol.internal.util.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedActionException; @@ -291,8 +292,6 @@ protected static class GssApiAuthenticator extends BaseDseAuthenticator { private SaslClient saslClient; private EndPoint endPoint; - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); - // Kerberos authentication needs a single canonical hostname for SASL service name resolution. protected GssApiAuthenticator( GssApiOptions options, EndPoint endPoint, String serverAuthenticator) { super(serverAuthenticator); @@ -321,7 +320,7 @@ protected GssApiAuthenticator( SUPPORTED_MECHANISMS, options.getAuthorizationId(), protocol, - ((InetSocketAddress) endPoint.resolve()).getAddress().getCanonicalHostName(), + serverName(endPoint), options.getSaslProperties(), null); } catch (LoginException | SaslException e) { @@ -330,6 +329,25 @@ protected GssApiAuthenticator( this.endPoint = endPoint; } + /** + * The host name to build the Kerberos service principal from. + * + *

Prefers the canonical name of the resolved address, which is what Kerberos expects. The + * driver's own endpoints always hand this a resolved address — the channel carries an endpoint + * bound to the address it connected to (see {@code PinnableEndPoint}) — but a custom {@link + * EndPoint} implementation may still yield an unresolved one, in which case {@code + * getAddress()} is null. Fall back to the host string rather than throwing a {@link + * NullPointerException}: the hostname is usually the right service name anyway, and a failed + * reverse lookup should not take authentication down. + */ + @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); + // Kerberos authentication needs a single canonical hostname for SASL service name resolution. + private static String serverName(EndPoint endPoint) { + InetSocketAddress address = (InetSocketAddress) endPoint.resolve(); + InetAddress inetAddress = address.getAddress(); + return inetAddress != null ? inetAddress.getCanonicalHostName() : address.getHostString(); + } + @NonNull @Override protected ByteBuffer getMechanism() { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java index b4c6a03286b..54407b6c0e8 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java @@ -49,14 +49,24 @@ public interface EndPoint { SocketAddress resolve(); /** - * Resolves this instance to all known socket addresses, asynchronously. + * Resolves this instance to all the candidate socket addresses a connection may be opened to, + * asynchronously. * *

This is called each time the driver opens a new connection to the node. For endpoints backed - * by a plain IP address the returned array contains exactly one element. For endpoints whose - * hostname resolves to multiple IPs (e.g. a DNS round-robin entry) all addresses are returned so + * by a plain IP address the returned array contains exactly one element. For endpoints that know + * of several addresses (e.g. an SNI proxy with multiple A-records) all of them are returned so * that the driver can try each one in sequence and fall back gracefully when individual IPs are * unreachable. * + *

Returning a hostname is fine. Candidates need not be resolved: an {@linkplain + * java.net.InetSocketAddress#isUnresolved() unresolved} {@link java.net.InetSocketAddress} is + * expanded to every address it maps to by the driver, through Netty's configured {@code + * AddressResolverGroup}. That is what {@link + * com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint} does, and it is preferable to + * looking the name up here: it keeps a custom resolver installed via {@code + * NettyOptions#afterBootstrapInitialized(Bootstrap)} in the loop, which a direct {@link + * java.net.InetAddress#getAllByName(String)} call would bypass. + * *

Resolution is asynchronous on purpose: name resolution can block (e.g. {@link * java.net.InetAddress#getAllByName(String)}), and the driver calls this from its admin event * loop, which must never block. Implementations whose resolution may block must run it on diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index 85a2e0488b3..a6ca2da78e6 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -167,14 +167,14 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042. * *

The driver automatically expands any contact point backed by an unresolved hostname to all - * its DNS-mapped IPs at connection time (via {@code EndPoint.resolveAll()}), so passing a single - * hostname is sufficient to try all its IPs on initial connect. This applies equally to hostnames - * provided here programmatically (build an unresolved {@link InetSocketAddress} with {@link - * InetSocketAddress#createUnresolved(String, int)} to opt in) and to hostnames specified in the - * configuration. An already-resolved address passed here (the common case when constructing an - * {@code InetSocketAddress} directly from a hostname, which resolves eagerly) is used as - * provided, with no further expansion. The {@code advanced.resolve-contact-points} option is - * deprecated and has no effect. + * its DNS-mapped IPs at connection time (through Netty's configured resolver, so a custom {@code + * AddressResolverGroup} still applies), so passing a single hostname is sufficient to try all its + * IPs on initial connect. This applies equally to hostnames provided here programmatically (build + * an unresolved {@link InetSocketAddress} with {@link InetSocketAddress#createUnresolved(String, + * int)} to opt in) and to hostnames specified in the configuration. An already-resolved address + * passed here (the common case when constructing an {@code InetSocketAddress} directly from a + * hostname, which resolves eagerly) is used as provided, with no further expansion. The {@code + * advanced.resolve-contact-points} option is deprecated and has no effect. */ @NonNull public SelfT addContactPoints(@NonNull Collection contactPoints) { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index bb58a5f54fe..d2ccae3d428 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -39,6 +39,7 @@ import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.driver.internal.core.metrics.SessionMetricUpdater; @@ -54,11 +55,16 @@ import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; import java.io.IOException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -122,11 +128,19 @@ public class ChannelFactory { */ private static final int RESOLVER_MAX_THREADS = 16; - // Runs potentially-blocking EndPoint name resolution (EndPoint.resolveAll()) off the caller - // thread: connect() is invoked from the admin event loop, which must never block on DNS. Bounded - // at RESOLVER_MAX_THREADS threads with an unbounded queue, so a submission is never rejected and - // never runs inline on the caller -- excess resolutions simply queue until a thread frees up. - // Idle threads are reclaimed after ~60s. + /** + * Handed to {@link EndPoint#resolveAll(java.util.concurrent.Executor)} so implementations whose + * resolution blocks ({@code SniEndPoint}, {@code ClientRoutesEndPoint}, third-party endpoints) + * run it off the caller thread: {@code connect()} is invoked from the admin event loop, which + * must never block. + * + *

{@code DefaultEndPoint} does not use this — hostname expansion goes through Netty's resolver + * instead, see {@link #resolveCandidates}. + * + *

Bounded at {@link #RESOLVER_MAX_THREADS} threads with an unbounded queue, so a submission is + * never rejected and never runs inline on the caller: excess resolutions simply queue until a + * thread frees up. Idle threads are reclaimed after ~60s. + */ private final ExecutorService resolverExecutor; /** either set from the configuration, or null and will be negotiated */ @@ -146,6 +160,11 @@ public ChannelFactory(InternalDriverContext context) { this.logPrefix = context.getSessionName(); this.context = context; + DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); + + // Same setting that governs the Netty I/O, admin and timer threads (see DefaultNettyOptions): + // every thread the driver creates should behave the same way with respect to JVM exit. + boolean daemon = defaultConfig.getBoolean(DefaultDriverOption.NETTY_DAEMON); AtomicInteger resolverThreadCount = new AtomicInteger(); ThreadPoolExecutor resolverThreadPool = new ThreadPoolExecutor( @@ -159,13 +178,12 @@ public ChannelFactory(InternalDriverContext context) { new Thread( runnable, logPrefix + "-connection-resolver-" + resolverThreadCount.incrementAndGet()); - thread.setDaemon(true); + thread.setDaemon(daemon); return thread; }); resolverThreadPool.allowCoreThreadTimeOut(true); this.resolverExecutor = resolverThreadPool; - DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); if (defaultConfig.isDefined(DefaultDriverOption.PROTOCOL_VERSION)) { String versionName = defaultConfig.getString(DefaultDriverOption.PROTOCOL_VERSION); this.protocolVersion = context.getProtocolVersionRegistry().fromName(versionName); @@ -196,8 +214,12 @@ public String getClusterName() { } /** - * Shuts down the internal DNS-resolver executor. Invoked during session shutdown. The executor's - * threads are daemon threads, so a missed call cannot prevent JVM exit. + * Shuts down the internal name-resolver executor. Invoked during session shutdown, from {@code + * DefaultSession}'s close sequence. + * + *

Its threads follow {@code advanced.netty.daemon}, exactly like the Netty I/O and admin + * threads, so with the default (non-daemon) setting this call is what lets the JVM exit — the + * same contract as {@link NettyOptions#onClose()}. */ public void close() { resolverExecutor.shutdownNow(); @@ -265,6 +287,18 @@ private void connect( boolean isNegotiating, CompletableFuture resultFuture) { + // Built once per connect() rather than once per candidate: it is the only handle on the Netty + // AddressResolverGroup (see resolveCandidates()), and it means the user's + // afterBootstrapInitialized() hook runs once per logical connection instead of once per address + // attempt. Each attempt gets its own clone() with its own handler. + Bootstrap baseBootstrap; + try { + baseBootstrap = newBootstrap(); + } catch (Exception e) { + resultFuture.completeExceptionally(e); + return; + } + // Resolution may block (DNS); it is offloaded to resolverExecutor so the calling thread (the // admin event loop, for control-connection reconnects) never waits on it. The continuation // below runs off the admin loop -- tryNextCandidate()/Bootstrap.connect() are safe from any @@ -278,44 +312,171 @@ private void connect( return; } - resolveStage.whenComplete( - (candidates, error) -> { - if (error != null) { - // supplyAsync wraps supplier throwables in a CompletionException; unwrap so callers - // (and the guard test's isSameAs assertion) see the original cause. - Throwable cause = - (error instanceof CompletionException && error.getCause() != null) - ? error.getCause() - : error; - resultFuture.completeExceptionally(cause); - return; - } - if (candidates == null || candidates.length == 0) { - resultFuture.completeExceptionally( - new IllegalArgumentException( - "EndPoint.resolveAll() must return a non-null, non-empty array: " + endPoint)); + resolveStage + .thenCompose( + candidates -> { + if (candidates == null || candidates.length == 0) { + throw new IllegalArgumentException( + "EndPoint.resolveAll() must return a non-null, non-empty array: " + endPoint); + } + return resolveCandidates(baseBootstrap, candidates); + }) + .whenComplete( + (candidates, error) -> { + if (error != null) { + // supplyAsync wraps supplier throwables in a CompletionException; unwrap so callers + // (and the guard test's isSameAs assertion) see the original cause. + Throwable cause = + (error instanceof CompletionException && error.getCause() != null) + ? error.getCause() + : error; + resultFuture.completeExceptionally(cause); + return; + } + tryNextCandidate( + baseBootstrap, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + resultFuture, + candidates, + 0, + new ArrayList<>()); + }); + } + + /** + * Builds the {@link Bootstrap} shared by every connection attempt of a single {@code connect()} + * call, including the user's {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} hook. Per + * attempt, {@link #connectToAddress} takes a {@link Bootstrap#clone()} of it and installs its own + * handler; the copy carries the resolver configuration over. + */ + private Bootstrap newBootstrap() { + NettyOptions nettyOptions = context.getNettyOptions(); + Bootstrap bootstrap = + new Bootstrap() + .group(nettyOptions.ioEventLoopGroup()) + .channel(nettyOptions.channelClass()) + .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()); + nettyOptions.afterBootstrapInitialized(bootstrap); + return bootstrap; + } + + /** + * Turns the raw candidates returned by {@link EndPoint#resolveAll(java.util.concurrent.Executor)} + * into concrete, connectable addresses, expanding any unresolved candidate to all the + * addresses it maps to. + * + *

Expansion goes through the bootstrap's Netty {@link AddressResolverGroup} rather than a + * direct {@code InetAddress.getAllByName()} call, so a custom resolver installed via {@link + * NettyOptions#afterBootstrapInitialized(Bootstrap)} is honoured — that is the resolver an + * unresolved address would have reached had it been handed straight to {@code + * Bootstrap.connect()}, as it was before multi-address support. Candidates the resolver does not + * support (e.g. {@link io.netty.channel.local.LocalAddress}) or that are already resolved are + * passed through untouched; this mirrors {@code Bootstrap#doResolveAndConnect0}. A null group + * means the user called {@link Bootstrap#disableResolver()}, which is likewise respected. + * + *

Note that with Netty's default resolver the lookup blocks the event loop it runs on, + * because {@code DefaultNameResolver} performs {@code InetAddress.getAllByName()} inline. That is + * the pre-existing behaviour of handing an unresolved address to {@code Bootstrap.connect()}, and + * it is an I/O loop, never the admin loop that {@code connect()} is called from. Deployments that + * need non-blocking resolution can now install {@code DnsAddressResolverGroup} and have it take + * effect. + * + *

A candidate that fails to resolve is skipped, so one bad entry does not mask the others; the + * stage only fails if every candidate fails, carrying the first failure as the cause. + */ + private CompletionStage> resolveCandidates( + Bootstrap bootstrap, SocketAddress[] candidates) { + + AddressResolverGroup resolverGroup = bootstrap.config().resolver(); + if (resolverGroup == null) { + // Bootstrap.disableResolver(): the user wants addresses passed through as-is. + return CompletableFuture.completedFuture(Arrays.asList(candidates)); + } + + // The resolver must be obtained for -- and used from -- an event executor whose transport + // matches the channel class, since DnsAddressResolverGroup registers a datagram channel on it. + // An I/O event loop is also what Netty itself uses here: Bootstrap resolves on the connecting + // channel's own event loop. + EventExecutor eventExecutor = context.getNettyOptions().ioEventLoopGroup().next(); + CompletableFuture> result = new CompletableFuture<>(); + eventExecutor.execute( + () -> { + AddressResolver resolver; + try { + resolver = resolverGroup.getResolver(eventExecutor); + } catch (Throwable t) { + result.completeExceptionally(t); return; } - tryNextCandidate( - endPoint, - shardingInfo, - shardId, - options, - nodeMetricUpdater, - currentVersion, - isNegotiating, - resultFuture, - candidates, - 0, - new ArrayList<>()); + expandCandidate(resolver, candidates, 0, new ArrayList<>(), null, result); }); + return result; } /** - * Iterates through the candidate addresses from {@link - * EndPoint#resolveAll(java.util.concurrent.Executor)}. Tries each one in sequence; if an address - * fails for a reason other than protocol-version negotiation exhaustion, the next candidate is - * tried. Only when all candidates are exhausted is the overall {@code resultFuture} failed. + * Resolves {@code candidates[index]}, then recurses on the next one. Runs on the resolver's loop. + */ + private void expandCandidate( + AddressResolver resolver, + SocketAddress[] candidates, + int index, + List resolved, + Throwable firstError, + CompletableFuture> result) { + + if (index == candidates.length) { + if (resolved.isEmpty()) { + result.completeExceptionally( + firstError != null + ? firstError + : new IllegalArgumentException( + "Could not resolve any of " + Arrays.toString(candidates))); + } else { + result.complete(resolved); + } + return; + } + + SocketAddress candidate = candidates[index]; + if (!resolver.isSupported(candidate) || resolver.isResolved(candidate)) { + // Nothing for the resolver to do; same short-circuit as Bootstrap#doResolveAndConnect0. + resolved.add(candidate); + expandCandidate(resolver, candidates, index + 1, resolved, firstError, result); + return; + } + + resolver + .resolveAll(candidate) + .addListener( + (Future> future) -> { + Throwable error = firstError; + if (future.isSuccess()) { + @SuppressWarnings("unchecked") + List addresses = + (List) future.getNow(); + resolved.addAll(addresses); + } else { + LOG.debug( + "[{}] Could not resolve {}, skipping it", logPrefix, candidate, future.cause()); + if (error == null) { + error = future.cause(); + } + } + expandCandidate(resolver, candidates, index + 1, resolved, error, result); + }); + } + + /** + * Iterates through the candidate addresses produced by {@link #resolveCandidates}. Tries each one + * in sequence; if an address fails for a reason other than protocol-version negotiation + * exhaustion, the next candidate is tried. Only when all candidates are exhausted is the overall + * {@code resultFuture} failed. * *

Timeout note: addresses are tried serially, so the worst-case time before failure is * {@code N × connect-timeout} where N is the number of candidates. This is an intentional @@ -328,6 +489,7 @@ private void connect( * only logged at DEBUG). */ private void tryNextCandidate( + Bootstrap baseBootstrap, EndPoint endPoint, NodeShardingInfo shardingInfo, Integer shardId, @@ -336,18 +498,25 @@ private void tryNextCandidate( ProtocolVersion currentVersion, boolean isNegotiating, CompletableFuture resultFuture, - SocketAddress[] candidates, + List candidates, int index, List priorErrors) { - SocketAddress candidate = candidates[index]; + SocketAddress candidate = candidates.get(index); + // Everything downstream of here -- the channel, its pipeline (SSL engine, authenticator) and + // the + // DriverChannel handed to the caller -- sees an endpoint bound to this one address instead of + // the + // multi-address original. See PinnableEndPoint for why that matters. + EndPoint pinnedEndPoint = pin(endPoint, candidate); CompletableFuture perAddressFuture = new CompletableFuture<>(); // Fresh per candidate address: connectToAddress()'s downgrade retries stay on this one // address, so the final UnsupportedProtocolVersionException (if negotiation is what dooms // this candidate) only reports versions actually tried against it, not earlier candidates'. List attemptedVersions = new CopyOnWriteArrayList<>(); connectToAddress( - endPoint, + baseBootstrap, + pinnedEndPoint, shardingInfo, shardId, options, @@ -362,7 +531,7 @@ private void tryNextCandidate( (channel, error) -> { if (error == null) { resultFuture.complete(channel); - } else if (index + 1 < candidates.length) { + } else if (index + 1 < candidates.size()) { LOG.debug( "[{}] Failed to connect to {} ({}), trying next address", logPrefix, @@ -370,6 +539,9 @@ private void tryNextCandidate( error.getMessage()); priorErrors.add(error); tryNextCandidate( + baseBootstrap, + // Deliberately the original, not the pinned copy: the next candidate must be pinned + // from the unpinned endpoint. endPoint, shardingInfo, shardId, @@ -402,6 +574,7 @@ private void tryNextCandidate( * (try the next IP) from a successful protocol handshake. */ private void connectToAddress( + Bootstrap baseBootstrap, EndPoint endPoint, NodeShardingInfo shardingInfo, Integer shardId, @@ -413,19 +586,15 @@ private void connectToAddress( CompletableFuture perAddressFuture, SocketAddress resolvedAddress) { - NettyOptions nettyOptions = context.getNettyOptions(); - + // clone() so each attempt gets its own handler while sharing the group, options and resolver + // configuration (including anything afterBootstrapInitialized() set). Bootstrap bootstrap = - new Bootstrap() - .group(nettyOptions.ioEventLoopGroup()) - .channel(nettyOptions.channelClass()) - .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()) + baseBootstrap + .clone() .handler( initializer( endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); - nettyOptions.afterBootstrapInitialized(bootstrap); - ChannelFuture connectFuture; if (shardId == null || shardingInfo == null) { if (shardId != null) { @@ -496,6 +665,7 @@ private void connectToAddress( downgraded.get()); // Stay on the same address for protocol-version downgrade retries. connectToAddress( + baseBootstrap, endPoint, shardingInfo, shardId, @@ -520,6 +690,20 @@ private void connectToAddress( }); } + /** + * Binds {@code endPoint} to the address a connection is being opened to, when the implementation + * supports it. + * + *

Third-party {@link EndPoint}s that do not implement {@link PinnableEndPoint} are returned + * unchanged, so they keep behaving exactly as they did before multi-address support: the channel + * carries the endpoint it was given. + */ + private static EndPoint pin(EndPoint endPoint, SocketAddress resolvedAddress) { + return endPoint instanceof PinnableEndPoint + ? ((PinnableEndPoint) endPoint).pinTo(resolvedAddress) + : endPoint; + } + @VisibleForTesting ChannelInitializer initializer( EndPoint endPoint, diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java index 62976f08cb3..e6bad7bf8a3 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -31,11 +31,20 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.Executor; -public class ClientRoutesEndPoint implements EndPoint { +public class ClientRoutesEndPoint implements PinnableEndPoint { private final UUID hostId; private final ClientRoutesTopologyMonitor topologyMonitor; private final String metricPrefix; @NonNull private final EndPoint fallbackEndPoint; + /** Kept only so that {@link #pinTo(SocketAddress)} can rebuild an identical copy. */ + @Nullable private final InetAddress broadcastInetAddress; + + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}, + * which key off the host id alone. + */ + @Nullable private final SocketAddress pinnedAddress; /** * @param topologyMonitor the topology monitor used to resolve the endpoint address on demand. @@ -52,12 +61,23 @@ public ClientRoutesEndPoint( @NonNull UUID hostId, @Nullable InetAddress broadcastInetAddress, @NonNull EndPoint fallbackEndPoint) { + this(topologyMonitor, hostId, broadcastInetAddress, fallbackEndPoint, null); + } + + private ClientRoutesEndPoint( + @NonNull ClientRoutesTopologyMonitor topologyMonitor, + @NonNull UUID hostId, + @Nullable InetAddress broadcastInetAddress, + @NonNull EndPoint fallbackEndPoint, + @Nullable SocketAddress pinnedAddress) { this.topologyMonitor = Objects.requireNonNull(topologyMonitor, "Topology monitor cannot be null"); this.hostId = Objects.requireNonNull(hostId, "HOST uuid cannot be null"); this.fallbackEndPoint = Objects.requireNonNull(fallbackEndPoint, "Fallback endpoint cannot be null"); this.metricPrefix = buildMetricPrefix(broadcastInetAddress, hostId); + this.broadcastInetAddress = broadcastInetAddress; + this.pinnedAddress = pinnedAddress; } @NonNull @@ -68,6 +88,9 @@ public UUID getHostId() { @NonNull @Override public SocketAddress resolve() { + if (pinnedAddress != null) { + return pinnedAddress; + } try { InetSocketAddress address = topologyMonitor.resolve(hostId); if (address != null) { @@ -80,18 +103,55 @@ public SocketAddress resolve() { } /** - * Returns all socket addresses for this endpoint. + * Returns the candidate addresses for this endpoint. * - *

Delegates to {@link #resolve()} to obtain the single address provided by the topology - * monitor (or the fallback endpoint), then returns it as a one-element array in an already - * completed stage. The topology monitor resolves each node to exactly one address by design (via - * an in-memory per-host-id lookup), so multi-address expansion is not applicable here and the - * {@code executor} is not used. + *

The topology monitor resolves each node to exactly one address by design (a per-host-id + * lookup over {@code system.client_routes}), so this never expands to several candidates. It is + * still asynchronous, because the lookup is not purely in-memory: {@link + * ClientRoutesTopologyMonitor#resolve} can reach {@link + * ClientRoutesTopologyMonitor#resolveAddress} → {@link InetAddress#getByName}, which blocks. It + * therefore runs on {@code executor} rather than on the caller (the admin event loop, for + * control-connection reconnects). + * + *

When the topology monitor has no route for this host id — i.e. the node is not reached + * through a cloud private endpoint — this delegates to {@code + * fallbackEndPoint.resolveAll(executor)} so the fallback keeps whatever resolution semantics it + * defines, rather than being flattened to its single {@link EndPoint#resolve()} address. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} the pinned address is returned directly, with + * no lookup and no executor hop. */ @NonNull @Override public CompletionStage resolveAll(@NonNull Executor executor) { - return CompletableFuture.completedFuture(new SocketAddress[] {resolve()}); + if (pinnedAddress != null) { + return CompletableFuture.completedFuture(new SocketAddress[] {pinnedAddress}); + } + return CompletableFuture.>supplyAsync( + () -> { + InetSocketAddress address; + try { + address = topologyMonitor.resolve(hostId); + } catch (IOException e) { + throw new UncheckedIOException("DNS resolution failed for host_id=" + hostId, e); + } + return address != null + ? CompletableFuture.completedFuture(new SocketAddress[] {address}) + : fallbackEndPoint.resolveAll(executor); + }, + executor) + .thenCompose(stage -> stage); + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + if (resolvedAddress.equals(this.pinnedAddress)) { + return this; + } + return new ClientRoutesEndPoint( + topologyMonitor, hostId, broadcastInetAddress, fallbackEndPoint, resolvedAddress); } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index a7bec899ccd..6503d81b2cf 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -19,76 +19,77 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.io.Serializable; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.net.UnknownHostException; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.Executor; -public class DefaultEndPoint implements EndPoint, Serializable { +public class DefaultEndPoint implements PinnableEndPoint, Serializable { private static final long serialVersionUID = 1; private final InetSocketAddress address; private final String metricPrefix; + /** + * The address this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals}, {@link #hashCode} and + * {@link #asMetricPrefix()}: a pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + public DefaultEndPoint(InetSocketAddress address) { + this(address, null); + } + + private DefaultEndPoint(InetSocketAddress address, @Nullable InetSocketAddress pinnedAddress) { this.address = Objects.requireNonNull(address, "address can't be null"); this.metricPrefix = buildMetricPrefix(address); + this.pinnedAddress = pinnedAddress; } @NonNull @Override public InetSocketAddress resolve() { - return address; + return pinnedAddress != null ? pinnedAddress : address; } /** - * Returns all socket addresses for this endpoint. - * - *

If the stored address is already resolved, the returned stage completes immediately (on the - * calling thread) with a single-element array; no executor hop occurs. + * Returns the candidate addresses for this endpoint. * - *

If the stored address is unresolved (i.e. the driver was configured with {@code - * RESOLVE_CONTACT_POINTS=false} and the hostname has not been looked up yet), the blocking {@link - * InetAddress#getAllByName(String)} lookup is run on {@code executor} to expand the hostname to - * every IP it resolves to. Each resolved IP is returned as an {@link InetSocketAddress} with the - * same port as the original. + *

The returned stage always completes immediately, on the calling thread, with a single + * element, and never uses {@code executor}: this implementation performs no name resolution of + * its own. * - *

If DNS resolution fails, falls back to a single-element array containing {@link #resolve()}. - * - *

Note on resolver: DNS lookup is performed via {@link - * InetAddress#getAllByName(String)}, bypassing any custom Netty {@code AddressResolverGroup} - * configured on the driver. This is consistent with how {@link SniEndPoint} performs DNS - * resolution elsewhere in the driver. Users who rely on a custom Netty resolver should supply - * pre-resolved {@link java.net.InetSocketAddress} instances instead of hostnames. + *

If the stored address is a hostname (i.e. {@linkplain InetSocketAddress#isUnresolved() + * unresolved} — contact points are always kept unresolved, see {@link + * com.datastax.oss.driver.api.core.session.SessionBuilder#addContactPoint}), the unresolved + * address is returned as-is, and {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands it to every IP it maps to + * through Netty's configured {@code AddressResolverGroup}. Resolving there rather than here is + * deliberate: it keeps any custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized} in the + * loop, which a direct {@code InetAddress.getAllByName()} call from here would bypass. */ @NonNull @Override public CompletionStage resolveAll(@NonNull Executor executor) { - if (!address.isUnresolved()) { - return CompletableFuture.completedFuture(new SocketAddress[] {address}); + return CompletableFuture.completedFuture(new SocketAddress[] {resolve()}); + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress can't be null"); + if (!(resolvedAddress instanceof InetSocketAddress) + || resolvedAddress.equals(this.pinnedAddress)) { + return this; } - return CompletableFuture.supplyAsync( - () -> { - try { - InetAddress[] all = InetAddress.getAllByName(address.getHostString()); - SocketAddress[] result = new SocketAddress[all.length]; - for (int i = 0; i < all.length; i++) { - result[i] = new InetSocketAddress(all[i], address.getPort()); - } - return result; - } catch (UnknownHostException e) { - // Fallback: return the single unresolved address; the connect attempt will fail with a - // descriptive error rather than silently returning an empty array. - return new SocketAddress[] {address}; - } - }, - executor); + return new DefaultEndPoint(address, (InetSocketAddress) resolvedAddress); } @Override @@ -118,7 +119,9 @@ public int hashCode() { @Override public String toString() { - return address.toString(); + // Show both when pinned: the original identifies the node, the pinned address tells you which + // IP a connection actually landed on -- which is the useful bit in connection-level logs. + return pinnedAddress == null ? address.toString() : address + "(" + pinnedAddress + ")"; } @NonNull diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java new file mode 100644 index 00000000000..cfb34e1307f --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.SocketAddress; + +/** + * An {@link EndPoint} that can produce a copy of itself bound ("pinned") to one specific address. + * + *

An endpoint whose hostname maps to several IPs describes a set of candidate addresses, + * but a channel is always connected to exactly one of them. {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} pins the endpoint to the address it + * actually used, and hands the pinned copy to the channel. That matters for two reasons: + * + *

    + *
  • Node identity. Once the driver has learnt, over a given connection, that {@code + * host_id} X answers at a given IP, that node must keep reconnecting to that IP. If + * the node kept the multi-address endpoint, a later reconnect could land on a different node + * while still being treated as X (see {@code DefaultTopologyMonitor#buildNodeEndPoint} and + * {@code ControlConnection}, which skip identity re-resolution for nodes that already have a + * host id). + *
  • No re-resolution on the channel path. Components handed the channel's endpoint call + * {@link EndPoint#resolve()} — SSL engine creation, GSSAPI service-name lookup, {@code + * DefaultTopologyMonitor#savePort}. On a pinned endpoint that is a field read, so it neither + * blocks on DNS (SSL setup runs on a Netty event loop) nor risks picking a different address + * than the one the channel is connected to. + *
+ * + *

This is an internal extension point: {@code ChannelFactory} pins endpoints that implement it + * and leaves any other implementation untouched, so third-party {@link EndPoint}s keep working + * exactly as before. + * + *

Implementations must keep {@link Object#equals}, {@link Object#hashCode} and {@link + * EndPoint#asMetricPrefix()} identical to the unpinned original — a pinned copy denotes the same + * node, and metric names must not change depending on which IP a connection happened to land on. + * Equality must stay symmetric: {@code original.equals(pinned)} and {@code pinned.equals(original)} + * must agree, since endpoints are used as set and map keys. + */ +public interface PinnableEndPoint extends EndPoint { + + /** + * Returns a copy of this endpoint that resolves to exactly {@code resolvedAddress}. + * + *

Implementations may return {@code this} when pinning does not apply (for example when the + * address is not of a type they can hold on to). + * + * @param resolvedAddress the address a connection was successfully established to; must not be + * null and must already be resolved. + */ + @NonNull + EndPoint pinTo(@NonNull SocketAddress resolvedAddress); +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index a141ef1576b..f4cb0eeeba1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -20,6 +20,7 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.shaded.guava.common.primitives.UnsignedBytes; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -32,7 +33,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; -public class SniEndPoint implements EndPoint { +public class SniEndPoint implements PinnableEndPoint { // Rotates the single address returned by resolve() (still used for SSL engine setup). private static final AtomicInteger OFFSET = new AtomicInteger(); // Rotates the starting candidate of resolveAll(). Kept separate from OFFSET so that SSL engine @@ -45,6 +46,13 @@ public class SniEndPoint implements EndPoint { private final InetSocketAddress proxyAddress; private final String serverName; + /** + * The proxy IP this endpoint has been {@linkplain #pinTo(SocketAddress) pinned} to, or {@code + * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}: a + * pinned copy denotes the same node as the original. + */ + @Nullable private final InetSocketAddress pinnedAddress; + /** * @param proxyAddress the address of the proxy. If it is {@linkplain * InetSocketAddress#isUnresolved() unresolved}, each call to {@link #resolve()} will @@ -54,17 +62,37 @@ public class SniEndPoint implements EndPoint { * representation of the host id. */ public SniEndPoint(InetSocketAddress proxyAddress, String serverName) { + this(proxyAddress, serverName, null); + } + + private SniEndPoint( + InetSocketAddress proxyAddress, + String serverName, + @Nullable InetSocketAddress pinnedAddress) { this.proxyAddress = Objects.requireNonNull(proxyAddress, "SNI address cannot be null"); this.serverName = Objects.requireNonNull(serverName, "SNI Server name cannot be null"); + this.pinnedAddress = pinnedAddress; } public String getServerName() { return serverName; } + /** + * Resolves this endpoint to a single proxy address. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} this is a field read. That matters because + * {@link com.datastax.oss.driver.internal.core.ssl.SniSslEngineFactory#newSslEngine} calls this + * from Netty's channel initializer, i.e. on an I/O event loop: on an unpinned endpoint it would + * perform a blocking {@link InetAddress#getAllByName(String)} lookup there, and could pick a + * different proxy IP than the one the channel is actually connected to. + */ @NonNull @Override public InetSocketAddress resolve() { + if (pinnedAddress != null) { + return pinnedAddress; + } try { InetAddress[] aRecords = InetAddress.getAllByName(proxyAddress.getHostName()); if (aRecords.length == 0) { @@ -103,10 +131,16 @@ public InetSocketAddress resolve() { * loop) thread; the returned stage completes with the rotated candidate array, or completes * exceptionally with an {@link IllegalArgumentException} if the proxy hostname cannot be * resolved. + * + *

Once {@linkplain #pinTo(SocketAddress) pinned} this returns the pinned address only, without + * any lookup or rotation: the endpoint then denotes one specific proxy IP. */ @NonNull @Override public CompletionStage resolveAll(@NonNull Executor executor) { + if (pinnedAddress != null) { + return CompletableFuture.completedFuture(new SocketAddress[] {pinnedAddress}); + } return CompletableFuture.supplyAsync( () -> { try { @@ -137,6 +171,17 @@ public CompletionStage resolveAll(@NonNull Executor executor) { executor); } + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); + if (!(resolvedAddress instanceof InetSocketAddress) + || resolvedAddress.equals(this.pinnedAddress)) { + return this; + } + return new SniEndPoint(proxyAddress, serverName, (InetSocketAddress) resolvedAddress); + } + @Override public boolean equals(Object other) { if (other == this) { @@ -156,10 +201,13 @@ public int hashCode() { @Override public String toString() { - // Note that this uses the original proxy address, so if there are multiple A-records it won't - // show which one was selected. If that turns out to be a problem for debugging, we might need - // to store the result of resolve() in Connection and log that instead of the endpoint. - return proxyAddress.toString() + ":" + serverName; + // An unpinned endpoint prints the original proxy address, so with multiple A-records it does + // not + // say which one a given connection selected. A pinned copy does -- and that is what channels + // carry, so connection-level logs identify the actual proxy IP. + return pinnedAddress == null + ? proxyAddress + ":" + serverName + : proxyAddress + "(" + pinnedAddress + "):" + serverName; } @NonNull diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 2ce07eb48c3..e94e9398e61 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1227,11 +1227,16 @@ datastax-java-driver { # DEPRECATED: this option no longer has any effect and will be removed in a future release. # # Contact points are now always kept as unresolved hostnames and expanded to all of their - # DNS-mapped IPs lazily at connection time (see EndPoint.resolveAll()). This means the driver - # tries every IP a hostname resolves to, and re-resolves the hostname on each new connection so - # DNS changes are picked up automatically. Previously this option selected between resolving a - # contact-point hostname once (true) and re-resolving it on every connection (false); that - # distinction no longer applies. + # DNS-mapped IPs lazily at connection time. This means the driver tries every IP a hostname + # resolves to, and re-resolves the hostname on each new connection so DNS changes are picked up + # automatically. Previously this option selected between resolving a contact-point hostname once + # (true) and re-resolving it on every connection (false); that distinction no longer applies. + # + # The lookup goes through Netty's configured AddressResolverGroup -- the same resolver an + # unresolved address would have reached had it been passed straight to Bootstrap.connect() -- so a + # custom resolver installed via NettyOptions.afterBootstrapInitialized() still applies. With + # Netty's default (JDK) resolver the lookup blocks the I/O event loop it runs on; install + # DnsAddressResolverGroup if you need it to be non-blocking. # # This option only ever applied to the contact points specified in the configuration -- never to # programmatic contact points passed to SessionBuilder.addContactPoints, nor to dynamically @@ -2335,8 +2340,8 @@ datastax-java-driver { # plan, after the live nodes reported by the load balancing policy. # # This is also the driver's DNS re-resolution path. Contact points are kept as unresolved - # hostnames and expanded to their current DNS IPs at connection time (see - # EndPoint.resolveAll()). Metadata nodes, in contrast, store an already-resolved endpoint that + # hostnames and expanded to their current DNS IPs at connection time, through Netty's + # configured resolver. Metadata nodes, in contrast, store an already-resolved endpoint that # is never re-resolved, so once DNS records change they would otherwise become stale. Keeping # this enabled lets control-connection reconnects re-resolve the original hostnames and pick up # the new IPs once the live-node plan is exhausted. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java new file mode 100644 index 00000000000..be85a9b0a68 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CopyOnWriteArrayList; +import org.junit.Test; + +/** + * Verifies that {@link ChannelFactory} expands unresolved candidate addresses through Netty's + * configured {@link AddressResolverGroup}, rather than doing its own JVM DNS lookup. + * + *

This is what keeps a custom resolver installed via {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)} + * effective: before multi-address support, an unresolved address was handed straight to {@code + * Bootstrap.connect()} and Netty's resolver expanded it, so resolving anywhere else would silently + * bypass the user's configuration. + */ +public class ChannelFactoryNettyResolverTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryNettyResolverTest.class.getSimpleName() + "-unreachable"); + + /** The hostname the endpoint reports, and that only the custom resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + + @Test + public void should_expand_unresolved_address_through_the_custom_netty_resolver() { + // Given – a resolver that maps the hostname to an unreachable address followed by the running + // local server, mimicking a DNS round-robin entry whose first record is dead. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestResolverGroup resolverGroup = + new TestResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint itself performs no resolution at all; it just yields the hostname. + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + // The handshake only happens once we fall back to the reachable second address. + completeSimpleChannelInit(); + + // Then – the custom resolver was consulted for the hostname, and *all* the addresses it + // returned + // were tried, so the connection survived the dead first record. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the custom Netty resolver must be the one expanding the hostname") + .containsExactly(HOSTNAME); + } + + @Test + public void should_fail_when_the_custom_resolver_cannot_resolve_the_only_candidate() { + // Given – a resolver that fails every lookup. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestResolverGroup resolverGroup = new TestResolverGroup(null); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then – no candidate survived resolution, so the connect fails with the resolver's own cause + // rather than, say, an empty-candidate-list error. + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).hasMessageContaining("mock resolver failure")); + } + + @Test + public void should_not_resolve_at_all_when_the_user_disabled_the_resolver() { + // Given – Bootstrap.disableResolver() means config().resolver() is null. ChannelFactory must + // treat that as "pass the candidates through" instead of dereferencing the missing group. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestResolverGroup resolverGroup = new TestResolverGroup(Collections.singletonList(UNREACHABLE)); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(resolverGroup).disableResolver(); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint yields an already-usable address. + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – connection succeeds and the resolver was never even instantiated, let alone consulted. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.resolverRequested).isFalse(); + assertThat(resolverGroup.queried).isEmpty(); + } + + @Test + public void should_pass_already_resolved_candidates_through_untouched() { + // Given – an endpoint that returns addresses the resolver reports as already resolved. Mirrors + // Bootstrap#doResolveAndConnect0, which skips resolution in that case. Metadata nodes are in + // this + // situation: their endpoints hold resolved addresses from the peers rows. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestResolverGroup resolverGroup = new TestResolverGroup(Collections.singletonList(UNREACHABLE)); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – had the resolver been consulted it would have redirected us to UNREACHABLE and the + // connection would have failed. + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried).isEmpty(); + } + + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ + private void installResolver(AddressResolverGroup group) { + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(group); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + } + + /** + * A stand-in for a user-supplied {@code AddressResolverGroup} (e.g. Netty's {@code + * DnsAddressResolverGroup}). Records what it was asked to resolve, and answers with a fixed list + * of addresses so the test can assert that every one of them is tried. + * + *

Implements {@link AddressResolver} directly rather than extending {@code + * AbstractAddressResolver} so it can hand back {@link LocalAddress}es — the unit tests connect + * over Netty's local transport, which is not reachable through an {@link InetSocketAddress}. + */ + private static class TestResolverGroup extends AddressResolverGroup { + + final List queried = new CopyOnWriteArrayList<>(); + volatile boolean resolverRequested; + + /** The addresses to answer with, or {@code null} to fail every lookup. */ + private final List answer; + + TestResolverGroup(List answer) { + this.answer = answer; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + resolverRequested = true; + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + return true; + } + + @Override + public boolean isResolved(SocketAddress address) { + // Only hostnames need resolving; anything else (including the local-transport addresses + // we + // hand back) is already usable. + return !(address instanceof InetSocketAddress) + || !((InetSocketAddress) address).isUnresolved(); + } + + @Override + public Future resolve(SocketAddress address) { + return resolve(address, executor.newPromise()); + } + + @Override + public Future resolve( + SocketAddress address, Promise promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer.get(0)); + } + + @Override + public Future> resolveAll(SocketAddress address) { + return resolveAll(address, executor.newPromise()); + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer); + } + + @Override + public void close() { + // nothing to do + } + }; + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java new file mode 100644 index 00000000000..5f11fbc1594 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.channel.local.LocalAddress; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import org.junit.Test; + +/** + * Verifies that a successfully connected {@link DriverChannel} carries an endpoint bound to the + * address the connection actually used, not the multi-address original. + * + *

Without this, a hostname shared by several nodes would let a later reconnect land on a + * different node while still being treated as the original {@code host_id}: {@code + * DefaultTopologyMonitor#buildNodeEndPoint} stores the channel's endpoint for the control node, and + * {@code ControlConnection} skips identity re-resolution for nodes that already have a host id. See + * {@link PinnableEndPoint}. + */ +public class ChannelFactoryPinnedEndPointTest extends ChannelFactoryTestBase { + + // A local address that no server is bound to: connecting to it fails immediately. + private static final SocketAddress UNREACHABLE = + new LocalAddress(ChannelFactoryPinnedEndPointTest.class.getSimpleName() + "-unreachable"); + + @Test + public void should_pin_channel_endpoint_to_the_address_that_connected() { + // Given – two candidates; only the second one has a server listening. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + TestPinnableEndPoint endPoint = new TestPinnableEndPoint(UNREACHABLE, reachable); + + // When + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + // The handshake only happens once we fall back to the reachable second candidate. + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture) + .isSuccess( + channel -> { + EndPoint channelEndPoint = channel.getEndPoint(); + // The channel resolves to the address it is actually connected to... + assertThat(channelEndPoint.resolve()).isEqualTo(reachable); + assertThat(channelEndPoint.resolveAll(Runnable::run).toCompletableFuture().join()) + .containsExactly(reachable); + // ...while still denoting the same node, so node lookups and metric names are stable. + assertThat(channelEndPoint).isEqualTo(endPoint); + assertThat(channelEndPoint.asMetricPrefix()).isEqualTo(endPoint.asMetricPrefix()); + }); + } + + @Test + public void should_leave_non_pinnable_endpoints_untouched() { + // A third-party EndPoint that does not implement PinnableEndPoint must reach the channel + // exactly + // as it was given, so existing implementations keep working. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess(channel -> assertThat(channel.getEndPoint()).isSameAs(SERVER_ADDRESS)); + } + + /** + * A {@link PinnableEndPoint} over Netty's local transport, so the test can exercise pinning + * against the real embedded server. Identity is the candidate list, so a pinned copy stays equal + * to the original — the contract {@link PinnableEndPoint} requires. + */ + private static class TestPinnableEndPoint implements PinnableEndPoint { + + private final List candidates; + private final SocketAddress pinnedAddress; + + TestPinnableEndPoint(SocketAddress... candidates) { + this(Arrays.asList(candidates), null); + } + + private TestPinnableEndPoint(List candidates, SocketAddress pinnedAddress) { + this.candidates = candidates; + this.pinnedAddress = pinnedAddress; + } + + @NonNull + @Override + public SocketAddress resolve() { + return pinnedAddress != null ? pinnedAddress : candidates.get(0); + } + + @NonNull + @Override + public CompletionStage resolveAll(@NonNull Executor executor) { + return CompletableFuture.completedFuture( + pinnedAddress != null + ? new SocketAddress[] {pinnedAddress} + : candidates.toArray(new SocketAddress[0])); + } + + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + return new TestPinnableEndPoint(candidates, resolvedAddress); + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + + @Override + public boolean equals(Object other) { + return (other instanceof TestPinnableEndPoint) + && candidates.equals(((TestPinnableEndPoint) other).candidates); + } + + @Override + public int hashCode() { + return Objects.hash(candidates); + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java index 2780d5bdec9..91da31f0606 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java @@ -127,6 +127,9 @@ public void setup() throws InterruptedException { .thenReturn(Duration.ofSeconds(30)); when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_CONNECT_TIMEOUT)) .thenReturn(Duration.ofSeconds(5)); + // The factory's name-resolver threads follow this setting; daemon here so a test that does not + // close the factory cannot hold the surefire JVM open. + when(defaultProfile.getBoolean(DefaultDriverOption.NETTY_DAEMON)).thenReturn(true); when(context.getProtocolVersionRegistry()).thenReturn(protocolVersionRegistry); when(context.getNettyOptions()).thenReturn(nettyOptions); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java index f31dd2861ed..479730606f3 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java @@ -19,14 +19,23 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.metadata.EndPoint; import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.net.UnknownHostException; +import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -96,6 +105,96 @@ public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownH assertThat(ep.resolve()).isEqualTo(addr2); } + // ---- resolveAll() ------------------------------------------------------- + + @Test + public void should_resolve_all_on_the_supplied_executor() throws UnknownHostException { + // topologyMonitor.resolve() can reach InetAddress.getByName(), which blocks, so it must not run + // on the caller -- ChannelFactory calls this from the admin event loop. + UUID hostId = UUID.randomUUID(); + InetSocketAddress expected = new InetSocketAddress("127.0.0.1", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(expected); + + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + + List resolutionThreads = new CopyOnWriteArrayList<>(); + ExecutorService executor = + Executors.newSingleThreadExecutor(r -> new Thread(r, "resolver-under-test")); + try { + when(topologyMonitor.resolve(hostId)) + .thenAnswer( + invocation -> { + resolutionThreads.add(Thread.currentThread().getName()); + return expected; + }); + + assertThat(ep.resolveAll(executor).toCompletableFuture().join()).containsExactly(expected); + } finally { + executor.shutdownNow(); + } + + assertThat(resolutionThreads).containsExactly("resolver-under-test"); + } + + @Test + public void should_delegate_resolve_all_to_fallback_endpoint_when_there_is_no_route() + throws UnknownHostException { + // No route for this host id: the fallback must be asked through its own resolveAll(), so it + // keeps + // whatever resolution semantics it defines, rather than being flattened to resolve(). + UUID hostId = UUID.randomUUID(); + InetSocketAddress fallback1 = new InetSocketAddress("10.0.0.1", 9042); + InetSocketAddress fallback2 = new InetSocketAddress("10.0.0.2", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(null); + when(fallbackEndPoint.resolveAll(any())) + .thenReturn(CompletableFuture.completedFuture(new SocketAddress[] {fallback1, fallback2})); + + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + + assertThat(ep.resolveAll(Runnable::run).toCompletableFuture().join()) + .containsExactly(fallback1, fallback2); + verify(fallbackEndPoint).resolveAll(any()); + verify(fallbackEndPoint, never()).resolve(); + } + + @Test + public void should_fail_resolve_all_stage_when_topology_monitor_throws() + throws UnknownHostException { + UUID hostId = UUID.randomUUID(); + when(topologyMonitor.resolve(hostId)).thenThrow(new UnknownHostException("no-such-host")); + + ClientRoutesEndPoint ep = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + + assertThatThrownBy(() -> ep.resolveAll(Runnable::run).toCompletableFuture().join()) + .hasCauseInstanceOf(UncheckedIOException.class); + } + + // ---- pinTo() ------------------------------------------------------------ + + @Test + public void pin_to_should_stop_consulting_the_topology_monitor() throws UnknownHostException { + UUID hostId = UUID.randomUUID(); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + assertThat(pinned.resolveAll(Runnable::run).toCompletableFuture().join()) + .containsExactly(pinnedTo); + // No lookup at all -- that is the point: DefaultTopologyMonitor#savePort and the SSL factories + // read the channel's endpoint, and must not trigger a blocking re-resolution there. + verify(topologyMonitor, never()).resolve(hostId); + // Identity is keyed off the host id, so the pinned copy is still the same node. + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + } + // ---- equals / hashCode -------------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index 404de8f7f4c..6e04038725f 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -20,12 +20,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.net.InetAddress; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import io.netty.channel.local.LocalAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.net.UnknownHostException; -import java.util.HashSet; -import java.util.Set; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; public class DefaultEndPointTest { @@ -73,42 +73,108 @@ public void resolve_all_returns_single_element_for_already_resolved_address() { } @Test - public void resolve_all_expands_unresolved_hostname_to_all_dns_ips() throws UnknownHostException { - // localhost reliably resolves to at least 127.0.0.1 (and possibly ::1). + public void resolve_all_passes_unresolved_hostname_through_without_looking_it_up() { + // This endpoint does NOT expand hostnames itself. It hands the unresolved address to + // ChannelFactory, which expands it through Netty's AddressResolverGroup so that a custom + // resolver installed via NettyOptions#afterBootstrapInitialized still applies -- a direct + // InetAddress.getAllByName() call here would bypass it. "localhost" would resolve fine, so the + // assertion below is only meaningful because we check the address comes back *unresolved*. DefaultEndPoint endPoint = new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - // The complete DNS result set must be expanded -- one resolved InetSocketAddress per record -- - // so a connection attempt can fall back across every IP, not just the first. Compare against an - // independent getAllByName() lookup so the test fails if production returned only a subset. - InetAddress[] expectedIps = InetAddress.getAllByName("localhost"); - Set expected = new HashSet<>(); - for (InetAddress ip : expectedIps) { - expected.add(new InetSocketAddress(ip, 9042)); - } - Set actual = new HashSet<>(); - for (SocketAddress addr : all) { - InetSocketAddress inet = (InetSocketAddress) addr; - assertThat(inet.isUnresolved()).isFalse(); - assertThat(inet.getPort()).isEqualTo(9042); - actual.add(inet); - } - assertThat(all).hasSize(expectedIps.length); - assertThat(actual).isEqualTo(expected); + assertThat(all).hasSize(1); + InetSocketAddress only = (InetSocketAddress) all[0]; + assertThat(only.isUnresolved()).isTrue(); + assertThat(only.getHostString()).isEqualTo("localhost"); + assertThat(only.getPort()).isEqualTo(9042); } @Test - public void resolve_all_falls_back_to_single_element_when_hostname_is_unresolvable() { - // Unresolvable hostname: resolveAll() must not throw; it returns the unresolved address. + public void resolve_all_does_not_throw_for_unresolvable_hostname() { + // No lookup happens, so an unresolvable name is not an error at this level: the connect attempt + // fails later with a descriptive error instead. DefaultEndPoint endPoint = new DefaultEndPoint( InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042)); + SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + assertThat(all).hasSize(1); - // The fallback address is the original unresolved one. assertThat(((InetSocketAddress) all[0]).getHostString()) .isEqualTo("this-host-does-not-exist.invalid"); } + + @Test + public void resolve_all_never_uses_the_executor() { + // Resolution is a field read, so the stage must already be complete on return: ChannelFactory + // calls this from the admin event loop and an unnecessary hop would just add latency. + DefaultEndPoint endPoint = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + + AtomicInteger executorCalls = new AtomicInteger(); + CompletionStage stage = + endPoint.resolveAll( + command -> { + executorCalls.incrementAndGet(); + command.run(); + }); + + assertThat(stage.toCompletableFuture()).isCompleted(); + assertThat(executorCalls).hasValue(0); + } + + @Test + public void pin_to_should_override_resolution_but_preserve_identity() { + InetSocketAddress hostname = InetSocketAddress.createUnresolved("test.com", 9042); + DefaultEndPoint original = new DefaultEndPoint(hostname); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + // Resolution now yields exactly the pinned address... + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + assertThat(pinned.resolveAll(Runnable::run).toCompletableFuture().join()) + .containsExactly(pinnedTo); + // ...but the copy still denotes the same node, and metric names must not change depending on + // which IP a connection happened to land on. + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned).isEqualTo(original); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + // Equality has to hold in both directions: endpoints are used as set and map keys. + assertThat(original).isEqualTo(pinned); + // The original is untouched. + assertThat(original.resolve()).isEqualTo(hostname); + } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + DefaultEndPoint original = + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.com", 9042)); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((DefaultEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_reject_null_address() { + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThatThrownBy(() -> endPoint.pinTo(null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("resolvedAddress can't be null"); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // ChannelFactory pins whatever address it connected to; a non-Inet one (e.g. the local + // transport + // used by unit tests) cannot be held in an InetSocketAddress field, so pinning is skipped + // rather + // than failing the connection. + DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java index 99672006747..69109687d93 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -130,4 +130,30 @@ public void resolve_all_rotation_is_not_disturbed_by_interleaved_resolve() { .isTrue(); } } + + @Test + public void pin_to_should_make_resolve_a_field_read_and_preserve_identity() { + // This is what stops SniSslEngineFactory#newSslEngine -- which runs inside Netty's channel + // initializer, i.e. on an I/O event loop -- from doing a blocking DNS lookup, and guarantees it + // sees the very proxy IP the channel is connected to rather than another A-record. + SniEndPoint original = + new SniEndPoint( + new InetSocketAddress("this-host-does-not-exist.invalid", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + // An unresolvable hostname proves no lookup is attempted: the unpinned endpoint throws. + assertThat(pinned.resolve()).isEqualTo(pinnedTo); + assertThat(pinned.resolveAll(Runnable::run).toCompletableFuture().join()) + .containsExactly(pinnedTo); + assertThatThrownBy(original::resolve).isInstanceOf(IllegalArgumentException.class); + + // The pinned copy still denotes the same node. + assertThat(pinned.getServerName()).isEqualTo(original.getServerName()); + assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned).isEqualTo(original); + assertThat(original).isEqualTo(pinned); + assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); + } } diff --git a/manual/core/address_resolution/README.md b/manual/core/address_resolution/README.md index ae44feea3ea..543802e8b15 100644 --- a/manual/core/address_resolution/README.md +++ b/manual/core/address_resolution/README.md @@ -185,8 +185,9 @@ datastax-java-driver { DNS is resolved at connection time (not at route discovery time). The driver delegates to `InetAddress.getByName()`, which is a blocking call that uses the JVM's built-in DNS cache -(30 s default TTL in the JDK). Because this runs on Netty I/O threads, slow or unresponsive -DNS can block connection establishment and impact driver throughput. To mitigate this, configure +(30 s default TTL in the JDK). It runs on the driver's dedicated name-resolver executor rather than +on a Netty event loop, so a slow or unresponsive DNS server delays the connection attempt itself but +does not stall the event loops. Connection establishment is still delayed, so it is worth configuring the JVM DNS cache TTL via the `networkaddress.cache.ttl` security property (e.g. in `$JAVA_HOME/conf/security/java.security` or programmatically with `java.security.Security.setProperty("networkaddress.cache.ttl", "60")`). diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index f1dfdceb82e..694a11273bf 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -24,10 +24,17 @@ under the License. #### Contact points are expanded to all their DNS addresses at connection time Contact points backed by a hostname are now kept unresolved and expanded to **all** the IP -addresses the hostname maps to, at connection time (via `EndPoint.resolveAll()`). Previously only -the first address returned by DNS was tried, so a single non-responsive IP behind a multi-record -hostname could fail the initial connection (or a control-connection reconnect) even when the other -addresses were healthy. No configuration change is required to benefit from this. +addresses the hostname maps to, at connection time. Previously only the first address returned by +DNS was tried, so a single non-responsive IP behind a multi-record hostname could fail the initial +connection (or a control-connection reconnect) even when the other addresses were healthy. No +configuration change is required to benefit from this. + +The expansion goes through Netty's configured `AddressResolverGroup`, which is the resolver an +unresolved address already reached when it was handed to `Bootstrap.connect()`. A custom resolver +installed through `NettyOptions.afterBootstrapInitialized()` therefore keeps working, and +`Bootstrap.disableResolver()` is still honored. As before, with Netty's default (JDK) resolver the +lookup blocks the I/O event loop it runs on — install `DnsAddressResolverGroup` for non-blocking +resolution. As part of this change: From 603fa03ae47e3b6865c3ef34a1052c4c455ecd6a Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Wed, 29 Jul 2026 17:11:05 +0200 Subject: [PATCH 15/33] test: cover the removed local-DC contact-point check (CUSTOMER-588) This PR's first commit removes OptionalLocalDcHelper#checkLocalDatacenterCompatibility, but nothing tested that removal. CUSTOMER-588 is the bug it caused. A contact point given as a hostname is represented, before the control connection resolves it, by an ephemeral placeholder Node built by MetadataManager#addContactPoints via DefaultNode#newContactPoint. Its datacenter is always null and never populated -- real topology is attached to a different Node object matched by hostId (see MetadataManager#registerNode). The removed check compared the configured local DC against those placeholders, so it warned unconditionally whenever a local DC was configured, no matter where the contact points actually were: You specified as the local DC, but some contact points are from a different DC: Node(endPoint=..., hostId=null, hashCode=...)=null The new test builds a real placeholder Node the same way production does, and a resolved node that genuinely is in the configured local DC, then asserts no warning is logged. It asserts on the absence of any WARN rather than of one particular message, so a regression that reintroduces the false positive under different wording is still caught; should_warn_if_configured_dc_matches_no_node is the positive control for the same appender, so a silent capture failure cannot make it pass by accident. The retained "configured local DC does not match any node's datacenter" check, which inspects the resolved node map, is unaffected. Co-Authored-By: Claude Opus 5 (1M context) --- .../DefaultLoadBalancingPolicyInitTest.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java index b5e843d77d2..9c36cbebfee 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyInitTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -29,9 +30,12 @@ import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistance; import com.datastax.oss.driver.api.core.metadata.NodeState; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; +import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.InetSocketAddress; import java.util.UUID; import org.junit.Test; import org.junit.runner.RunWith; @@ -210,6 +214,38 @@ public void should_warn_if_configured_dc_matches_no_node() { .isTrue(); } + @Test + public void should_not_warn_about_dc_mismatch_when_the_only_real_node_matches_configured_dc() { + // Given — CUSTOMER-588. A contact point given as a hostname is represented, before the control + // connection resolves it, by an ephemeral placeholder Node (built by + // MetadataManager#addContactPoints via DefaultNode#newContactPoint) whose datacenter is always + // null: it is never populated, because real topology is attached to a *different* Node object + // matched by hostId (see MetadataManager#registerNode). + // + // The removed OptionalLocalDcHelper#checkLocalDatacenterCompatibility compared the configured + // local DC against *those* placeholders, so it warned unconditionally whenever a local DC was + // configured, no matter where the contact points actually were. Here the only node carrying + // real, resolved metadata (node1) genuinely is in the configured local DC ("dc1", per base + // setup). + DefaultNode ephemeralContactPointNode = + DefaultNode.newContactPoint( + new DefaultEndPoint(new InetSocketAddress("127.0.0.9", 9042)), context); + when(metadataManager.getContactPoints()).thenReturn(ImmutableSet.of(ephemeralContactPointNode)); + DefaultLoadBalancingPolicy policy = createPolicy(); + + // When + policy.init(ImmutableMap.of(UUID.randomUUID(), node1), distanceReporter); + + // Then — no WARN at all. The retained check inspects the resolved node map, where node1 + // matches. + // Asserting that nothing is warned, rather than that one particular message is absent, also + // catches a regression that brings the false positive back under different wording. + // should_warn_if_configured_dc_matches_no_node is the positive control for this same appender, + // so a silent capture failure cannot make this pass by accident. + verify(appender, never()).doAppend(argThat(event -> event.getLevel() == Level.WARN)); + assertThat(policy.getLocalDatacenter()).isEqualTo("dc1"); + } + @NonNull protected DefaultLoadBalancingPolicy createPolicy() { return new DefaultLoadBalancingPolicy(context, DriverExecutionProfile.DEFAULT_NAME); From b83a8ae8a49a67fad2f94535f8b6d8aa5a0410e4 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Thu, 30 Jul 2026 00:28:35 +0200 Subject: [PATCH 16/33] fix: do not pin an endpoint to the address it already holds (DRIVER-201) Both CI failures introduced by 648c824aac trace to the pinning half of it. ZeroTokenNodesIT (5 tests, Scylla serial jobs) ---------------------------------------------- ChannelFactory pins every connection's endpoint to the address it reached, including already-resolved ones -- and for those the "address it reached" is the address the endpoint already holds, since a resolved candidate passes through the resolver untouched. The resulting copy was indistinguishable from the original except in toString(), which grew a redundant suffix: /127.0.13.3:9042(/127.0.13.3:9042) DefaultEndPoint.pinTo() now returns this when the requested address is the one it already holds. For this class that is a genuine no-op -- resolve(), resolveAll() and toString() all keep yielding exactly what they did -- so it also spares an allocation on every connect to a resolved endpoint, which is every node discovered from the peers rows. Deliberately not applied to SniEndPoint or ClientRoutesEndPoint: their unpinned resolve() resolves lazily (getAllByName() on the proxy hostname, ClientRoutesTopologyMonitor.resolveAddress()), so for them a pinned copy is meaningful even when it matches the stored address -- that is what took the blocking lookup off the event loop in SniSslEngineFactory#newSslEngine(). MockResolverIT.should_connect_with_mocked_hostname (isolated jobs) ----------------------------------------------------------------- This one is the intended behaviour change, so the assertion is updated rather than the code. The control node's endpoint is now the pinned copy (DefaultTopologyMonitor#buildNodeEndPoint stores the channel's endpoint), so resolve() yields the IP the control connection landed on instead of the unresolved hostname. The test now asserts that, plus that asMetricPrefix() still keys off the hostname -- the pinned copy denotes the same node. The guarantee the old assertion protected is unaffected: contact points stay unresolved and are re-added to the reconnection plan, which is what lets a replaced cluster be picked up. replace_cluster_test() covers that and passes. The residual trade-off is deliberate: a node identified through a hostname keeps reconnecting to the pinned IP, so if that IP changes under a stable host id, recovery goes through the contact points rather than through the node itself. That is the cost of the stable node identity requested in review. Co-Authored-By: Claude Opus 5 (1M context) --- .../internal/core/metadata/DefaultEndPoint.java | 9 ++++++++- .../internal/core/metadata/PinnableEndPoint.java | 3 ++- .../core/metadata/DefaultEndPointTest.java | 13 +++++++++++++ .../oss/driver/core/resolver/MockResolverIT.java | 14 ++++++++++---- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index 6503d81b2cf..ddf9e050107 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -86,7 +86,14 @@ public CompletionStage resolveAll(@NonNull Executor executor) { public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { Objects.requireNonNull(resolvedAddress, "resolvedAddress can't be null"); if (!(resolvedAddress instanceof InetSocketAddress) - || resolvedAddress.equals(this.pinnedAddress)) { + || resolvedAddress.equals(this.pinnedAddress) + // The address we already hold: pinning to it changes nothing, since resolve(), resolveAll() + // and toString() would keep yielding what they already do. Skipping the copy spares + // toString() a redundant "addr(addr)" suffix on every already-resolved endpoint -- which is + // all of them, once a node is discovered from the peers rows. This shortcut is specific to + // this class, where resolve() is a field read; SniEndPoint and ClientRoutesEndPoint resolve + // lazily, so for them a pinned copy is meaningful even when it matches. + || resolvedAddress.equals(this.address)) { return this; } return new DefaultEndPoint(address, (InetSocketAddress) resolvedAddress); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java index cfb34e1307f..fcfbb43d9ab 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java @@ -59,7 +59,8 @@ public interface PinnableEndPoint extends EndPoint { * Returns a copy of this endpoint that resolves to exactly {@code resolvedAddress}. * *

Implementations may return {@code this} when pinning does not apply (for example when the - * address is not of a type they can hold on to). + * address is not of a type they can hold on to), or when it would be a no-op because the endpoint + * already resolves to exactly that address. * * @param resolvedAddress the address a connection was successfully established to; must not be * null and must already be resolved. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index 6e04038725f..a4b821b8d21 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -159,6 +159,19 @@ public void pin_to_should_return_same_instance_when_already_pinned_to_that_addre assertThat(((DefaultEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); } + @Test + public void pin_to_should_return_same_instance_when_address_is_already_the_endpoints_own() { + // An already-resolved endpoint expands to exactly one candidate -- itself -- so ChannelFactory + // pins it to the address it already holds. That copy would be indistinguishable from the + // original, and would only add a redundant suffix to toString(). Every node discovered from the + // peers rows takes this path. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + DefaultEndPoint endPoint = new DefaultEndPoint(resolved); + + assertThat(endPoint.pinTo(new InetSocketAddress("127.0.0.1", 9042))).isSameAs(endPoint); + assertThat(endPoint.toString()).isEqualTo(resolved.toString()).doesNotContain("("); + } + @Test public void pin_to_should_reject_null_address() { DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java index cb4f1abfa5c..6ae767389eb 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/resolver/MockResolverIT.java @@ -25,7 +25,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import com.datastax.oss.driver.api.core.CqlSession; @@ -106,9 +105,16 @@ public void should_connect_with_mocked_hostname() { .filter(x -> x.toString().contains("test.cluster.fake")) .collect(Collectors.toSet()); assertThat(filteredNodes).hasSize(1); - InetSocketAddress address = - (InetSocketAddress) filteredNodes.iterator().next().getEndPoint().resolve(); - assertTrue(address.isUnresolved()); + Node node = filteredNodes.iterator().next(); + InetSocketAddress address = (InetSocketAddress) node.getEndPoint().resolve(); + // ChannelFactory pins the control connection's endpoint to the address it actually reached, + // and DefaultTopologyMonitor#buildNodeEndPoint stores that copy for the control node, so + // resolution yields that concrete IP rather than the hostname. + assertFalse(address.isUnresolved()); + assertThat(address.getAddress().getHostAddress()).isEqualTo(ccmBridge.getNodeIpAddress(1)); + // The pinned copy still denotes the same node by hostname though -- it is what the filter + // above matched on -- so metric names do not depend on which IP a connection landed on. + assertThat(node.getEndPoint().asMetricPrefix()).isEqualTo("test_cluster_fake:9042"); } } } From 5b79b630b6b401377d9ff40251a676a5216247af Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 01:16:43 +0200 Subject: [PATCH 17/33] refactor: make resolution a connection-layer concern, drop the EndPoint API addition (DRIVER-201) `EndPoint.resolveAll(Executor)` was added for two reasons: return every IP a hostname maps to, and do it asynchronously so the admin event loop never blocks on DNS. `resolve()` was deprecated on that basis. Moving resolution into ChannelFactory (648c824aac) invalidated both: - Multiplicity is produced by ChannelFactory through Netty's resolver, not by resolveAll(). DefaultEndPoint.resolveAll() had become literally `completedFuture(new SocketAddress[]{resolve()})` -- one element, no lookup, the executor never touched. - Asynchrony was only still needed because SniEndPoint and ClientRoutesEndPoint chose to resolve internally. Both can simply stop, which is what this does. Meanwhile resolve() -- deprecated for "missing fallback IPs" -- is what the driver itself calls in eight places, and pinning had made it the precise accessor for "the address this channel is on". The deprecation had become advice against the driver's own design. Neither resolveAll() nor the @Deprecated exists on scylla-4.x: both were new in this PR, so there is nothing to keep compatible with. Every EndPoint implementation in the repo is internal. The principle ------------- An EndPoint describes *where* a node is. It never performs name resolution. Resolution happens once, in ChannelFactory, through Netty's AddressResolverGroup. - EndPoint: resolveAll() removed, resolve() un-deprecated. Its contract now states that returning an unresolved address is how multi-address support works, and that implementations must neither resolve names nor block. - SniEndPoint: no more getAllByName(), no rotation counters, no IP comparator. resolve() returns the pinned proxy IP, else the configured proxy address -- already unresolved, as CloudConfigFactory builds it. Netty expands it, so SNI gains multi-proxy-IP fallback *and* custom-resolver support, neither of which it had. - ClientRoutesEndPoint / ClientRoutesTopologyMonitor: the route hostname is returned unresolved from the in-memory cache instead of going through InetAddress.getByName(). resolve() is now a pure cache read; the protected resolveAddress() hook is gone with its only caller. - ChannelFactory: takes the single address from resolve() and expands it. The resolver thread pool is deleted outright -- nothing blocks any more -- along with its advanced.netty.daemon handling, close(), and the DefaultSession call. The round-robin SniEndPoint used to do moves here as rotate(), so it now applies to every endpoint type rather than only SNI. - PinnableEndPoint is kept as-is: internal, and the part of 648c824aac that earns its place. Against dkropachev's review round, this leaves three comments fixed as they were (GSSAPI NPE, node identity, blocking DNS in newSslEngine -- all by pinning), gives a better answer to two (the client-routes blocking is eliminated rather than offloaded; the custom resolver now reaches SNI and client routes too), and makes one moot (no resolver threads left to honour advanced.netty.daemon). Also fixed here --------------- - DefaultNode.setEndPoint() gated its whole body on !equals(), and equals() ignores pinnedAddress by contract -- so a stale pin could never be replaced and the control node stayed frozen on the first address it connected to, even after the control connection had moved and told us about it. It now always adopts the newest instance, with only the metric-updater rebuild still gated on a genuine address change (asMetricPrefix() is pin-independent, so a pin-only change must not churn metrics). - TopologyMonitor.reresolvesNodeAddresses() claimed the connected node's endpoint re-resolves on every connection attempt. Pinning made that false; the javadoc now says the endpoint is bound to the address its control connection reached, and that recovery depends on this flag being false. - Eight @SuppressWarnings("deprecation") annotations that existed only for the resolve() deprecation are removed. Behaviour worth calling out: resolve() on an *unpinned* SniEndPoint or ClientRoutesEndPoint may now return an unresolved address where it previously returned a resolved one. Every in-tree caller holds a channel endpoint, which is always pinned (SniSslEngineFactory, DefaultTopologyMonitor#savePort and #getBroadcastRpcAddress, GssApiAuthenticator); InsightsClient reads node endpoints, which are resolved for peers and pinned for the control node. A third-party EndPoint that blocks inside resolve() will block the admin loop again, exactly as in the released driver -- this gives up an improvement the previous revision of this PR briefly offered, in exchange for no public API change at all. Tests: ChannelFactoryAsyncResolveTest and ChannelFactoryResolveAllGuardTest are deleted (they guarded contracts that no longer exist); ChannelFactoryMultiAddressTest now drives expansion through a resolver and covers rotation plus a throwing resolve(); the resolver stub is extracted to TestAddressResolverGroup and shared; the endpoint tests assert that no endpoint performs a lookup; DefaultNodeTest covers pin adoption and metric non-churn. Co-Authored-By: Claude Opus 5 (1M context) --- .../core/auth/DseGssApiAuthProviderBase.java | 2 - .../core/insights/InsightsClient.java | 6 - .../api/core/config/TypedDriverOption.java | 6 +- .../driver/api/core/metadata/EndPoint.java | 77 ++---- .../api/core/session/SessionBuilder.java | 2 +- .../internal/core/channel/ChannelFactory.java | 227 +++++++----------- .../core/metadata/ClientRoutesEndPoint.java | 65 ++--- .../metadata/ClientRoutesTopologyMonitor.java | 25 +- .../core/metadata/DefaultEndPoint.java | 47 ++-- .../internal/core/metadata/DefaultNode.java | 13 +- .../core/metadata/DefaultTopologyMonitor.java | 4 - .../metadata/LoadBalancingPolicyWrapper.java | 6 +- .../core/metadata/MetadataManager.java | 3 +- .../internal/core/metadata/SniEndPoint.java | 128 ++-------- .../core/metadata/TopologyMonitor.java | 12 +- .../internal/core/session/DefaultSession.java | 8 - .../core/ssl/DefaultSslEngineFactory.java | 2 - .../core/ssl/SniSslEngineFactory.java | 2 - .../ChannelFactoryAsyncResolveTest.java | 146 ----------- .../ChannelFactoryMultiAddressTest.java | 155 ++++++++---- .../ChannelFactoryNettyResolverTest.java | 105 +------- .../ChannelFactoryPinnedEndPointTest.java | 69 +++--- .../ChannelFactoryResolveAllGuardTest.java | 136 ----------- .../internal/core/channel/LocalEndPoint.java | 10 - .../channel/TestAddressResolverGroup.java | 111 +++++++++ .../metadata/ClientRoutesEndPointTest.java | 95 +------- .../ClientRoutesTopologyMonitorTest.java | 13 +- .../core/metadata/DefaultEndPointTest.java | 60 ++--- .../core/metadata/DefaultNodeTest.java | 42 ++++ .../core/metadata/SniEndPointTest.java | 143 +++-------- .../core/clientroutes/ClientRoutesIT.java | 4 +- manual/core/address_resolution/README.md | 21 +- upgrade_guide/README.md | 10 + 33 files changed, 597 insertions(+), 1158 deletions(-) delete mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java delete mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java diff --git a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java index 8d913127355..beab2f488a8 100644 --- a/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java +++ b/core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java @@ -340,8 +340,6 @@ protected GssApiAuthenticator( * NullPointerException}: the hostname is usually the right service name anyway, and a failed * reverse lookup should not take authentication down. */ - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); - // Kerberos authentication needs a single canonical hostname for SASL service name resolution. private static String serverName(EndPoint endPoint) { InetSocketAddress address = (InetSocketAddress) endPoint.resolve(); InetAddress inetAddress = address.getAddress(); diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java b/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java index 54d723152a4..168477894ed 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java @@ -288,8 +288,6 @@ private InsightsStatusData createStatusData() { .build(); } - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); - // address reporting needs a single canonical address per node. private Map getConnectedNodes() { Map pools = driverContext.getPoolManager().getPools(); return pools.entrySet().stream() @@ -304,8 +302,6 @@ private SessionStateForNode constructSessionStateForNode(Map.Entry startupOptions = driverContext.getStartupOptions(); return InsightsStartupData.builder() @@ -458,8 +454,6 @@ private PoolSizeByHostDistance getPoolSizeByHostDistance() { 0); } - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); - // address reporting needs a single canonical address for the control connection. private String getControlConnectionSocketAddress() { SocketAddress controlConnectionAddress = controlConnection.channel().getEndPoint().resolve(); return AddressFormatter.nullSafeToString(controlConnectionAddress); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index 21c5f68d3d1..a46757fb967 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -605,9 +605,9 @@ public String toString() { * after the live nodes reported by the load balancing policy (defaults to {@code true}). * *

Contact points are appended as-is (unresolved hostnames); each is expanded to all of its - * current DNS IPs at connection time via {@code EndPoint.resolveAll()}, which is also the - * driver's DNS re-resolution mechanism. The append is skipped for topology monitors that - * re-resolve node addresses themselves (such as the cloud/proxy monitors). + * current DNS IPs at connection time, which is also the driver's DNS re-resolution mechanism. The + * append is skipped for topology monitors that re-resolve node addresses themselves (such as the + * cloud/proxy monitors). */ public static final TypedDriverOption CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS = new TypedDriverOption<>( diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java index 54407b6c0e8..af15682a0f3 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java @@ -19,9 +19,6 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.net.SocketAddress; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; /** * Encapsulates the information needed to open connections to a node. @@ -34,68 +31,34 @@ public interface EndPoint { /** - * Resolves this instance to a single socket address. + * Resolves this instance to the socket address connections should be opened to. * *

This will be called each time the driver opens a new connection to the node. The returned * address cannot be null. * - * @deprecated Use {@link #resolveAll(Executor)} instead. When a hostname maps to multiple IPs - * (e.g. in dynamic DNS environments) only one address is returned here, causing the driver to - * miss fallback IPs when the first one is unreachable. {@code resolveAll(Executor)} returns - * the full set, resolved asynchronously off the calling thread. - */ - @Deprecated - @NonNull - SocketAddress resolve(); - - /** - * Resolves this instance to all the candidate socket addresses a connection may be opened to, - * asynchronously. + *

Returning a hostname is fine, and is how multi-address support works. The returned + * address need not be resolved: an {@linkplain java.net.InetSocketAddress#isUnresolved() + * unresolved} {@link java.net.InetSocketAddress} is expanded by the driver to every + * address the name maps to, and each one is tried in turn until a connection succeeds. That is + * what {@link com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint} does for contact + * points backed by a hostname, so a single unreachable IP behind a multi-record name no longer + * fails the connection. * - *

This is called each time the driver opens a new connection to the node. For endpoints backed - * by a plain IP address the returned array contains exactly one element. For endpoints that know - * of several addresses (e.g. an SNI proxy with multiple A-records) all of them are returned so - * that the driver can try each one in sequence and fall back gracefully when individual IPs are - * unreachable. + *

Implementations must not resolve names themselves, and must not block. The driver + * calls this from its admin event loop, and it performs the expansion through Netty's configured + * {@code AddressResolverGroup} — the same resolver an unresolved address reaches when it is + * handed to {@code Bootstrap.connect()}. Looking the name up here instead (for example with + * {@link java.net.InetAddress#getAllByName(String)}) would both block that loop and bypass a + * custom resolver installed via {@code NettyOptions#afterBootstrapInitialized(Bootstrap)}. * - *

Returning a hostname is fine. Candidates need not be resolved: an {@linkplain - * java.net.InetSocketAddress#isUnresolved() unresolved} {@link java.net.InetSocketAddress} is - * expanded to every address it maps to by the driver, through Netty's configured {@code - * AddressResolverGroup}. That is what {@link - * com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint} does, and it is preferable to - * looking the name up here: it keeps a custom resolver installed via {@code - * NettyOptions#afterBootstrapInitialized(Bootstrap)} in the loop, which a direct {@link - * java.net.InetAddress#getAllByName(String)} call would bypass. - * - *

Resolution is asynchronous on purpose: name resolution can block (e.g. {@link - * java.net.InetAddress#getAllByName(String)}), and the driver calls this from its admin event - * loop, which must never block. Implementations whose resolution may block must run it on - * the supplied {@code executor} rather than on the calling thread (see the default - * implementation). Implementations that resolve from memory (e.g. an already-resolved address or - * an in-memory lookup) may return an {@linkplain CompletableFuture#completedFuture(Object) - * already completed stage} and ignore the executor. - * - *

The default implementation offloads {@link #resolve()} to {@code executor} and wraps the - * result in a single-element array. Implementations that can supply multiple addresses should - * override this method. - * - *

The returned stage must not be null and must complete with a non-null, non-empty array. - * - * @param executor the executor to run potentially-blocking resolution on; must not be null. The - * driver supplies a dedicated resolver executor so that blocking name resolution never runs - * on an event loop. - * @apiNote Timeout note: {@link - * com.datastax.oss.driver.internal.core.channel.ChannelFactory} tries each address in - * sequence. If a hostname resolves to N addresses and each attempt times out, the worst-case - * connection time before declaring a node unreachable is {@code N × - * advanced.connection.connect-timeout}. In practice DNS round-robin entries have only a small - * number of records, so this is rarely a concern, but callers should be aware of this when - * configuring connect timeouts. + * @apiNote Timeout note: when a name expands to several addresses they are tried in + * sequence, so if every attempt times out the worst-case time before the node is declared + * unreachable is {@code N × advanced.connection.connect-timeout}. In practice DNS round-robin + * entries have only a small number of records, so this is rarely a concern, but it is worth + * bearing in mind when configuring connect timeouts. */ @NonNull - default CompletionStage resolveAll(@NonNull Executor executor) { - return CompletableFuture.supplyAsync(() -> new SocketAddress[] {resolve()}, executor); - } + SocketAddress resolve(); /** * Returns an alternate string representation for use in node-level metric names. diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index a6ca2da78e6..bcb54fa81dd 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -962,7 +962,7 @@ protected final CompletionStage buildDefaultSessionAsync() { } // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved - // hostnames and expanded to all their DNS IPs at connection time via EndPoint.resolveAll(). + // hostnames, and expanded to all their DNS IPs at connection time by ChannelFactory. Set contactPoints = ContactPoints.merge(programmaticContactPoints, configContactPoints, false); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index d2ccae3d428..8b86ea36c23 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -64,7 +64,8 @@ import java.net.ServerSocket; import java.net.SocketAddress; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -72,10 +73,6 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import net.jcip.annotations.ThreadSafe; @@ -120,28 +117,12 @@ public class ChannelFactory { protected final InternalDriverContext context; /** - * Maximum number of threads used to run {@link - * EndPoint#resolveAll(java.util.concurrent.Executor)} calls concurrently (see {@link - * #resolverExecutor}). DNS resolution is I/O-bound, and a fixed bound avoids unbounded thread - * creation during a large simultaneous reconnect burst while still giving plenty of headroom (DNS - * lookups are fast; contention here should be rare). + * Round-robin counter used by {@link #rotate} to vary which of a name's addresses a connection + * tries first. Static so the rotation spans every endpoint and session in the JVM, which is all + * the spreading it needs to do; {@code SniEndPoint} used to hold an equivalent counter of its + * own, before resolution moved here. */ - private static final int RESOLVER_MAX_THREADS = 16; - - /** - * Handed to {@link EndPoint#resolveAll(java.util.concurrent.Executor)} so implementations whose - * resolution blocks ({@code SniEndPoint}, {@code ClientRoutesEndPoint}, third-party endpoints) - * run it off the caller thread: {@code connect()} is invoked from the admin event loop, which - * must never block. - * - *

{@code DefaultEndPoint} does not use this — hostname expansion goes through Netty's resolver - * instead, see {@link #resolveCandidates}. - * - *

Bounded at {@link #RESOLVER_MAX_THREADS} threads with an unbounded queue, so a submission is - * never rejected and never runs inline on the caller: excess resolutions simply queue until a - * thread frees up. Idle threads are reclaimed after ~60s. - */ - private final ExecutorService resolverExecutor; + private static final AtomicInteger ROTATION_OFFSET = new AtomicInteger(); /** either set from the configuration, or null and will be negotiated */ @VisibleForTesting volatile ProtocolVersion protocolVersion; @@ -162,28 +143,6 @@ public ChannelFactory(InternalDriverContext context) { DriverExecutionProfile defaultConfig = context.getConfig().getDefaultProfile(); - // Same setting that governs the Netty I/O, admin and timer threads (see DefaultNettyOptions): - // every thread the driver creates should behave the same way with respect to JVM exit. - boolean daemon = defaultConfig.getBoolean(DefaultDriverOption.NETTY_DAEMON); - AtomicInteger resolverThreadCount = new AtomicInteger(); - ThreadPoolExecutor resolverThreadPool = - new ThreadPoolExecutor( - RESOLVER_MAX_THREADS, - RESOLVER_MAX_THREADS, - 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - runnable -> { - Thread thread = - new Thread( - runnable, - logPrefix + "-connection-resolver-" + resolverThreadCount.incrementAndGet()); - thread.setDaemon(daemon); - return thread; - }); - resolverThreadPool.allowCoreThreadTimeOut(true); - this.resolverExecutor = resolverThreadPool; - if (defaultConfig.isDefined(DefaultDriverOption.PROTOCOL_VERSION)) { String versionName = defaultConfig.getString(DefaultDriverOption.PROTOCOL_VERSION); this.protocolVersion = context.getProtocolVersionRegistry().fromName(versionName); @@ -213,18 +172,6 @@ public String getClusterName() { return clusterName; } - /** - * Shuts down the internal name-resolver executor. Invoked during session shutdown, from {@code - * DefaultSession}'s close sequence. - * - *

Its threads follow {@code advanced.netty.daemon}, exactly like the Netty I/O and admin - * threads, so with the default (non-daemon) setting this call is what lets the JVM exit — the - * same contract as {@link NettyOptions#onClose()}. - */ - public void close() { - resolverExecutor.shutdownNow(); - } - public CompletionStage connect(Node node, DriverChannelOptions options) { NodeMetricUpdater nodeMetricUpdater; if (node instanceof DefaultNode) { @@ -299,33 +246,21 @@ private void connect( return; } - // Resolution may block (DNS); it is offloaded to resolverExecutor so the calling thread (the - // admin event loop, for control-connection reconnects) never waits on it. The continuation - // below runs off the admin loop -- tryNextCandidate()/Bootstrap.connect() are safe from any - // thread and per-candidate retries already run on Netty I/O threads. - CompletionStage resolveStage; + // EndPoint.resolve() is contractually non-blocking and performs no name resolution, so it is + // safe to call here even though connect() runs on the admin event loop for control-connection + // reconnects. Everything a name needs to become connectable happens in resolveCandidates(). + SocketAddress address; try { - resolveStage = endPoint.resolveAll(resolverExecutor); + address = endPoint.resolve(); } catch (Exception e) { - // An implementation may throw synchronously while building the stage. resultFuture.completeExceptionally(e); return; } - resolveStage - .thenCompose( - candidates -> { - if (candidates == null || candidates.length == 0) { - throw new IllegalArgumentException( - "EndPoint.resolveAll() must return a non-null, non-empty array: " + endPoint); - } - return resolveCandidates(baseBootstrap, candidates); - }) + resolveCandidates(baseBootstrap, address) .whenComplete( (candidates, error) -> { if (error != null) { - // supplyAsync wraps supplier throwables in a CompletionException; unwrap so callers - // (and the guard test's isSameAs assertion) see the original cause. Throwable cause = (error instanceof CompletionException && error.getCause() != null) ? error.getCause() @@ -367,18 +302,22 @@ private Bootstrap newBootstrap() { } /** - * Turns the raw candidates returned by {@link EndPoint#resolveAll(java.util.concurrent.Executor)} - * into concrete, connectable addresses, expanding any unresolved candidate to all the - * addresses it maps to. + * Turns the address an {@link EndPoint} denotes into the concrete, connectable addresses to try, + * expanding it to all the addresses it maps to when it is a name. * *

Expansion goes through the bootstrap's Netty {@link AddressResolverGroup} rather than a * direct {@code InetAddress.getAllByName()} call, so a custom resolver installed via {@link * NettyOptions#afterBootstrapInitialized(Bootstrap)} is honoured — that is the resolver an * unresolved address would have reached had it been handed straight to {@code - * Bootstrap.connect()}, as it was before multi-address support. Candidates the resolver does not - * support (e.g. {@link io.netty.channel.local.LocalAddress}) or that are already resolved are - * passed through untouched; this mirrors {@code Bootstrap#doResolveAndConnect0}. A null group - * means the user called {@link Bootstrap#disableResolver()}, which is likewise respected. + * Bootstrap.connect()}, as it was before multi-address support. This is also why endpoints are + * forbidden from resolving names themselves (see {@link EndPoint#resolve()}): doing it here is + * the only way to keep that configuration point working, and the only way to keep {@code + * resolve()} non-blocking. + * + *

An address the resolver does not support (e.g. {@link io.netty.channel.local.LocalAddress}) + * or that is already resolved is passed through untouched, mirroring {@code + * Bootstrap#doResolveAndConnect0}. A null group means the user called {@link + * Bootstrap#disableResolver()}, which is likewise respected. * *

Note that with Netty's default resolver the lookup blocks the event loop it runs on, * because {@code DefaultNameResolver} performs {@code InetAddress.getAllByName()} inline. That is @@ -386,17 +325,23 @@ private Bootstrap newBootstrap() { * it is an I/O loop, never the admin loop that {@code connect()} is called from. Deployments that * need non-blocking resolution can now install {@code DnsAddressResolverGroup} and have it take * effect. - * - *

A candidate that fails to resolve is skipped, so one bad entry does not mask the others; the - * stage only fails if every candidate fails, carrying the first failure as the cause. */ private CompletionStage> resolveCandidates( - Bootstrap bootstrap, SocketAddress[] candidates) { + Bootstrap bootstrap, SocketAddress address) { + + // Nothing to expand, and nothing for a resolver to contribute: an already-resolved address is + // exactly what we will connect to, and the resolver would pass it through untouched anyway. + // Worth + // short-circuiting because this is the common case -- every node discovered from the peers rows + // holds a resolved address, so this is every pool refill and every reconnect. + if (isResolved(address)) { + return CompletableFuture.completedFuture(Collections.singletonList(address)); + } AddressResolverGroup resolverGroup = bootstrap.config().resolver(); if (resolverGroup == null) { - // Bootstrap.disableResolver(): the user wants addresses passed through as-is. - return CompletableFuture.completedFuture(Arrays.asList(candidates)); + // Bootstrap.disableResolver(): the user wants the address passed through as-is. + return CompletableFuture.completedFuture(Collections.singletonList(address)); } // The resolver must be obtained for -- and used from -- an event executor whose transport @@ -414,62 +359,66 @@ private CompletionStage> resolveCandidates( result.completeExceptionally(t); return; } - expandCandidate(resolver, candidates, 0, new ArrayList<>(), null, result); + if (!resolver.isSupported(address) || resolver.isResolved(address)) { + // Nothing for the resolver to do; same short-circuit as Bootstrap#doResolveAndConnect0. + result.complete(Collections.singletonList(address)); + return; + } + resolver + .resolveAll(address) + .addListener( + (Future> future) -> { + if (!future.isSuccess()) { + result.completeExceptionally(future.cause()); + return; + } + @SuppressWarnings("unchecked") + List addresses = + (List) future.getNow(); + if (addresses == null || addresses.isEmpty()) { + result.completeExceptionally( + new IllegalStateException("Resolver returned no address for " + address)); + return; + } + result.complete(rotate(addresses)); + }); }); return result; } + /** Whether {@code address} is already connectable, i.e. needs no resolver at all. */ + private static boolean isResolved(SocketAddress address) { + return address instanceof InetSocketAddress && !((InetSocketAddress) address).isUnresolved(); + } + /** - * Resolves {@code candidates[index]}, then recurses on the next one. Runs on the resolver's loop. + * Rotates the expanded address list so that successive connections to the same name do not all + * start at the same address. + * + *

Without this, every connection would try the resolver's first address first and healthy + * connections would pile onto one IP; the whole point of a multi-record name is usually to spread + * them. All addresses are still returned, and in the same cyclic order, so a single attempt can + * still fall back across every one of them. + * + *

The list is sorted first so that a given rotation offset maps to the same address on every + * call, regardless of the order the resolver happened to return. Only that determinism matters + * here, not the ordering itself, so a plain string comparison is enough. */ - private void expandCandidate( - AddressResolver resolver, - SocketAddress[] candidates, - int index, - List resolved, - Throwable firstError, - CompletableFuture> result) { - - if (index == candidates.length) { - if (resolved.isEmpty()) { - result.completeExceptionally( - firstError != null - ? firstError - : new IllegalArgumentException( - "Could not resolve any of " + Arrays.toString(candidates))); - } else { - result.complete(resolved); - } - return; + @VisibleForTesting + static List rotate(List addresses) { + int size = addresses.size(); + if (size == 1) { + // Nothing to rotate, and don't burn a rotation offset on it. + return new ArrayList<>(addresses); } - - SocketAddress candidate = candidates[index]; - if (!resolver.isSupported(candidate) || resolver.isResolved(candidate)) { - // Nothing for the resolver to do; same short-circuit as Bootstrap#doResolveAndConnect0. - resolved.add(candidate); - expandCandidate(resolver, candidates, index + 1, resolved, firstError, result); - return; + List sorted = new ArrayList<>(addresses); + sorted.sort(Comparator.comparing(SocketAddress::toString)); + int start = Math.floorMod(ROTATION_OFFSET.getAndIncrement(), size); + List result = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + result.add(sorted.get((start + i) % size)); } - - resolver - .resolveAll(candidate) - .addListener( - (Future> future) -> { - Throwable error = firstError; - if (future.isSuccess()) { - @SuppressWarnings("unchecked") - List addresses = - (List) future.getNow(); - resolved.addAll(addresses); - } else { - LOG.debug( - "[{}] Could not resolve {}, skipping it", logPrefix, candidate, future.cause()); - if (error == null) { - error = future.cause(); - } - } - expandCandidate(resolver, candidates, index + 1, resolved, error, result); - }); + return result; } /** diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java index e6bad7bf8a3..571c5146702 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -20,16 +20,11 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; public class ClientRoutesEndPoint implements PinnableEndPoint { private final UUID hostId; @@ -85,62 +80,30 @@ public UUID getHostId() { return hostId; } - @NonNull - @Override - public SocketAddress resolve() { - if (pinnedAddress != null) { - return pinnedAddress; - } - try { - InetSocketAddress address = topologyMonitor.resolve(hostId); - if (address != null) { - return address; - } - } catch (IOException e) { - throw new UncheckedIOException("DNS resolution failed for host_id=" + hostId, e); - } - return fallbackEndPoint.resolve(); - } - /** - * Returns the candidate addresses for this endpoint. + * Returns the address connections should be opened to. * - *

The topology monitor resolves each node to exactly one address by design (a per-host-id - * lookup over {@code system.client_routes}), so this never expands to several candidates. It is - * still asynchronous, because the lookup is not purely in-memory: {@link - * ClientRoutesTopologyMonitor#resolve} can reach {@link - * ClientRoutesTopologyMonitor#resolveAddress} → {@link InetAddress#getByName}, which blocks. It - * therefore runs on {@code executor} rather than on the caller (the admin event loop, for - * control-connection reconnects). + *

The client route for this host id is an in-memory lookup over the cached {@code + * system.client_routes} contents, and it yields exactly one address by design, so this neither + * blocks nor expands to several candidates. The route's hostname is returned {@linkplain + * InetSocketAddress#isUnresolved() unresolved}: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup}, so a custom resolver is honoured and no DNS lookup + * runs on the caller (the admin event loop, for control-connection reconnects). * *

When the topology monitor has no route for this host id — i.e. the node is not reached - * through a cloud private endpoint — this delegates to {@code - * fallbackEndPoint.resolveAll(executor)} so the fallback keeps whatever resolution semantics it - * defines, rather than being flattened to its single {@link EndPoint#resolve()} address. + * through a cloud private endpoint — this delegates to the fallback endpoint. * - *

Once {@linkplain #pinTo(SocketAddress) pinned} the pinned address is returned directly, with - * no lookup and no executor hop. + *

Once {@linkplain #pinTo(SocketAddress) pinned} the pinned address is returned directly. */ @NonNull @Override - public CompletionStage resolveAll(@NonNull Executor executor) { + public SocketAddress resolve() { if (pinnedAddress != null) { - return CompletableFuture.completedFuture(new SocketAddress[] {pinnedAddress}); + return pinnedAddress; } - return CompletableFuture.>supplyAsync( - () -> { - InetSocketAddress address; - try { - address = topologyMonitor.resolve(hostId); - } catch (IOException e) { - throw new UncheckedIOException("DNS resolution failed for host_id=" + hostId, e); - } - return address != null - ? CompletableFuture.completedFuture(new SocketAddress[] {address}) - : fallbackEndPoint.resolveAll(executor); - }, - executor) - .thenCompose(stage -> stage); + InetSocketAddress address = topologyMonitor.resolve(hostId); + return address != null ? address : fallbackEndPoint.resolve(); } @NonNull diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java index a93c34463bd..31e476d866d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java @@ -33,7 +33,6 @@ import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.UnknownHostException; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -196,9 +195,18 @@ void setResolvedRoutes(Map routes) { resolvedRoutesCache.set(Collections.unmodifiableMap(new HashMap<>(routes))); } + /** + * Returns the client route for {@code hostId} as an {@linkplain InetSocketAddress#isUnresolved() + * unresolved} address, or {@code null} if this node has no route. + * + *

The route's hostname is deliberately left unresolved: {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} resolves it through Netty's + * configured {@code AddressResolverGroup} at connection time. That keeps this method a pure + * in-memory cache lookup, so it is safe to call from an event loop, and it means a custom + * resolver applies to client routes just like it does to contact points. + */ @Nullable - public InetSocketAddress resolve(@NonNull UUID hostId) - throws IllegalStateException, UnknownHostException { + public InetSocketAddress resolve(@NonNull UUID hostId) throws IllegalStateException { if (closed) { throw new IllegalStateException("Topology monitor is closed"); } @@ -207,7 +215,7 @@ public InetSocketAddress resolve(@NonNull UUID hostId) return null; // no client route for this node — caller falls back to default } - return new InetSocketAddress(resolveAddress(route.getHostname()), route.getPort()); + return InetSocketAddress.createUnresolved(route.getHostname(), route.getPort()); } /** @@ -662,13 +670,4 @@ public CompletionStage closeAsync() { LOG.debug("[{}] ClientRoutesTopologyMonitor closed", logPrefix); return super.closeAsync(); } - - /** - * Resolves a hostname to an {@link InetAddress}. Extracted as a protected method so that unit - * tests can override it to return stubbed addresses without hitting the network. - */ - @NonNull - protected InetAddress resolveAddress(@NonNull String hostname) throws UnknownHostException { - return InetAddress.getByName(hostname); - } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index ddf9e050107..9ff35594377 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -24,9 +24,6 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; public class DefaultEndPoint implements PinnableEndPoint, Serializable { @@ -52,33 +49,24 @@ private DefaultEndPoint(InetSocketAddress address, @Nullable InetSocketAddress p this.pinnedAddress = pinnedAddress; } - @NonNull - @Override - public InetSocketAddress resolve() { - return pinnedAddress != null ? pinnedAddress : address; - } - /** - * Returns the candidate addresses for this endpoint. - * - *

The returned stage always completes immediately, on the calling thread, with a single - * element, and never uses {@code executor}: this implementation performs no name resolution of - * its own. + * Returns the address connections should be opened to: the {@linkplain #pinTo(SocketAddress) + * pinned} one if this is a pinned copy, otherwise the stored address as-is. * - *

If the stored address is a hostname (i.e. {@linkplain InetSocketAddress#isUnresolved() - * unresolved} — contact points are always kept unresolved, see {@link - * com.datastax.oss.driver.api.core.session.SessionBuilder#addContactPoint}), the unresolved - * address is returned as-is, and {@link - * com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands it to every IP it maps to - * through Netty's configured {@code AddressResolverGroup}. Resolving there rather than here is - * deliberate: it keeps any custom resolver installed via {@link + *

This performs no name resolution. If the stored address is a hostname (i.e. {@linkplain + * InetSocketAddress#isUnresolved() unresolved} — contact points are always kept unresolved, see + * {@link com.datastax.oss.driver.api.core.session.SessionBuilder#addContactPoint}) it is returned + * unresolved, and {@link com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands it + * to every IP it maps to through Netty's configured {@code AddressResolverGroup}. Resolving there + * rather than here is deliberate: it keeps any custom resolver installed via {@link * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized} in the - * loop, which a direct {@code InetAddress.getAllByName()} call from here would bypass. + * loop, which a direct {@code InetAddress.getAllByName()} call from here would bypass, and it + * keeps this method non-blocking so it is safe to call from an event loop. */ @NonNull @Override - public CompletionStage resolveAll(@NonNull Executor executor) { - return CompletableFuture.completedFuture(new SocketAddress[] {resolve()}); + public InetSocketAddress resolve() { + return pinnedAddress != null ? pinnedAddress : address; } @NonNull @@ -87,12 +75,11 @@ public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { Objects.requireNonNull(resolvedAddress, "resolvedAddress can't be null"); if (!(resolvedAddress instanceof InetSocketAddress) || resolvedAddress.equals(this.pinnedAddress) - // The address we already hold: pinning to it changes nothing, since resolve(), resolveAll() - // and toString() would keep yielding what they already do. Skipping the copy spares - // toString() a redundant "addr(addr)" suffix on every already-resolved endpoint -- which is - // all of them, once a node is discovered from the peers rows. This shortcut is specific to - // this class, where resolve() is a field read; SniEndPoint and ClientRoutesEndPoint resolve - // lazily, so for them a pinned copy is meaningful even when it matches. + // The address we already hold: pinning to it changes nothing, since resolve() and + // toString() would keep yielding what they already do. Skipping the copy spares toString() + // a + // redundant "addr(addr)" suffix on every already-resolved endpoint -- which is all of them, + // once a node is discovered from the peers rows. || resolvedAddress.equals(this.address)) { return this; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java index 1b09c26ce16..e42c0b0d3cd 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java @@ -102,8 +102,17 @@ public EndPoint getEndPoint() { } public void setEndPoint(@NonNull EndPoint newEndPoint, @NonNull InternalDriverContext context) { - if (!newEndPoint.equals(endPoint)) { - endPoint = newEndPoint; + boolean differentNode = !newEndPoint.equals(endPoint); + // Adopt the newest instance even when it compares equal. A PinnableEndPoint copy differs from + // the original only by the address it is pinned to, and equals() ignores that by contract (see + // PinnableEndPoint) -- but it is the address every subsequent connection to this node will use, + // so refusing to adopt it would freeze the node on the first address it ever connected to, even + // after the control connection has moved to another one and told us about it. + endPoint = newEndPoint; + if (differentNode) { + // Only a genuine address change may rebuild the updater: asMetricPrefix() is pin-independent, + // so doing it for a pin-only change would clear and re-register metrics under identical + // names. // metricUpdater is transient, so it can be null on deserialized nodes. NodeMetricUpdater previousMetricUpdater = metricUpdater; if (previousMetricUpdater != null diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java index 2098a3e6553..5a82bfe2c86 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java @@ -701,8 +701,6 @@ private Optional findInPeers( // Current versions of Cassandra (3.11 at the time of writing), require the same port for all // nodes. As a consequence, the port is not stored in system tables. // We save it the first time we get a control connection channel. - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); a single - // canonical address is all that is needed here to extract the port. protected void savePort(DriverChannel channel) { if (port < 0) { SocketAddress address = channel.getEndPoint().resolve(); @@ -725,8 +723,6 @@ protected void savePort(DriverChannel channel) { * otherwise. */ @Nullable - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); a single - // canonical address is all that is needed here for the peer-vs-local comparison. protected InetSocketAddress getBroadcastRpcAddress( @NonNull AdminRow row, @NonNull EndPoint localEndPoint) { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index 1bee460d389..42382519601 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -150,7 +150,7 @@ public Queue newQueryPlan( case BEFORE_INIT: case DURING_INIT: // The contact points are not stored in the metadata yet. Each unresolved hostname is - // expanded to all its DNS IPs at connection time by EndPoint.resolveAll(), so one entry per + // expanded to all its DNS IPs at connection time by ChannelFactory, so one entry per // contact point is enough here. List nodes = new ArrayList<>(context.getMetadataManager().getContactPoints()); Collections.shuffle(nodes); @@ -201,8 +201,8 @@ public Queue newControlReconnectionQueryPlan() { && (!context.getTopologyMonitor().reresolvesNodeAddresses() || regularQueryPlan.isEmpty())) { // Append the original (unresolved) contact points so every IP their hostname resolves to is - // tried as a fallback: EndPoint.resolveAll() expands each one at connection time, instead of - // the driver being stuck with whatever single IP a metadata node happens to hold. + // tried as a fallback: ChannelFactory expands each one at connection time, instead of the + // driver being stuck with whatever single IP a metadata node happens to hold. List contactNodes = new ArrayList<>(); for (DefaultNode node : context.getMetadataManager().getContactPoints()) { contactNodes.add(DefaultNode.newContactPoint(node.getEndPoint(), context)); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java index 2ca9b6f1012..d8671678306 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/MetadataManager.java @@ -193,8 +193,7 @@ public boolean wasImplicitContactPoint() { * its own. Re-resolving the original contact-point hostname to pick up current DNS only happens * through the original-contact-point reconnection fallback (see {@code * advanced.control-connection.reconnection.fallback-to-original-contact-points}), which re-enters - * the contact points and lets {@link EndPoint#resolveAll(java.util.concurrent.Executor)} expand - * each hostname at connection time. + * the contact points and lets {@code ChannelFactory} expand each hostname at connection time. */ public CompletionStage registerNode(NodeInfo nodeInfo) { Preconditions.checkNotNull(nodeInfo.getHostId(), "Cannot register node without hostId"); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index f4cb0eeeba1..b70de4498e0 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -18,30 +18,13 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.shaded.guava.common.primitives.UnsignedBytes; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Comparator; import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicInteger; public class SniEndPoint implements PinnableEndPoint { - // Rotates the single address returned by resolve() (still used for SSL engine setup). - private static final AtomicInteger OFFSET = new AtomicInteger(); - // Rotates the starting candidate of resolveAll(). Kept separate from OFFSET so that SSL engine - // creation (which calls the deprecated resolve() once per connection) does not advance the - // resolveAll() counter a second time -- otherwise the start index would move in steps of 2 and - // rotation would collapse to a single IP whenever the proxy resolves to an even number of - // A-records (DRIVER-201). - private static final AtomicInteger RESOLVE_ALL_OFFSET = new AtomicInteger(); private final InetSocketAddress proxyAddress; private final String serverName; @@ -54,10 +37,9 @@ public class SniEndPoint implements PinnableEndPoint { @Nullable private final InetSocketAddress pinnedAddress; /** - * @param proxyAddress the address of the proxy. If it is {@linkplain - * InetSocketAddress#isUnresolved() unresolved}, each call to {@link #resolve()} will - * re-resolve it, fetch all of its A-records, and if there are more than 1 pick one in a - * round-robin fashion. + * @param proxyAddress the address of the proxy. It is returned by {@link #resolve()} as-is, so if + * it is {@linkplain InetSocketAddress#isUnresolved() unresolved} the driver expands it to all + * of the proxy's A-records at connection time and tries each of them. * @param serverName the SNI server name. In the context of Cloud, this is the string * representation of the host id. */ @@ -79,96 +61,24 @@ public String getServerName() { } /** - * Resolves this endpoint to a single proxy address. + * Returns the proxy address connections should be opened to. * - *

Once {@linkplain #pinTo(SocketAddress) pinned} this is a field read. That matters because - * {@link com.datastax.oss.driver.internal.core.ssl.SniSslEngineFactory#newSslEngine} calls this - * from Netty's channel initializer, i.e. on an I/O event loop: on an unpinned endpoint it would - * perform a blocking {@link InetAddress#getAllByName(String)} lookup there, and could pick a - * different proxy IP than the one the channel is actually connected to. - */ - @NonNull - @Override - public InetSocketAddress resolve() { - if (pinnedAddress != null) { - return pinnedAddress; - } - try { - InetAddress[] aRecords = InetAddress.getAllByName(proxyAddress.getHostName()); - if (aRecords.length == 0) { - // Probably never happens, but the JDK docs don't explicitly say so - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName()); - } - // The order of the returned address is unspecified. Sort by IP to make sure we get a true - // round-robin - Arrays.sort(aRecords, IP_COMPARATOR); - int index = - (aRecords.length == 1) - ? 0 - : OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) % aRecords.length; - return new InetSocketAddress(aRecords[index], proxyAddress.getPort()); - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName(), e); - } - } - - /** - * Returns all socket addresses for this SNI proxy endpoint. - * - *

Re-resolves the proxy hostname on each call and returns one {@link InetSocketAddress} per - * A-record, so that the driver can try every proxy IP in sequence if one is unreachable. - * - *

All A-records are returned so a single connection attempt can fall back across every proxy - * IP, but the candidate order is rotated on each call using a dedicated round-robin counter - * ({@link #RESOLVE_ALL_OFFSET}). This spreads healthy connections across proxy IPs instead of - * always starting at the first one, preserving the previous load-balancing behavior. The counter - * is intentionally separate from the one used by {@link #resolve()} so that SSL engine setup - * (which calls {@code resolve()} once per connection) does not perturb this rotation. + *

Unpinned, this is the configured proxy address as-is. For Cloud that is a hostname (see + * {@code CloudConfigFactory}), which {@link + * com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands to every proxy A-record, + * trying each in turn — so a single unreachable proxy IP no longer fails the connection. + * Re-resolving here instead would block whichever event loop called us, and would bypass a custom + * Netty resolver. * - *

The blocking DNS lookup is run on {@code executor} so it never blocks the calling (event - * loop) thread; the returned stage completes with the rotated candidate array, or completes - * exceptionally with an {@link IllegalArgumentException} if the proxy hostname cannot be - * resolved. - * - *

Once {@linkplain #pinTo(SocketAddress) pinned} this returns the pinned address only, without - * any lookup or rotation: the endpoint then denotes one specific proxy IP. + *

Once {@linkplain #pinTo(SocketAddress) pinned} this returns that one proxy IP. That is what + * {@link com.datastax.oss.driver.internal.core.ssl.SniSslEngineFactory#newSslEngine} sees: it + * runs inside Netty's channel initializer, so it gets the exact IP the channel is connected to + * without a lookup on the event loop. */ @NonNull @Override - public CompletionStage resolveAll(@NonNull Executor executor) { - if (pinnedAddress != null) { - return CompletableFuture.completedFuture(new SocketAddress[] {pinnedAddress}); - } - return CompletableFuture.supplyAsync( - () -> { - try { - InetAddress[] aRecords = InetAddress.getAllByName(proxyAddress.getHostName()); - if (aRecords.length == 0) { - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName()); - } - // The order of the returned addresses is unspecified. Sort by IP so the round-robin - // rotation below is deterministic across calls. - Arrays.sort(aRecords, IP_COMPARATOR); - int start = - (aRecords.length == 1) - ? 0 - : RESOLVE_ALL_OFFSET.getAndUpdate(x -> x == Integer.MAX_VALUE ? 0 : x + 1) - % aRecords.length; - SocketAddress[] result = new SocketAddress[aRecords.length]; - for (int i = 0; i < aRecords.length; i++) { - InetAddress aRecord = aRecords[(start + i) % aRecords.length]; - result[i] = new InetSocketAddress(aRecord, proxyAddress.getPort()); - } - return result; - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Could not resolve proxy address " + proxyAddress.getHostName(), e); - } - }, - executor); + public InetSocketAddress resolve() { + return pinnedAddress != null ? pinnedAddress : proxyAddress; } @NonNull @@ -220,10 +130,4 @@ public String asMetricPrefix() { } return hostString.replace('.', '_') + ':' + proxyAddress.getPort() + '_' + serverName; } - - @SuppressWarnings("UnnecessaryLambda") - private static final Comparator IP_COMPARATOR = - (InetAddress address1, InetAddress address2) -> - UnsignedBytes.lexicographicalComparator() - .compare(address1.getAddress(), address2.getAddress()); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java index cf00fd39b32..9be8c4c94bd 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/TopologyMonitor.java @@ -156,10 +156,14 @@ default void resetColumnCaches() {} *

The default implementation returns {@code false}, which is correct for {@link * DefaultTopologyMonitor}: the peer nodes it registers hold a {@code DefaultEndPoint} built from * an already-resolved physical IP (from {@code system.peers}), which never needs re-resolving. - * This is separate from the connected node's own {@code EndPoint}, which may originate from an - * unresolved contact-point hostname -- that one does re-resolve via {@code EndPoint.resolveAll()} - * on every connection attempt, independently of this flag. Proxy-based monitors that re-resolve - * per call should override this to return {@code true}. + * + *

The connected node's own {@code EndPoint} is a different case again. It originates from the + * contact point the control connection used, and {@code ChannelFactory} binds it to the single + * address that connection reached (see {@code PinnableEndPoint}), so it does not re-expand + * on later connection attempts. Recovering from an address change for that node therefore depends + * on this flag being {@code false}, i.e. on the contact-point fallback described above. + * + *

Proxy-based monitors that re-resolve per call should override this to return {@code true}. */ default boolean reresolvesNodeAddresses() { return false; diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java index b2ed5111077..c9fee86f2c1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java @@ -630,14 +630,6 @@ private void onChildrenClosed(List> childrenCloseStages) { for (CompletionStage stage : childrenCloseStages) { warnIfFailed(stage); } - // The channel factory owns a DNS-resolver executor that is not an AsyncAutoCloseable child; - // shut it down here, after all pools/control-connection that used it are closed. Guarded like - // the other context-component accesses below (the factory may have failed to initialize). - try { - context.getChannelFactory().close(); - } catch (Throwable t) { - // ignore: the factory may have failed to initialize, nothing to close - } context .getNettyOptions() .onClose() diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java index c64a334b1fe..343d3f9e4e7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java @@ -116,8 +116,6 @@ protected String hostNoLookup(InetSocketAddress addr) { @NonNull @Override - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); SSL - // factories legitimately need a single address for hostname verification. public SSLEngine newSslEngine(@NonNull EndPoint remoteEndpoint) { SSLEngine engine; SocketAddress remoteAddress = remoteEndpoint.resolve(); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java index 424120a99f3..4d2cb69fbfc 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/SniSslEngineFactory.java @@ -51,8 +51,6 @@ public SniSslEngineFactory(SSLContext sslContext, boolean allowDnsReverseLookupS @NonNull @Override - @SuppressWarnings("deprecation") // resolve() is deprecated in favour of resolveAll(); SSL - // factories legitimately need a single address for SNI hostname verification. public SSLEngine newSslEngine(@NonNull EndPoint remoteEndpoint) { if (!(remoteEndpoint instanceof SniEndPoint)) { throw new IllegalArgumentException( diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java deleted file mode 100644 index d604e4709fb..00000000000 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryAsyncResolveTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.oss.driver.internal.core.channel; - -import static com.datastax.oss.driver.Assertions.assertThat; -import static com.datastax.oss.driver.Assertions.assertThatStage; -import static org.mockito.Mockito.when; - -import com.datastax.oss.driver.api.core.DefaultProtocolVersion; -import com.datastax.oss.driver.api.core.config.DefaultDriverOption; -import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; -import edu.umd.cs.findbugs.annotations.NonNull; -import java.net.SocketAddress; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import org.junit.Test; - -/** - * Verifies that {@link ChannelFactory#connect} resolves endpoint addresses asynchronously, off the - * calling (admin event loop) thread, so DNS never blocks the caller. - */ -public class ChannelFactoryAsyncResolveTest extends ChannelFactoryTestBase { - - @Test - public void should_resolve_on_dedicated_resolver_thread() throws Exception { - // Given - when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); - when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - ChannelFactory factory = newChannelFactory(); - - CompletableFuture resolverThreadName = new CompletableFuture<>(); - EndPoint endPoint = - new EndPoint() { - @NonNull - @Override - public SocketAddress resolve() { - return SERVER_ADDRESS.resolve(); - } - - @NonNull - @Override - public CompletionStage resolveAll(@NonNull Executor executor) { - return CompletableFuture.supplyAsync( - () -> { - resolverThreadName.complete(Thread.currentThread().getName()); - return new SocketAddress[] {SERVER_ADDRESS.resolve()}; - }, - executor); - } - - @NonNull - @Override - public String asMetricPrefix() { - return "test"; - } - }; - - String callerThreadName = Thread.currentThread().getName(); - - // When - CompletionStage channelFuture = - factory.connect( - endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - completeSimpleChannelInit(); - - // Then – resolution ran on the factory's dedicated resolver executor, not the caller thread - String actualThreadName = resolverThreadName.get(2, TimeUnit.SECONDS); - assertThat(actualThreadName).isNotEqualTo(callerThreadName); - assertThat(actualThreadName).contains("-connection-resolver-"); - assertThatStage(channelFuture).isSuccess(); - } - - @Test - public void should_not_block_caller_while_resolving() throws Exception { - // Given - when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); - when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - ChannelFactory factory = newChannelFactory(); - - CountDownLatch resolutionEntered = new CountDownLatch(1); - CountDownLatch releaseResolution = new CountDownLatch(1); - EndPoint endPoint = - new EndPoint() { - @NonNull - @Override - public SocketAddress resolve() { - return SERVER_ADDRESS.resolve(); - } - - @NonNull - @Override - public CompletionStage resolveAll(@NonNull Executor executor) { - return CompletableFuture.supplyAsync( - () -> { - resolutionEntered.countDown(); - try { - releaseResolution.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return new SocketAddress[] {SERVER_ADDRESS.resolve()}; - }, - executor); - } - - @NonNull - @Override - public String asMetricPrefix() { - return "test"; - } - }; - - // When - CompletionStage channelFuture = - factory.connect( - endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - - // Then – connect() returned control while resolution is still blocked (caller not blocked) - assertThat(resolutionEntered.await(2, TimeUnit.SECONDS)).isTrue(); - assertThat(channelFuture.toCompletableFuture().isDone()).isFalse(); - - // Once resolution is unblocked the connection proceeds to completion. - releaseResolution.countDown(); - completeSimpleChannelInit(); - assertThatStage(channelFuture).isSuccess(); - } -} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 5e56c7c07c9..a008fb95434 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -19,99 +19,150 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.DefaultProtocolVersion; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.bootstrap.Bootstrap; import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolverGroup; +import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.concurrent.CompletableFuture; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; import org.junit.Test; /** - * Verifies that {@link ChannelFactory#connect} tries every candidate returned by {@link - * EndPoint#resolveAll(java.util.concurrent.Executor)} in sequence: it falls back to the next - * address when one is unreachable, and only fails the overall future once all candidates are - * exhausted, carrying the earlier failures as suppressed exceptions. + * Verifies how {@link ChannelFactory#connect} treats the several addresses a name expands to: every + * one of them is tried in sequence, failures are aggregated rather than dropped, and the starting + * address is rotated so healthy connections do not all pile onto the same one. + * + *

The expansion itself is exercised in {@link ChannelFactoryNettyResolverTest}; here the + * resolver is only the mechanism for producing more than one address from a single endpoint. */ public class ChannelFactoryMultiAddressTest extends ChannelFactoryTestBase { - // A local address that no server is bound to: connecting to it fails immediately. + // Local addresses that no server is bound to: connecting to them fails immediately. private static final SocketAddress UNREACHABLE_1 = new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-1"); private static final SocketAddress UNREACHABLE_2 = new LocalAddress(ChannelFactoryMultiAddressTest.class.getSimpleName() + "-unreachable-2"); + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + @Test - public void should_fall_back_to_next_candidate_when_first_is_unreachable() { - // Given + public void should_fail_with_suppressed_causes_when_all_addresses_are_unreachable() { + // Given – a name that expands to two dead addresses. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE_1, UNREACHABLE_2))); ChannelFactory factory = newChannelFactory(); - // First candidate is unreachable, second is the running local server. - EndPoint endPoint = endPointReturning(UNREACHABLE_1, SERVER_ADDRESS.resolve()); - // When CompletionStage channelFuture = factory.connect( - endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - // The handshake only happens once we fall back to the reachable second candidate. - completeSimpleChannelInit(); + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); - // Then – the connection succeeds via the second candidate. - assertThatStage(channelFuture).isSuccess(); + // Then – the future fails, and the earlier address's failure is preserved as a suppressed + // exception on the last one's error rather than being silently dropped. + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e.getSuppressed()) + .as("earlier address failures should be attached as suppressed exceptions") + .isNotEmpty()); } @Test - public void should_fail_with_suppressed_causes_when_all_candidates_are_unreachable() { - // Given + public void should_rotate_the_starting_address_across_successive_expansions() { + // Every address must still be offered -- a single attempt has to be able to fall back across + // all + // of them -- but the one tried *first* has to move, otherwise every connection piles onto the + // resolver's first record and a multi-record name buys no spreading at all. + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2); + + List first = ChannelFactory.rotate(addresses); + List second = ChannelFactory.rotate(addresses); + + assertThat(first).containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2); + assertThat(second).containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2); + assertThat(second.get(0)) + .as("successive expansions must not start at the same address") + .isNotEqualTo(first.get(0)); + } + + @Test + public void should_leave_a_single_address_alone() { + // Nothing to spread, and the rotation offset must not advance for it either. + assertThat(ChannelFactory.rotate(Collections.singletonList(UNREACHABLE_1))) + .containsExactly(UNREACHABLE_1); + } + + @Test + public void should_fail_future_when_endpoint_resolve_throws() { + // ChannelFactory calls EndPoint.resolve() directly on the caller thread, so a third-party + // implementation that throws must surface as a failed future rather than an escaping exception. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); ChannelFactory factory = newChannelFactory(); + IllegalStateException failure = new IllegalStateException("resolve() blew up"); - EndPoint endPoint = endPointReturning(UNREACHABLE_1, UNREACHABLE_2); - - // When CompletionStage channelFuture = factory.connect( - endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + new ThrowingEndPoint(failure), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); - // Then – the future fails, and the earlier candidate's failure is preserved as a suppressed - // exception on the last candidate's error (rather than being silently dropped). - assertThatStage(channelFuture) - .isFailed( - e -> - assertThat(e.getSuppressed()) - .as("earlier candidate failures should be attached as suppressed exceptions") - .isNotEmpty()); + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ + private void installResolver(AddressResolverGroup group) { + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(group); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); } - /** An endpoint whose {@link EndPoint#resolveAll} yields the given candidates, in order. */ - private static EndPoint endPointReturning(SocketAddress... candidates) { - return new EndPoint() { - @NonNull - @Override - public SocketAddress resolve() { - return candidates[candidates.length - 1]; - } - - @NonNull - @Override - public CompletionStage resolveAll(@NonNull Executor executor) { - return CompletableFuture.completedFuture(candidates.clone()); - } - - @NonNull - @Override - public String asMetricPrefix() { - return "test"; - } - }; + /** An endpoint whose {@link EndPoint#resolve()} throws, standing in for a broken third party. */ + private static class ThrowingEndPoint implements EndPoint { + + private final RuntimeException failure; + + ThrowingEndPoint(RuntimeException failure) { + this.failure = failure; + } + + @NonNull + @Override + public SocketAddress resolve() { + throw failure; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java index be85a9b0a68..45971bc1560 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -29,18 +29,12 @@ import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import io.netty.bootstrap.Bootstrap; import io.netty.channel.local.LocalAddress; -import io.netty.resolver.AddressResolver; import io.netty.resolver.AddressResolverGroup; -import io.netty.util.concurrent.EventExecutor; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.Promise; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Arrays; import java.util.Collections; -import java.util.List; import java.util.concurrent.CompletionStage; -import java.util.concurrent.CopyOnWriteArrayList; import org.junit.Test; /** @@ -69,8 +63,8 @@ public void should_expand_unresolved_address_through_the_custom_netty_resolver() // local server, mimicking a DNS round-robin entry whose first record is dead. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - TestResolverGroup resolverGroup = - new TestResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve())); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, SERVER_ADDRESS.resolve())); installResolver(resolverGroup); ChannelFactory factory = newChannelFactory(); @@ -99,7 +93,7 @@ public void should_fail_when_the_custom_resolver_cannot_resolve_the_only_candida // Given – a resolver that fails every lookup. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - TestResolverGroup resolverGroup = new TestResolverGroup(null); + TestAddressResolverGroup resolverGroup = new TestAddressResolverGroup(null); installResolver(resolverGroup); ChannelFactory factory = newChannelFactory(); @@ -124,7 +118,8 @@ public void should_not_resolve_at_all_when_the_user_disabled_the_resolver() { // treat that as "pass the candidates through" instead of dereferencing the missing group. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - TestResolverGroup resolverGroup = new TestResolverGroup(Collections.singletonList(UNREACHABLE)); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); doAnswer( invocation -> { Bootstrap bootstrap = invocation.getArgument(0); @@ -152,14 +147,16 @@ public void should_not_resolve_at_all_when_the_user_disabled_the_resolver() { } @Test - public void should_pass_already_resolved_candidates_through_untouched() { - // Given – an endpoint that returns addresses the resolver reports as already resolved. Mirrors - // Bootstrap#doResolveAndConnect0, which skips resolution in that case. Metadata nodes are in - // this - // situation: their endpoints hold resolved addresses from the peers rows. + public void should_pass_already_resolved_address_through_untouched() { + // Given – an endpoint whose address is already resolved, which is the common case: metadata + // nodes + // hold resolved addresses from the peers rows, so this is every pool refill and every + // reconnect. + // Mirrors Bootstrap#doResolveAndConnect0, which skips resolution in that case. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - TestResolverGroup resolverGroup = new TestResolverGroup(Collections.singletonList(UNREACHABLE)); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(UNREACHABLE)); installResolver(resolverGroup); ChannelFactory factory = newChannelFactory(); @@ -190,80 +187,4 @@ private void installResolver(AddressResolverGroup group) { .when(nettyOptions) .afterBootstrapInitialized(any(Bootstrap.class)); } - - /** - * A stand-in for a user-supplied {@code AddressResolverGroup} (e.g. Netty's {@code - * DnsAddressResolverGroup}). Records what it was asked to resolve, and answers with a fixed list - * of addresses so the test can assert that every one of them is tried. - * - *

Implements {@link AddressResolver} directly rather than extending {@code - * AbstractAddressResolver} so it can hand back {@link LocalAddress}es — the unit tests connect - * over Netty's local transport, which is not reachable through an {@link InetSocketAddress}. - */ - private static class TestResolverGroup extends AddressResolverGroup { - - final List queried = new CopyOnWriteArrayList<>(); - volatile boolean resolverRequested; - - /** The addresses to answer with, or {@code null} to fail every lookup. */ - private final List answer; - - TestResolverGroup(List answer) { - this.answer = answer; - } - - @Override - protected AddressResolver newResolver(EventExecutor executor) { - resolverRequested = true; - return new AddressResolver() { - - @Override - public boolean isSupported(SocketAddress address) { - return true; - } - - @Override - public boolean isResolved(SocketAddress address) { - // Only hostnames need resolving; anything else (including the local-transport addresses - // we - // hand back) is already usable. - return !(address instanceof InetSocketAddress) - || !((InetSocketAddress) address).isUnresolved(); - } - - @Override - public Future resolve(SocketAddress address) { - return resolve(address, executor.newPromise()); - } - - @Override - public Future resolve( - SocketAddress address, Promise promise) { - queried.add(address); - return answer == null - ? promise.setFailure(new IllegalStateException("mock resolver failure")) - : promise.setSuccess(answer.get(0)); - } - - @Override - public Future> resolveAll(SocketAddress address) { - return resolveAll(address, executor.newPromise()); - } - - @Override - public Future> resolveAll( - SocketAddress address, Promise> promise) { - queried.add(address); - return answer == null - ? promise.setFailure(new IllegalStateException("mock resolver failure")) - : promise.setSuccess(answer); - } - - @Override - public void close() { - // nothing to do - } - }; - } - } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java index 5f11fbc1594..7f6d2d57b5c 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java @@ -19,6 +19,8 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.DefaultProtocolVersion; @@ -27,14 +29,13 @@ import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import edu.umd.cs.findbugs.annotations.NonNull; +import io.netty.bootstrap.Bootstrap; import io.netty.channel.local.LocalAddress; +import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Arrays; -import java.util.List; import java.util.Objects; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; import org.junit.Test; /** @@ -53,20 +54,34 @@ public class ChannelFactoryPinnedEndPointTest extends ChannelFactoryTestBase { private static final SocketAddress UNREACHABLE = new LocalAddress(ChannelFactoryPinnedEndPointTest.class.getSimpleName() + "-unreachable"); + /** The name the endpoint reports, and that only the resolver knows how to expand. */ + private static final InetSocketAddress HOSTNAME = + InetSocketAddress.createUnresolved("test.cluster.fake", 9042); + @Test public void should_pin_channel_endpoint_to_the_address_that_connected() { - // Given – two candidates; only the second one has a server listening. + // Given – an endpoint reporting a name, which the resolver expands to a dead address and the + // running local server. Whichever of the two the connection ends up on, the channel must carry + // that one. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - ChannelFactory factory = newChannelFactory(); SocketAddress reachable = SERVER_ADDRESS.resolve(); - TestPinnableEndPoint endPoint = new TestPinnableEndPoint(UNREACHABLE, reachable); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver( + new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, reachable))); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + TestPinnableEndPoint endPoint = new TestPinnableEndPoint(HOSTNAME); // When CompletionStage channelFuture = factory.connect( endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - // The handshake only happens once we fall back to the reachable second candidate. completeSimpleChannelInit(); // Then @@ -74,10 +89,10 @@ public void should_pin_channel_endpoint_to_the_address_that_connected() { .isSuccess( channel -> { EndPoint channelEndPoint = channel.getEndPoint(); - // The channel resolves to the address it is actually connected to... + // The channel resolves to the address it is actually connected to -- the name it was + // built from is gone from resolve(), which is what SSL engines and authenticators + // need. assertThat(channelEndPoint.resolve()).isEqualTo(reachable); - assertThat(channelEndPoint.resolveAll(Runnable::run).toCompletableFuture().join()) - .containsExactly(reachable); // ...while still denoting the same node, so node lookups and metric names are stable. assertThat(channelEndPoint).isEqualTo(endPoint); assertThat(channelEndPoint.asMetricPrefix()).isEqualTo(endPoint.asMetricPrefix()); @@ -107,43 +122,35 @@ public void should_leave_non_pinnable_endpoints_untouched() { } /** - * A {@link PinnableEndPoint} over Netty's local transport, so the test can exercise pinning - * against the real embedded server. Identity is the candidate list, so a pinned copy stays equal - * to the original — the contract {@link PinnableEndPoint} requires. + * A {@link PinnableEndPoint} that can hold a pin to any {@link SocketAddress}, including the + * local-transport addresses these tests connect over (which {@code DefaultEndPoint} cannot). + * Identity is the unpinned address, so a pinned copy stays equal to the original — the contract + * {@link PinnableEndPoint} requires. */ private static class TestPinnableEndPoint implements PinnableEndPoint { - private final List candidates; + private final SocketAddress address; private final SocketAddress pinnedAddress; - TestPinnableEndPoint(SocketAddress... candidates) { - this(Arrays.asList(candidates), null); + TestPinnableEndPoint(SocketAddress address) { + this(address, null); } - private TestPinnableEndPoint(List candidates, SocketAddress pinnedAddress) { - this.candidates = candidates; + private TestPinnableEndPoint(SocketAddress address, SocketAddress pinnedAddress) { + this.address = address; this.pinnedAddress = pinnedAddress; } @NonNull @Override public SocketAddress resolve() { - return pinnedAddress != null ? pinnedAddress : candidates.get(0); - } - - @NonNull - @Override - public CompletionStage resolveAll(@NonNull Executor executor) { - return CompletableFuture.completedFuture( - pinnedAddress != null - ? new SocketAddress[] {pinnedAddress} - : candidates.toArray(new SocketAddress[0])); + return pinnedAddress != null ? pinnedAddress : address; } @NonNull @Override public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { - return new TestPinnableEndPoint(candidates, resolvedAddress); + return new TestPinnableEndPoint(address, resolvedAddress); } @NonNull @@ -155,12 +162,12 @@ public String asMetricPrefix() { @Override public boolean equals(Object other) { return (other instanceof TestPinnableEndPoint) - && candidates.equals(((TestPinnableEndPoint) other).candidates); + && address.equals(((TestPinnableEndPoint) other).address); } @Override public int hashCode() { - return Objects.hash(candidates); + return Objects.hash(address); } } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java deleted file mode 100644 index 99fbd156f7a..00000000000 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryResolveAllGuardTest.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.oss.driver.internal.core.channel; - -import static com.datastax.oss.driver.Assertions.assertThatStage; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.datastax.oss.driver.api.core.DefaultProtocolVersion; -import com.datastax.oss.driver.api.core.config.DefaultDriverOption; -import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; -import java.net.SocketAddress; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.CompletionStage; -import org.junit.Test; - -/** - * Verifies that {@link ChannelFactory#connect} completes the result future exceptionally (rather - * than throwing or hanging) when {@link EndPoint#resolveAll(java.util.concurrent.Executor)} - * completes with {@code null}, an empty array, throws synchronously, or completes exceptionally. - * Resolution is now asynchronous, so these assertions exercise the {@code whenComplete} callback. - */ -public class ChannelFactoryResolveAllGuardTest extends ChannelFactoryTestBase { - - @Test - public void should_fail_future_when_resolve_all_returns_null() { - // Given - when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); - when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - ChannelFactory factory = newChannelFactory(); - - EndPoint badEndPoint = mock(EndPoint.class); - when(badEndPoint.resolveAll(any())).thenReturn(CompletableFuture.completedFuture(null)); - - // When - CompletionStage channelFuture = - factory.connect( - badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - - // Then – future must complete exceptionally without hanging - assertThatStage(channelFuture) - .isFailed( - e -> - assertThat(e) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("resolveAll() must return a non-null, non-empty array")); - } - - @Test - public void should_fail_future_when_resolve_all_returns_empty_array() { - // Given - when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); - when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - ChannelFactory factory = newChannelFactory(); - - EndPoint badEndPoint = mock(EndPoint.class); - when(badEndPoint.resolveAll(any())) - .thenReturn(CompletableFuture.completedFuture(new SocketAddress[0])); - - // When - CompletionStage channelFuture = - factory.connect( - badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - - // Then – future must complete exceptionally without hanging - assertThatStage(channelFuture) - .isFailed( - e -> - assertThat(e) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("resolveAll() must return a non-null, non-empty array")); - } - - @Test - public void should_fail_future_when_resolve_all_throws_synchronously() { - // Given - when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); - when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - ChannelFactory factory = newChannelFactory(); - - EndPoint badEndPoint = mock(EndPoint.class); - RuntimeException testException = new RuntimeException("DNS lookup failed"); - when(badEndPoint.resolveAll(any())).thenThrow(testException); - - // When - CompletionStage channelFuture = - factory.connect( - badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - - // Then – future must complete exceptionally with the thrown exception - assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(testException)); - } - - @Test - public void should_fail_future_with_unwrapped_cause_when_resolve_all_stage_fails() { - // Given - when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); - when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - ChannelFactory factory = newChannelFactory(); - - EndPoint badEndPoint = mock(EndPoint.class); - RuntimeException testException = new RuntimeException("DNS lookup failed"); - // A real impl offloads via supplyAsync, which wraps the supplier's throwable in a - // CompletionException; connect() must unwrap it so callers see the original cause. - CompletableFuture failedStage = new CompletableFuture<>(); - failedStage.completeExceptionally(new CompletionException(testException)); - when(badEndPoint.resolveAll(any())).thenReturn(failedStage); - - // When - CompletionStage channelFuture = - factory.connect( - badEndPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); - - // Then – future must complete exceptionally with the unwrapped original exception - assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(testException)); - } -} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java index e9c8f836de4..c90731eece9 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/LocalEndPoint.java @@ -21,9 +21,6 @@ import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.channel.local.LocalAddress; import java.net.SocketAddress; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; /** Endpoint implementation for unit tests that use the local Netty transport. */ public class LocalEndPoint implements EndPoint { @@ -40,13 +37,6 @@ public SocketAddress resolve() { return localAddress; } - @NonNull - @Override - public CompletionStage resolveAll(@NonNull Executor executor) { - // In-memory local address; resolve synchronously without hopping to the executor. - return CompletableFuture.completedFuture(new SocketAddress[] {localAddress}); - } - @NonNull @Override public String asMetricPrefix() { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java new file mode 100644 index 00000000000..abf69508a85 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import edu.umd.cs.findbugs.annotations.Nullable; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * A stand-in for a user-supplied {@code AddressResolverGroup} (e.g. Netty's {@code + * DnsAddressResolverGroup}), installed the way a user would install one: through {@link + * com.datastax.oss.driver.internal.core.context.NettyOptions#afterBootstrapInitialized(Bootstrap)}. + * + *

Records what it was asked to resolve, and answers with a fixed list of addresses so tests can + * assert that every one of them is tried, and in what order. + * + *

Implements {@link AddressResolver} directly rather than extending {@code + * AbstractAddressResolver} so it can hand back {@link LocalAddress}es — the unit tests connect over + * Netty's local transport, which is not reachable through an {@link InetSocketAddress}. + */ +class TestAddressResolverGroup extends AddressResolverGroup { + + /** Every address this group was asked to resolve, in order. */ + final List queried = new CopyOnWriteArrayList<>(); + + /** Whether a resolver was ever obtained from this group at all. */ + volatile boolean resolverRequested; + + /** The addresses to answer with, or {@code null} to fail every lookup. */ + @Nullable private final List answer; + + TestAddressResolverGroup(@Nullable List answer) { + this.answer = answer; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + resolverRequested = true; + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + return true; + } + + @Override + public boolean isResolved(SocketAddress address) { + // Only hostnames need resolving; anything else (including the local-transport addresses we + // hand back) is already usable. + return !(address instanceof InetSocketAddress) + || !((InetSocketAddress) address).isUnresolved(); + } + + @Override + public Future resolve(SocketAddress address) { + return resolve(address, executor.newPromise()); + } + + @Override + public Future resolve(SocketAddress address, Promise promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer.get(0)); + } + + @Override + public Future> resolveAll(SocketAddress address) { + return resolveAll(address, executor.newPromise()); + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + queried.add(address); + return answer == null + ? promise.setFailure(new IllegalStateException("mock resolver failure")) + : promise.setSuccess(answer); + } + + @Override + public void close() { + // nothing to do + } + }; + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java index 479730606f3..a6374d73a36 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java @@ -18,24 +18,15 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.SocketAddress; import java.net.UnknownHostException; -import java.util.List; import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -75,20 +66,23 @@ public void should_fallback_when_resolve_returns_null() throws UnknownHostExcept } @Test - public void should_wrap_io_exceptions_in_unchecked_io_exception() throws UnknownHostException { + public void should_return_the_route_address_unresolved() { + // The route hostname is handed over unresolved on purpose: ChannelFactory resolves it through + // Netty's AddressResolverGroup, so a custom resolver applies to client routes too and no DNS + // lookup runs on the admin event loop that connect() is called from. UUID hostId = UUID.randomUUID(); - when(topologyMonitor.resolve(hostId)).thenThrow(new UnknownHostException("no-such-host")); + InetSocketAddress route = InetSocketAddress.createUnresolved("route.example.com", 9042); + when(topologyMonitor.resolve(hostId)).thenReturn(route); ClientRoutesEndPoint ep = new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); - assertThatThrownBy(ep::resolve) - .isInstanceOf(UncheckedIOException.class) - .hasCauseInstanceOf(UnknownHostException.class); + assertThat(ep.resolve()).isSameAs(route); + assertThat(((InetSocketAddress) ep.resolve()).isUnresolved()).isTrue(); } @Test - public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownHostException { + public void should_reflect_route_changes_on_subsequent_resolve() { UUID hostId = UUID.randomUUID(); InetSocketAddress addr1 = new InetSocketAddress("127.0.0.1", 9042); InetSocketAddress addr2 = new InetSocketAddress("10.0.0.1", 9043); @@ -105,77 +99,10 @@ public void should_reflect_route_changes_on_subsequent_resolve() throws UnknownH assertThat(ep.resolve()).isEqualTo(addr2); } - // ---- resolveAll() ------------------------------------------------------- - - @Test - public void should_resolve_all_on_the_supplied_executor() throws UnknownHostException { - // topologyMonitor.resolve() can reach InetAddress.getByName(), which blocks, so it must not run - // on the caller -- ChannelFactory calls this from the admin event loop. - UUID hostId = UUID.randomUUID(); - InetSocketAddress expected = new InetSocketAddress("127.0.0.1", 9042); - when(topologyMonitor.resolve(hostId)).thenReturn(expected); - - ClientRoutesEndPoint ep = - new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); - - List resolutionThreads = new CopyOnWriteArrayList<>(); - ExecutorService executor = - Executors.newSingleThreadExecutor(r -> new Thread(r, "resolver-under-test")); - try { - when(topologyMonitor.resolve(hostId)) - .thenAnswer( - invocation -> { - resolutionThreads.add(Thread.currentThread().getName()); - return expected; - }); - - assertThat(ep.resolveAll(executor).toCompletableFuture().join()).containsExactly(expected); - } finally { - executor.shutdownNow(); - } - - assertThat(resolutionThreads).containsExactly("resolver-under-test"); - } - - @Test - public void should_delegate_resolve_all_to_fallback_endpoint_when_there_is_no_route() - throws UnknownHostException { - // No route for this host id: the fallback must be asked through its own resolveAll(), so it - // keeps - // whatever resolution semantics it defines, rather than being flattened to resolve(). - UUID hostId = UUID.randomUUID(); - InetSocketAddress fallback1 = new InetSocketAddress("10.0.0.1", 9042); - InetSocketAddress fallback2 = new InetSocketAddress("10.0.0.2", 9042); - when(topologyMonitor.resolve(hostId)).thenReturn(null); - when(fallbackEndPoint.resolveAll(any())) - .thenReturn(CompletableFuture.completedFuture(new SocketAddress[] {fallback1, fallback2})); - - ClientRoutesEndPoint ep = - new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); - - assertThat(ep.resolveAll(Runnable::run).toCompletableFuture().join()) - .containsExactly(fallback1, fallback2); - verify(fallbackEndPoint).resolveAll(any()); - verify(fallbackEndPoint, never()).resolve(); - } - - @Test - public void should_fail_resolve_all_stage_when_topology_monitor_throws() - throws UnknownHostException { - UUID hostId = UUID.randomUUID(); - when(topologyMonitor.resolve(hostId)).thenThrow(new UnknownHostException("no-such-host")); - - ClientRoutesEndPoint ep = - new ClientRoutesEndPoint(topologyMonitor, hostId, null, fallbackEndPoint); - - assertThatThrownBy(() -> ep.resolveAll(Runnable::run).toCompletableFuture().join()) - .hasCauseInstanceOf(UncheckedIOException.class); - } - // ---- pinTo() ------------------------------------------------------------ @Test - public void pin_to_should_stop_consulting_the_topology_monitor() throws UnknownHostException { + public void pin_to_should_stop_consulting_the_topology_monitor() { UUID hostId = UUID.randomUUID(); InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); @@ -184,8 +111,6 @@ public void pin_to_should_stop_consulting_the_topology_monitor() throws UnknownH EndPoint pinned = original.pinTo(pinnedTo); assertThat(pinned.resolve()).isEqualTo(pinnedTo); - assertThat(pinned.resolveAll(Runnable::run).toCompletableFuture().join()) - .containsExactly(pinnedTo); // No lookup at all -- that is the point: DefaultTopologyMonitor#savePort and the SSL factories // read the channel's endpoint, and must not trigger a blocking re-resolution there. verify(topologyMonitor, never()).resolve(hostId); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java index 0a6268f3378..e6ab0895034 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java @@ -198,14 +198,21 @@ public void should_throw_after_close() { } @Test - public void should_throw_for_unresolvable_hostname() { + public void should_not_look_up_the_route_hostname() { UUID hostId = UUID.randomUUID(); - // Use a hostname guaranteed not to resolve + // A hostname guaranteed not to resolve: this must still succeed, because resolve() is a pure + // in-memory cache lookup that hands the name over unresolved. ChannelFactory resolves it later + // through Netty's AddressResolverGroup, so a custom resolver applies to client routes too and + // nothing blocks the admin event loop here. handler.setRoutes( ImmutableMap.of( hostId, new ClientRouteRecord(hostId, "this.host.does.not.exist.invalid", 9042))); - assertThatThrownBy(() -> handler.resolve(hostId)).isInstanceOf(UnknownHostException.class); + InetSocketAddress result = handler.resolve(hostId); + + assertThat(result.isUnresolved()).isTrue(); + assertThat(result.getHostString()).isEqualTo("this.host.does.not.exist.invalid"); + assertThat(result.getPort()).isEqualTo(9042); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index a4b821b8d21..e716bf19371 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -23,9 +23,6 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import io.netty.channel.local.LocalAddress; import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; public class DefaultEndPointTest { @@ -64,65 +61,40 @@ public void should_reject_null_address() { } @Test - public void resolve_all_returns_single_element_for_already_resolved_address() { + public void resolve_returns_already_resolved_address_as_is() { DefaultEndPoint endPoint = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); - SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - assertThat(all).hasSize(1); - assertThat(((InetSocketAddress) all[0]).isUnresolved()).isFalse(); - assertThat(((InetSocketAddress) all[0]).getHostString()).isEqualTo("127.0.0.1"); + InetSocketAddress resolved = endPoint.resolve(); + assertThat(resolved.isUnresolved()).isFalse(); + assertThat(resolved.getHostString()).isEqualTo("127.0.0.1"); } @Test - public void resolve_all_passes_unresolved_hostname_through_without_looking_it_up() { - // This endpoint does NOT expand hostnames itself. It hands the unresolved address to + public void resolve_passes_unresolved_hostname_through_without_looking_it_up() { + // This endpoint does NOT resolve hostnames itself. It hands the unresolved address to // ChannelFactory, which expands it through Netty's AddressResolverGroup so that a custom // resolver installed via NettyOptions#afterBootstrapInitialized still applies -- a direct - // InetAddress.getAllByName() call here would bypass it. "localhost" would resolve fine, so the - // assertion below is only meaningful because we check the address comes back *unresolved*. + // InetAddress.getAllByName() call here would bypass it, and would block the admin event loop + // that connect() runs on. "localhost" would resolve fine, so this assertion is only meaningful + // because we check the address comes back *unresolved*. DefaultEndPoint endPoint = new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); - SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); + InetSocketAddress resolved = endPoint.resolve(); - assertThat(all).hasSize(1); - InetSocketAddress only = (InetSocketAddress) all[0]; - assertThat(only.isUnresolved()).isTrue(); - assertThat(only.getHostString()).isEqualTo("localhost"); - assertThat(only.getPort()).isEqualTo(9042); + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo("localhost"); + assertThat(resolved.getPort()).isEqualTo(9042); } @Test - public void resolve_all_does_not_throw_for_unresolvable_hostname() { + public void resolve_does_not_throw_for_unresolvable_hostname() { // No lookup happens, so an unresolvable name is not an error at this level: the connect attempt // fails later with a descriptive error instead. DefaultEndPoint endPoint = new DefaultEndPoint( InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042)); - SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - - assertThat(all).hasSize(1); - assertThat(((InetSocketAddress) all[0]).getHostString()) - .isEqualTo("this-host-does-not-exist.invalid"); - } - - @Test - public void resolve_all_never_uses_the_executor() { - // Resolution is a field read, so the stage must already be complete on return: ChannelFactory - // calls this from the admin event loop and an unnecessary hop would just add latency. - DefaultEndPoint endPoint = - new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); - - AtomicInteger executorCalls = new AtomicInteger(); - CompletionStage stage = - endPoint.resolveAll( - command -> { - executorCalls.incrementAndGet(); - command.run(); - }); - - assertThat(stage.toCompletableFuture()).isCompleted(); - assertThat(executorCalls).hasValue(0); + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); } @Test @@ -135,8 +107,6 @@ public void pin_to_should_override_resolution_but_preserve_identity() { // Resolution now yields exactly the pinned address... assertThat(pinned.resolve()).isEqualTo(pinnedTo); - assertThat(pinned.resolveAll(Runnable::run).toCompletableFuture().join()) - .containsExactly(pinnedTo); // ...but the copy still denotes the same node, and metric names must not change depending on // which IP a connection happened to land on. assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java index 6a53fe3e433..18a85bcfa3d 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java @@ -20,7 +20,9 @@ import static org.assertj.core.api.Assertions.assertThat; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.MockedDriverContextFactory; +import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; import java.net.InetSocketAddress; import java.util.UUID; import org.junit.Test; @@ -55,4 +57,44 @@ public void should_have_expected_string_representation_if_hostid_is_null() { "Node(endPoint=localhost/127.0.0.1:9042, hostId=null, hashCode=%x)", node.hashCode()); assertThat(node.toString()).isEqualTo(expected); } + + @Test + public void should_adopt_a_newer_endpoint_that_only_differs_by_its_pinned_address() { + // A PinnableEndPoint copy compares equal to the original -- pinnedAddress is excluded from + // equals() by contract, so that a pinned copy still denotes the same node. setEndPoint() must + // therefore not use equals() to decide whether to adopt it: the pinned address is the one every + // subsequent connection to this node will use, so refusing the newer instance would freeze the + // node on the first address it ever connected to, even after the control connection has moved + // and told us about it. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + DefaultNode node = new DefaultNode(endPoint, context); + + EndPoint pinnedToFirst = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)); + node.setEndPoint(pinnedToFirst, context); + assertThat(node.getEndPoint()).isSameAs(pinnedToFirst); + + EndPoint pinnedToSecond = + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.3", 9042)); + // Same node by equals(), different pinned address. + assertThat(pinnedToSecond).isEqualTo(pinnedToFirst); + node.setEndPoint(pinnedToSecond, context); + + assertThat(node.getEndPoint()).isSameAs(pinnedToSecond); + assertThat(node.getEndPoint().resolve()).isEqualTo(new InetSocketAddress("127.0.0.3", 9042)); + } + + @Test + public void should_not_rebuild_the_metric_updater_for_a_pin_only_change() { + // asMetricPrefix() is pin-independent, so rebuilding would clear and re-register metrics under + // identical names for no reason. + InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + DefaultNode node = new DefaultNode(endPoint, context); + NodeMetricUpdater before = node.getMetricUpdater(); + + node.setEndPoint( + ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)), context); + + assertThat(node.getMetricUpdater()).isSameAs(before); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java index 69109687d93..28da6e59ddd 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -18,136 +18,53 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; import org.junit.Test; public class SniEndPointTest { @Test - public void resolve_all_returns_all_proxy_addresses_for_resolvable_hostname() { - // localhost reliably resolves to at least one address - SniEndPoint endPoint = - new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); - SocketAddress[] all = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - assertThat(all).isNotEmpty(); - for (SocketAddress addr : all) { - InetSocketAddress inet = (InetSocketAddress) addr; - assertThat(inet.isUnresolved()).isFalse(); - assertThat(inet.getPort()).isEqualTo(9042); - } + public void resolve_returns_the_proxy_address_as_is_without_looking_it_up() { + // The proxy address is a hostname (that is how CloudConfigFactory builds it) and this endpoint + // must not resolve it: ChannelFactory expands it through Netty's AddressResolverGroup, which is + // what makes a custom resolver apply to the SNI proxy too, and what keeps resolve() safe to + // call + // from an event loop -- SniSslEngineFactory#newSslEngine does exactly that. + InetSocketAddress proxy = InetSocketAddress.createUnresolved("proxy.example.com", 9042); + SniEndPoint endPoint = new SniEndPoint(proxy, "test-server-name"); + + assertThat(endPoint.resolve()).isSameAs(proxy); + assertThat(endPoint.resolve().isUnresolved()).isTrue(); } @Test - public void resolve_all_fails_for_unresolvable_hostname() { + public void resolve_does_not_throw_for_unresolvable_proxy_hostname() { + // No lookup happens here, so an unresolvable name only fails later, at connect time. SniEndPoint endPoint = new SniEndPoint( - new InetSocketAddress("this-host-does-not-exist.invalid", 9042), "test-server-name"); - // Resolution is async now: the failure surfaces as an exceptionally-completed stage whose cause - // is the IllegalArgumentException. - assertThatThrownBy(() -> endPoint.resolveAll(Runnable::run).toCompletableFuture().join()) - .hasCauseInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Could not resolve proxy address"); - } - - @Test - public void resolve_returns_single_address_from_round_robin() { - // Sanity check: resolve() still works and returns a single address - SniEndPoint endPoint = - new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); - InetSocketAddress addr = endPoint.resolve(); - assertThat(addr.isUnresolved()).isFalse(); - assertThat(addr.getPort()).isEqualTo(9042); - } - - @Test - public void resolve_all_returns_complete_and_rotated_candidate_order() { - // resolveAll() must always return the full set of A-records (so a single connection attempt can - // fall back across every proxy IP), while rotating the starting element on each call to - // preserve the round-robin behavior of resolve(). We assert both invariants without depending - // on how many addresses "localhost" resolves to in a given environment. - SniEndPoint endPoint = - new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); - - SocketAddress[] first = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - int size = first.length; - assertThat(size).isGreaterThanOrEqualTo(1); - Set expected = new HashSet<>(Arrays.asList(first)); - - // Every call returns the same complete set of candidates, regardless of rotation. - boolean sawRotation = false; - SocketAddress firstElementSeed = first[0]; - for (int call = 0; call < size * 2; call++) { - SocketAddress[] candidates = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - assertThat(candidates).hasSize(size); - assertThat(new HashSet<>(Arrays.asList(candidates))).isEqualTo(expected); - if (!candidates[0].equals(firstElementSeed)) { - sawRotation = true; - } - } + InetSocketAddress.createUnresolved("this-host-does-not-exist.invalid", 9042), + "test-server-name"); - // When there is more than one A-record the starting candidate must rotate across calls. - if (size > 1) { - assertThat(sawRotation) - .as("resolveAll() should rotate the starting candidate when multiple IPs exist") - .isTrue(); - } + assertThat(endPoint.resolve().getHostString()).isEqualTo("this-host-does-not-exist.invalid"); } @Test - public void resolve_all_rotation_is_not_disturbed_by_interleaved_resolve() { - // resolveAll() rotates using a counter independent from resolve(). SSL engine setup calls the - // deprecated resolve() once per connection; if the two shared a counter, that extra advance - // would pin resolveAll()'s start index to 0 whenever the proxy resolves to an even number of - // A-records (DRIVER-201). Interleaving resolve() calls here must not stop resolveAll() from - // rotating. On single-address environments (size == 1) rotation is a no-op, matching the - // conditional assertion in resolve_all_returns_complete_and_rotated_candidate_order(). - SniEndPoint endPoint = - new SniEndPoint(new InetSocketAddress("localhost", 9042), "test-server-name"); - - SocketAddress[] first = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - int size = first.length; - SocketAddress firstStart = first[0]; - - boolean sawRotation = false; - for (int call = 0; call < size * 2; call++) { - SocketAddress[] candidates = endPoint.resolveAll(Runnable::run).toCompletableFuture().join(); - if (!candidates[0].equals(firstStart)) { - sawRotation = true; - } - // Interleave a resolve() call, as the SSL engine does on every connection. - endPoint.resolve(); - } - - if (size > 1) { - assertThat(sawRotation) - .as("resolveAll() rotation must survive interleaved resolve() calls") - .isTrue(); - } - } - - @Test - public void pin_to_should_make_resolve_a_field_read_and_preserve_identity() { - // This is what stops SniSslEngineFactory#newSslEngine -- which runs inside Netty's channel - // initializer, i.e. on an I/O event loop -- from doing a blocking DNS lookup, and guarantees it - // sees the very proxy IP the channel is connected to rather than another A-record. + public void pin_to_should_make_resolve_return_the_connected_proxy_ip_and_preserve_identity() { + // Pinning is what lets SniSslEngineFactory#newSslEngine -- which runs inside Netty's channel + // initializer -- see the very proxy IP the channel is connected to, rather than the hostname or + // another A-record. SniEndPoint original = new SniEndPoint( - new InetSocketAddress("this-host-does-not-exist.invalid", 9042), "test-server-name"); + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); - // An unresolvable hostname proves no lookup is attempted: the unpinned endpoint throws. assertThat(pinned.resolve()).isEqualTo(pinnedTo); - assertThat(pinned.resolveAll(Runnable::run).toCompletableFuture().join()) - .containsExactly(pinnedTo); - assertThatThrownBy(original::resolve).isInstanceOf(IllegalArgumentException.class); + assertThat(pinned.resolve().isUnresolved()).isFalse(); + // The original is untouched. + assertThat(original.resolve().isUnresolved()).isTrue(); // The pinned copy still denotes the same node. assertThat(pinned.getServerName()).isEqualTo(original.getServerName()); @@ -156,4 +73,16 @@ public void pin_to_should_make_resolve_a_field_read_and_preserve_identity() { assertThat(original).isEqualTo(pinned); assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); } + + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + SniEndPoint original = + new SniEndPoint( + InetSocketAddress.createUnresolved("proxy.example.com", 9042), "test-server-name"); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint pinned = (SniEndPoint) original.pinTo(pinnedTo); + + assertThat(pinned.pinTo(pinnedTo)).isSameAs(pinned); + } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java index 9a23f368546..66bf16a0390 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java @@ -186,12 +186,12 @@ private void requireSystemClientRoutesTable(CqlSession admin) { } private InetSocketAddress tryResolve(ClientRoutesTopologyMonitor handler, UUID hostId) { + // resolve() is an in-memory cache lookup that hands the route hostname over unresolved -- the + // connection layer resolves it -- so the only failure left is the monitor being closed. try { return handler.resolve(hostId); } catch (IllegalStateException e) { return null; - } catch (UnknownHostException e) { - throw new RuntimeException("DNS resolution failed for host_id=" + hostId, e); } } diff --git a/manual/core/address_resolution/README.md b/manual/core/address_resolution/README.md index 543802e8b15..c692aa63336 100644 --- a/manual/core/address_resolution/README.md +++ b/manual/core/address_resolution/README.md @@ -183,14 +183,19 @@ datastax-java-driver { #### DNS resolution -DNS is resolved at connection time (not at route discovery time). The driver delegates to -`InetAddress.getByName()`, which is a blocking call that uses the JVM's built-in DNS cache -(30 s default TTL in the JDK). It runs on the driver's dedicated name-resolver executor rather than -on a Netty event loop, so a slow or unresponsive DNS server delays the connection attempt itself but -does not stall the event loops. Connection establishment is still delayed, so it is worth configuring -the JVM DNS cache TTL via the `networkaddress.cache.ttl` security property (e.g. in -`$JAVA_HOME/conf/security/java.security` or programmatically with -`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`). +DNS is resolved at connection time (not at route discovery time), and through the same mechanism as +every other address the driver connects to: the route's hostname is handed to the connection layer +unresolved, and Netty's configured `AddressResolverGroup` expands it. A custom resolver installed via +`NettyOptions.afterBootstrapInitialized()` therefore applies to client routes as well, and a hostname +that maps to several addresses has all of them tried in turn. + +With Netty's default (JDK) resolver the lookup is a blocking `InetAddress` call that uses the JVM's +built-in DNS cache (30 s default TTL in the JDK). It runs on a Netty I/O event loop — never on the +admin event loop that drives control-connection reconnects — so it delays the connection attempt +itself. It is therefore worth configuring the JVM DNS cache TTL via the `networkaddress.cache.ttl` +security property (e.g. in `$JAVA_HOME/conf/security/java.security` or programmatically with +`java.security.Security.setProperty("networkaddress.cache.ttl", "60")`), or installing +`DnsAddressResolverGroup` for non-blocking resolution. - **Route-map refresh** — the driver re-queries `system.client_routes` and atomically swaps the in-memory route map in two situations: diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index 694a11273bf..53cbb2e2f5b 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -36,6 +36,16 @@ installed through `NettyOptions.afterBootstrapInitialized()` therefore keeps wor lookup blocks the I/O event loop it runs on — install `DnsAddressResolverGroup` for non-blocking resolution. +The same now applies to the two other address sources that used to perform their own JVM DNS lookups: +the Cloud/SNI proxy address and client-route hostnames are expanded by the connection layer too. A +custom Netty resolver applies to them for the first time, and a proxy hostname with several A-records +has all of them tried within a single connection attempt. + +There is **no public API change**: `EndPoint.resolve()` keeps its signature and is not deprecated. +Third-party `EndPoint` implementations keep working unchanged, with one new expectation — an +implementation should return its address as-is rather than looking names up itself, since resolution +now happens in the connection layer and `resolve()` is called from an event loop. + As part of this change: - `advanced.resolve-contact-points` is deprecated and now has **no effect**. Contact points are From 97ad98148aa5089094d71856eb7a53ac7bfe3e6f Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 01:39:00 +0200 Subject: [PATCH 18/33] test: assert on host strings, not resolved IPs, in ClientRoutesIT (DRIVER-201) Fallout from 5b79b630b6: a client route is now handed to the connection layer unresolved, so `endPoint.resolve()` yields an unresolved InetSocketAddress for a proxied node and `getAddress()` is null. Three assertions dereferenced it and NPE'd on the Scylla LATEST/LTS-LATEST isolated jobs (the two backends that have system.client_routes): ClientRoutesIT.classifyNodes:209 ClientRoutesIT.collectHostIds:323 ClientRoutesIT.should_refresh_routes_after_table_update:543 All three compare against IP literals (NLB_ADDRESS, the ccm node address), so getHostString() is the right accessor: it returns the literal for a resolved address and the hostname for an unresolved one, and is never null. The refresh-after-update assertion also now states outright that the route comes back unresolved, so the contract is pinned rather than incidental. Driver behaviour is unaffected -- this is test-side only. MockResolverIT (3/3, including replace_cluster_test and the dead-first-DNS-entry case) passed in the same run, as did every Cassandra isolated/serial job and every Scylla serial job. Co-Authored-By: Claude Opus 5 (1M context) --- .../oss/driver/core/clientroutes/ClientRoutesIT.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java index 66bf16a0390..e658f843360 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java @@ -206,7 +206,9 @@ private NodeClassification classifyNodes(CqlSession session) { NodeClassification result = new NodeClassification(); for (Node node : session.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // getHostString() rather than getAddress().getHostAddress(): a client route is handed over + // unresolved (the connection layer resolves it), so getAddress() is null for proxied nodes. + String ip = addr.getHostString(); UUID hostId = node.getHostId(); boolean connected = node.getOpenConnections() > 0; LOG.info( @@ -319,7 +321,8 @@ private Map collectHostIds(CcmBridge ccm, int nodeCount, String t .build()) { for (Node node : adminSession.getMetadata().getNodes().values()) { InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); - String ip = addr.getAddress().getHostAddress(); + // See classifyNodes(): a client route is unresolved, so getAddress() would be null. + String ip = addr.getHostString(); Integer nodeId = ipToNodeId.get(ip); if (nodeId != null && node.getHostId() != null) { hostIds.put(nodeId, node.getHostId()); @@ -540,7 +543,10 @@ public void should_refresh_routes_after_table_update() throws Exception { () -> { InetSocketAddress resolved = handler.resolve(hostId); assertThat(resolved).isNotNull(); - assertThat(resolved.getAddress().getHostAddress()).isEqualTo(nodeAddr); + // The route is returned unresolved on purpose -- ChannelFactory resolves it through + // Netty's AddressResolverGroup -- so assert on the host string, not getAddress(). + assertThat(resolved.isUnresolved()).isTrue(); + assertThat(resolved.getHostString()).isEqualTo(nodeAddr); assertThat(resolved.getPort()).isEqualTo(9042); }); } From a0328a7e8779cc609d531f190bb6f3bd738d91e5 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 15:39:19 +0200 Subject: [PATCH 19/33] fix: complete the connect future on failures in resolver and bootstrap callbacks (DRIVER-201) ChannelFactory.connect() had no timeout of its own, yet several of its async seams could die without completing the caller's future, hanging control-connection init or a pool reconnect forever: - resolveCandidates() only guarded getResolver(). A custom resolver throwing synchronously from isSupported()/isResolved()/resolveAll(), or a throw from the resolveAll listener body, killed the event-loop task with the future still pending (Netty only logs those). - eventExecutor.execute() itself throws RejectedExecutionException while the group shuts down, and escaped synchronously out of connect(), which never used to throw. - tryNextCandidate() runs in CompletionStage continuations that swallow throwables; a custom PinnableEndPoint.pinTo() throwing was lost. - connectToAddress()'s connect listener contains the downgrade recursion, the version-registry lookup and the cloud config override, all inside a Netty listener that swallows throwables. - A third-party EndPoint.resolve() returning null (contractually forbidden) NPE'd inside the event-loop task instead of failing fast; before multi-address support this failed synchronously in Bootstrap.connect(null). Establish the invariant that every path completes the future: blanket try/catch around the resolver task, the resolveAll listener, the execute() dispatch, tryNextCandidate() and its whenComplete continuation, connectToAddress()'s synchronous section and its connect listener, plus a fail-fast null check after EndPoint.resolve(). Double completion is harmless: completeExceptionally() on a completed future is a no-op, which the existing initializer error path already relies on. Tests cover each seam: a resolver whose every method throws, a null resolve(), a shut-down event loop group (rejected dispatch), a throwing pinTo(), and a version registry that throws inside the connect listener. Co-Authored-By: Claude Fable 5 --- .../internal/core/channel/ChannelFactory.java | 424 ++++++++++-------- .../ChannelFactoryMultiAddressTest.java | 67 +++ .../ChannelFactoryNettyResolverTest.java | 83 ++++ .../ChannelFactoryPinnedEndPointTest.java | 35 ++ ...ChannelFactoryProtocolNegotiationTest.java | 36 ++ 5 files changed, 456 insertions(+), 189 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 8b86ea36c23..b9b09639e42 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -256,6 +256,13 @@ private void connect( resultFuture.completeExceptionally(e); return; } + if (address == null) { + // EndPoint.resolve() is contractually non-null; fail fast instead of NPE-ing inside an + // event-loop task later, which would leave resultFuture hanging (see resolveCandidates()). + resultFuture.completeExceptionally( + new IllegalArgumentException("EndPoint.resolve() returned null: " + endPoint)); + return; + } resolveCandidates(baseBootstrap, address) .whenComplete( @@ -350,39 +357,53 @@ private CompletionStage> resolveCandidates( // channel's own event loop. EventExecutor eventExecutor = context.getNettyOptions().ioEventLoopGroup().next(); CompletableFuture> result = new CompletableFuture<>(); - eventExecutor.execute( - () -> { - AddressResolver resolver; - try { - resolver = resolverGroup.getResolver(eventExecutor); - } catch (Throwable t) { - result.completeExceptionally(t); - return; - } - if (!resolver.isSupported(address) || resolver.isResolved(address)) { - // Nothing for the resolver to do; same short-circuit as Bootstrap#doResolveAndConnect0. - result.complete(Collections.singletonList(address)); - return; - } - resolver - .resolveAll(address) - .addListener( - (Future> future) -> { - if (!future.isSuccess()) { - result.completeExceptionally(future.cause()); - return; - } - @SuppressWarnings("unchecked") - List addresses = - (List) future.getNow(); - if (addresses == null || addresses.isEmpty()) { - result.completeExceptionally( - new IllegalStateException("Resolver returned no address for " + address)); - return; - } - result.complete(rotate(addresses)); - }); - }); + // Every path below must complete `result`: nothing at this stage has a timeout, so a task or + // listener that dies with the future still pending (Netty swallows their throwables, it only + // logs them) would hang the connect attempt -- and with it control-connection init or a pool + // reconnect -- forever. Hence the blanket catches around the task body, the listener body, and + // the execute() call itself (which throws RejectedExecutionException while shutting down). + try { + eventExecutor.execute( + () -> { + try { + AddressResolver resolver = + resolverGroup.getResolver(eventExecutor); + if (!resolver.isSupported(address) || resolver.isResolved(address)) { + // Nothing for the resolver to do; same short-circuit as + // Bootstrap#doResolveAndConnect0. + result.complete(Collections.singletonList(address)); + return; + } + resolver + .resolveAll(address) + .addListener( + (Future> future) -> { + try { + if (!future.isSuccess()) { + result.completeExceptionally(future.cause()); + return; + } + @SuppressWarnings("unchecked") + List addresses = + (List) future.getNow(); + if (addresses == null || addresses.isEmpty()) { + result.completeExceptionally( + new IllegalStateException( + "Resolver returned no address for " + address)); + return; + } + result.complete(rotate(addresses)); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + } catch (Throwable t) { + result.completeExceptionally(t); + } return result; } @@ -451,69 +472,81 @@ private void tryNextCandidate( int index, List priorErrors) { - SocketAddress candidate = candidates.get(index); - // Everything downstream of here -- the channel, its pipeline (SSL engine, authenticator) and - // the - // DriverChannel handed to the caller -- sees an endpoint bound to this one address instead of - // the - // multi-address original. See PinnableEndPoint for why that matters. - EndPoint pinnedEndPoint = pin(endPoint, candidate); - CompletableFuture perAddressFuture = new CompletableFuture<>(); - // Fresh per candidate address: connectToAddress()'s downgrade retries stay on this one - // address, so the final UnsupportedProtocolVersionException (if negotiation is what dooms - // this candidate) only reports versions actually tried against it, not earlier candidates'. - List attemptedVersions = new CopyOnWriteArrayList<>(); - connectToAddress( - baseBootstrap, - pinnedEndPoint, - shardingInfo, - shardId, - options, - nodeMetricUpdater, - currentVersion, - isNegotiating, - attemptedVersions, - perAddressFuture, - candidate); - - perAddressFuture.whenComplete( - (channel, error) -> { - if (error == null) { - resultFuture.complete(channel); - } else if (index + 1 < candidates.size()) { - LOG.debug( - "[{}] Failed to connect to {} ({}), trying next address", - logPrefix, - candidate, - error.getMessage()); - priorErrors.add(error); - tryNextCandidate( - baseBootstrap, - // Deliberately the original, not the pinned copy: the next candidate must be pinned - // from the unpinned endpoint. - endPoint, - shardingInfo, - shardId, - options, - nodeMetricUpdater, - currentVersion, - isNegotiating, - resultFuture, - candidates, - index + 1, - priorErrors); - } else { - // All candidates exhausted. Surface the last error, carrying the earlier failures as - // suppressed exceptions so they are not lost (they were only logged at DEBUG above). - for (Throwable priorError : priorErrors) { - if (priorError != error) { - error.addSuppressed(priorError); + // Invariant: this method always (eventually) completes resultFuture. It is invoked from + // CompletionStage and Netty callbacks that swallow throwables, so a synchronous throw -- a + // custom PinnableEndPoint.pinTo() for instance -- would otherwise leave the connect attempt + // hanging forever. Double completion is harmless: completeExceptionally() on an already + // completed future is a no-op. + try { + SocketAddress candidate = candidates.get(index); + // Everything downstream of here -- the channel, its pipeline (SSL engine, authenticator) and + // the DriverChannel handed to the caller -- sees an endpoint bound to this one address + // instead of the multi-address original. See PinnableEndPoint for why that matters. + EndPoint pinnedEndPoint = pin(endPoint, candidate); + CompletableFuture perAddressFuture = new CompletableFuture<>(); + // Fresh per candidate address: connectToAddress()'s downgrade retries stay on this one + // address, so the final UnsupportedProtocolVersionException (if negotiation is what dooms + // this candidate) only reports versions actually tried against it, not earlier candidates'. + List attemptedVersions = new CopyOnWriteArrayList<>(); + connectToAddress( + baseBootstrap, + pinnedEndPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + attemptedVersions, + perAddressFuture, + candidate); + + perAddressFuture.whenComplete( + (channel, error) -> { + try { + if (error == null) { + resultFuture.complete(channel); + } else if (index + 1 < candidates.size()) { + LOG.debug( + "[{}] Failed to connect to {} ({}), trying next address", + logPrefix, + candidate, + error.getMessage()); + priorErrors.add(error); + tryNextCandidate( + baseBootstrap, + // Deliberately the original, not the pinned copy: the next candidate must be + // pinned from the unpinned endpoint. + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + currentVersion, + isNegotiating, + resultFuture, + candidates, + index + 1, + priorErrors); + } else { + // All candidates exhausted. Surface the last error, carrying the earlier failures + // as suppressed exceptions so they are not lost (they were only logged at DEBUG + // above). + for (Throwable priorError : priorErrors) { + if (priorError != error) { + error.addSuppressed(priorError); + } + } + // Note: might be completed already if the failure happened in initializer() + resultFuture.completeExceptionally(error); } + } catch (Throwable t) { + resultFuture.completeExceptionally(t); } - // Note: might be completed already if the failure happened in initializer() - resultFuture.completeExceptionally(error); - } - }); + }); + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } } /** @@ -535,108 +568,121 @@ private void connectToAddress( CompletableFuture perAddressFuture, SocketAddress resolvedAddress) { - // clone() so each attempt gets its own handler while sharing the group, options and resolver - // configuration (including anything afterBootstrapInitialized() set). - Bootstrap bootstrap = - baseBootstrap - .clone() - .handler( - initializer( - endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); - - ChannelFuture connectFuture; - if (shardId == null || shardingInfo == null) { - if (shardId != null) { - LOG.debug( - "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", - shardId, - endPoint); - } - connectFuture = bootstrap.connect(resolvedAddress); - } else { - int localPort = - PortAllocator.getNextAvailablePort(shardingInfo.getShardsCount(), shardId, context); - if (localPort == -1) { - LOG.warn( - "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", - shardId, - endPoint); + // Invariant, as in tryNextCandidate(): every path completes perAddressFuture. The synchronous + // section can throw from Bootstrap validation; the connect listener runs inside a Netty + // callback that swallows throwables and contains the downgrade recursion, the version-registry + // lookup and the config overrides, any of which throwing would otherwise hang the attempt. + try { + // clone() so each attempt gets its own handler while sharing the group, options and resolver + // configuration (including anything afterBootstrapInitialized() set). + Bootstrap bootstrap = + baseBootstrap + .clone() + .handler( + initializer( + endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); + + ChannelFuture connectFuture; + if (shardId == null || shardingInfo == null) { + if (shardId != null) { + LOG.debug( + "Requested connection to shard {} but shardingInfo is currently missing for Node at endpoint {}. Falling back to arbitrary local port.", + shardId, + endPoint); + } connectFuture = bootstrap.connect(resolvedAddress); } else { - connectFuture = bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + int localPort = + PortAllocator.getNextAvailablePort(shardingInfo.getShardsCount(), shardId, context); + if (localPort == -1) { + LOG.warn( + "Could not find free port for shard {} at {}. Falling back to arbitrary local port.", + shardId, + endPoint); + connectFuture = bootstrap.connect(resolvedAddress); + } else { + connectFuture = bootstrap.connect(resolvedAddress, new InetSocketAddress(localPort)); + } } - } - connectFuture.addListener( - cf -> { - if (connectFuture.isSuccess()) { - Channel channel = connectFuture.channel(); - DriverChannel driverChannel = - new DriverChannel(endPoint, channel, context.getWriteCoalescer(), currentVersion); - // If this is the first successful connection, remember the protocol version and - // cluster name for future connections. - if (isNegotiating) { - ChannelFactory.this.protocolVersion = currentVersion; - } - if (ChannelFactory.this.clusterName == null) { - ChannelFactory.this.clusterName = driverChannel.getClusterName(); - } - Map> supportedOptions = driverChannel.getOptions(); - if (ChannelFactory.this.productType == null && supportedOptions != null) { - List productTypes = supportedOptions.get("PRODUCT_TYPE"); - String productType = - productTypes != null && !productTypes.isEmpty() - ? productTypes.get(0) - : UNKNOWN_PRODUCT_TYPE; - ChannelFactory.this.productType = productType; - DriverConfig driverConfig = context.getConfig(); - if (driverConfig instanceof TypesafeDriverConfig - && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { - ((TypesafeDriverConfig) driverConfig) - .overrideDefaults( - ImmutableMap.of( - DefaultDriverOption.REQUEST_CONSISTENCY, - ConsistencyLevel.LOCAL_QUORUM.name())); - } - } - perAddressFuture.complete(driverChannel); - } else { - Throwable error = connectFuture.cause(); - if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { - attemptedVersions.add(currentVersion); - Optional downgraded = - context.getProtocolVersionRegistry().downgrade(currentVersion); - if (downgraded.isPresent()) { - LOG.debug( - "[{}] Failed to connect with protocol {}, retrying with {}", - logPrefix, - currentVersion, - downgraded.get()); - // Stay on the same address for protocol-version downgrade retries. - connectToAddress( - baseBootstrap, - endPoint, - shardingInfo, - shardId, - options, - nodeMetricUpdater, - downgraded.get(), - true, - attemptedVersions, - perAddressFuture, - resolvedAddress); + connectFuture.addListener( + cf -> { + try { + if (connectFuture.isSuccess()) { + Channel channel = connectFuture.channel(); + DriverChannel driverChannel = + new DriverChannel( + endPoint, channel, context.getWriteCoalescer(), currentVersion); + // If this is the first successful connection, remember the protocol version and + // cluster name for future connections. + if (isNegotiating) { + ChannelFactory.this.protocolVersion = currentVersion; + } + if (ChannelFactory.this.clusterName == null) { + ChannelFactory.this.clusterName = driverChannel.getClusterName(); + } + Map> supportedOptions = driverChannel.getOptions(); + if (ChannelFactory.this.productType == null && supportedOptions != null) { + List productTypes = supportedOptions.get("PRODUCT_TYPE"); + String productType = + productTypes != null && !productTypes.isEmpty() + ? productTypes.get(0) + : UNKNOWN_PRODUCT_TYPE; + ChannelFactory.this.productType = productType; + DriverConfig driverConfig = context.getConfig(); + if (driverConfig instanceof TypesafeDriverConfig + && productType.equals(DATASTAX_CLOUD_PRODUCT_TYPE)) { + ((TypesafeDriverConfig) driverConfig) + .overrideDefaults( + ImmutableMap.of( + DefaultDriverOption.REQUEST_CONSISTENCY, + ConsistencyLevel.LOCAL_QUORUM.name())); + } + } + perAddressFuture.complete(driverChannel); } else { - perAddressFuture.completeExceptionally( - UnsupportedProtocolVersionException.forNegotiation( - endPoint, attemptedVersions)); + Throwable error = connectFuture.cause(); + if (error instanceof UnsupportedProtocolVersionException && isNegotiating) { + attemptedVersions.add(currentVersion); + Optional downgraded = + context.getProtocolVersionRegistry().downgrade(currentVersion); + if (downgraded.isPresent()) { + LOG.debug( + "[{}] Failed to connect with protocol {}, retrying with {}", + logPrefix, + currentVersion, + downgraded.get()); + // Stay on the same address for protocol-version downgrade retries. + connectToAddress( + baseBootstrap, + endPoint, + shardingInfo, + shardId, + options, + nodeMetricUpdater, + downgraded.get(), + true, + attemptedVersions, + perAddressFuture, + resolvedAddress); + } else { + perAddressFuture.completeExceptionally( + UnsupportedProtocolVersionException.forNegotiation( + endPoint, attemptedVersions)); + } + } else { + // Note: might be completed already if the failure happened in initializer(), this + // is fine + perAddressFuture.completeExceptionally(error); + } } - } else { - // Note: might be completed already if the failure happened in initializer(), this is - // fine - perAddressFuture.completeExceptionally(error); + } catch (Throwable t) { + perAddressFuture.completeExceptionally(t); } - } - }); + }); + } catch (Throwable t) { + perAddressFuture.completeExceptionally(t); + } } /** diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index a008fb95434..5de46167042 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -38,6 +38,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletionStage; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; import org.junit.Test; /** @@ -132,6 +134,54 @@ public void should_fail_future_when_endpoint_resolve_throws() { assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); } + @Test + public void should_fail_future_when_endpoint_resolve_returns_null() { + // EndPoint.resolve() is contractually non-null, but a broken third-party implementation must + // fail fast rather than NPE later inside an event-loop task, which would leave the future + // hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new NullResolvingEndPoint(), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed( + e -> + assertThat(e) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("returned null")); + } + + @Test + public void should_fail_future_when_event_loop_group_is_rejecting_tasks() + throws InterruptedException { + // Resolution is dispatched to an I/O event loop; if the group is already shutting down, that + // dispatch is rejected synchronously. The rejection must fail the future rather than escape to + // the caller (connect() never used to throw) or leave the future hanging. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + ChannelFactory factory = newChannelFactory(); + clientGroup.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS).sync(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture) + .isFailed(e -> assertThat(e).isInstanceOf(RejectedExecutionException.class)); + } + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ private void installResolver(AddressResolverGroup group) { doAnswer( @@ -165,4 +215,21 @@ public String asMetricPrefix() { return "test"; } } + + /** A broken third-party endpoint that violates {@code resolve()}'s non-null contract. */ + private static class NullResolvingEndPoint implements EndPoint { + + @NonNull + @Override + @SuppressWarnings("NullAway") // deliberately broken, that is the point of the test + public SocketAddress resolve() { + return null; + } + + @NonNull + @Override + public String asMetricPrefix() { + return "test"; + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java index 45971bc1560..535dd8b6e27 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -29,11 +29,16 @@ import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import io.netty.bootstrap.Bootstrap; import io.netty.channel.local.LocalAddress; +import io.netty.resolver.AddressResolver; import io.netty.resolver.AddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletionStage; import org.junit.Test; @@ -176,6 +181,84 @@ public void should_pass_already_resolved_address_through_untouched() { assertThat(resolverGroup.queried).isEmpty(); } + @Test + public void should_fail_future_when_resolver_throws_synchronously() { + // Given – a broken custom resolver that throws instead of returning a failed future. The throw + // happens inside an event-loop task, where nothing else would ever complete the connect future: + // nothing at this stage has a timeout, so before the blanket catch in resolveCandidates() this + // hung the connect attempt (and with it control-connection init) forever. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("broken resolver"); + installResolver(new ThrowingAddressResolverGroup(failure)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + + /** A resolver whose every method throws, standing in for a broken third-party implementation. */ + private static class ThrowingAddressResolverGroup extends AddressResolverGroup { + + private final RuntimeException failure; + + ThrowingAddressResolverGroup(RuntimeException failure) { + this.failure = failure; + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) { + return new AddressResolver() { + + @Override + public boolean isSupported(SocketAddress address) { + throw failure; + } + + @Override + public boolean isResolved(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve(SocketAddress address) { + throw failure; + } + + @Override + public Future resolve( + SocketAddress address, Promise promise) { + throw failure; + } + + @Override + public Future> resolveAll(SocketAddress address) { + throw failure; + } + + @Override + public Future> resolveAll( + SocketAddress address, Promise> promise) { + throw failure; + } + + @Override + public void close() { + // nothing to do + } + }; + } + } + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ private void installResolver(AddressResolverGroup group) { doAnswer( diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java index 7f6d2d57b5c..a3b1c0c8352 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java @@ -34,6 +34,7 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Arrays; +import java.util.Collections; import java.util.Objects; import java.util.concurrent.CompletionStage; import org.junit.Test; @@ -121,6 +122,40 @@ public void should_leave_non_pinnable_endpoints_untouched() { .isSuccess(channel -> assertThat(channel.getEndPoint()).isSameAs(SERVER_ADDRESS)); } + @Test + public void should_fail_future_when_pin_to_throws() { + // pinTo() runs in the continuation after resolution, whose exceptions CompletionStage + // swallows; a throwing implementation must fail the connect future rather than hang it. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + SocketAddress reachable = SERVER_ADDRESS.resolve(); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver( + new TestAddressResolverGroup(Collections.singletonList(reachable))); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + RuntimeException failure = new IllegalStateException("pinTo blew up"); + TestPinnableEndPoint endPoint = + new TestPinnableEndPoint(HOSTNAME) { + @NonNull + @Override + public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { + throw failure; + } + }; + + CompletionStage channelFuture = + factory.connect( + endPoint, null, null, DriverChannelOptions.DEFAULT, NoopNodeMetricUpdater.INSTANCE); + + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + /** * A {@link PinnableEndPoint} that can hold a pin to any {@link SocketAddress}, including the * local-transport addresses these tests connect over (which {@code DefaultEndPoint} cannot). diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java index fceb8777904..610db0744cf 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java @@ -280,6 +280,42 @@ public void should_fail_if_negotiation_finds_no_matching_version(int errorCode) }); } + @Test + public void should_fail_future_when_downgrade_lookup_throws_in_connect_listener() { + // Given – a version registry that throws when the factory looks up the downgrade. The lookup + // runs inside the Netty connect listener, which swallows throwables: without the blanket catch + // in connectToAddress() the connect future would never complete and the attempt would hang. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + RuntimeException failure = new IllegalStateException("registry broken"); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)).thenThrow(failure); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + // Server does not support v4, which is what sends the factory to the downgrade lookup + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + // Then + assertThatStage(channelFuture).isFailed(e -> assertThat(e).isSameAs(failure)); + } + /** * Depending on the Cassandra version, an "unsupported protocol" response can use different error * codes, so we test all of them. From ea8e1e2328166884992b65307fe08435872672bc Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 15:42:54 +0200 Subject: [PATCH 20/33] fix: preserve the queried hostname on resolver-returned addresses (DRIVER-201) The channel's pinned endpoint is built from the candidate address the resolver returned, and it is what DefaultSslEngineFactory and SniSslEngineFactory derive the SSL peer host from, inside the channel initializer. The JDK and Netty-DNS resolvers attach the queried name to the InetAddresses they return, but a custom resolver may build its results from raw address bytes. With such a nameless address: - InetSocketAddress#getHostName() triggers a blocking reverse-DNS lookup on the Netty event loop during SSL engine creation -- the very thing pinning was introduced to eliminate; and - TLS hostname validation checks the certificate against the IP or the PTR record instead of the name the user configured, failing (or worse, passing against a name the operator never chose). Re-attach the queried hostname centrally in ChannelFactory, right after expansion, so every endpoint type is covered in one place and pinTo() stores an address that already carries the right name. InetAddress.getByAddress(host, bytes) performs no lookup; the TCP connect target, address equality and rotation determinism are all unchanged. A candidate that already carries a real name (e.g. a CNAME target) is respected, and scoped IPv6 addresses are left alone since a rebuild would drop the scope id. Tests cover the re-attach (asserting getHostName() itself, which proves no reverse lookup happens), the resolver-name-wins case, non-Inet and already-resolved pass-through, bare IPv6, and scoped IPv6. Co-Authored-By: Claude Fable 5 --- .../internal/core/channel/ChannelFactory.java | 65 +++++++++++++- .../ChannelFactoryMultiAddressTest.java | 85 +++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index b9b09639e42..6943510eb0b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -60,9 +60,12 @@ import io.netty.util.concurrent.EventExecutor; import io.netty.util.concurrent.Future; import java.io.IOException; +import java.net.Inet6Address; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -392,7 +395,7 @@ private CompletionStage> resolveCandidates( "Resolver returned no address for " + address)); return; } - result.complete(rotate(addresses)); + result.complete(rotate(reattachHostnames(address, addresses))); } catch (Throwable t) { result.completeExceptionally(t); } @@ -412,6 +415,66 @@ private static boolean isResolved(SocketAddress address) { return address instanceof InetSocketAddress && !((InetSocketAddress) address).isUnresolved(); } + /** Applies {@link #reattachHostname} to every expanded candidate. */ + private static List reattachHostnames( + SocketAddress original, List candidates) { + List result = new ArrayList<>(candidates.size()); + for (SocketAddress candidate : candidates) { + result.add(reattachHostname(original, candidate)); + } + return result; + } + + /** + * Re-attaches the {@code original} (unresolved) address's host name to a resolved candidate that + * carries no name of its own. + * + *

The JDK and Netty-DNS resolvers already attach the queried name to the {@link InetAddress}es + * they return, so this is a no-op for them. A custom resolver, however, may build its results + * from raw address bytes. The channel's pinned endpoint is built from the candidate (see {@link + * PinnableEndPoint}), and it is what {@code DefaultSslEngineFactory} and {@code + * SniSslEngineFactory} derive the SSL peer host from, inside the channel initializer: with a + * nameless address, {@code InetSocketAddress#getHostName()} triggers a blocking reverse-DNS + * lookup on the event loop, and TLS hostname validation ends up checking the certificate against + * the IP or the PTR record instead of the name the user configured. + * + *

Re-attaching changes nothing else: {@code InetAddress.getByAddress(host, bytes)} performs no + * lookup, the TCP connect target is the same IP, a resolved {@link InetSocketAddress}'s equality + * ignores host names (so pinning and pin-equality shortcuts are unaffected), and {@link #rotate} + * stays deterministic because the transform is applied identically on every expansion. A + * candidate that already carries a real name (e.g. from a resolver that returns CNAME targets) is + * respected, and a scoped IPv6 address is left alone, since rebuilding it would drop the scope. + */ + @VisibleForTesting + static SocketAddress reattachHostname(SocketAddress original, SocketAddress candidate) { + if (!(original instanceof InetSocketAddress) || !(candidate instanceof InetSocketAddress)) { + return candidate; + } + InetSocketAddress originalInet = (InetSocketAddress) original; + InetSocketAddress candidateInet = (InetSocketAddress) candidate; + InetAddress candidateIp = candidateInet.getAddress(); + if (!originalInet.isUnresolved() + || candidateIp == null + // The candidate already carries a real name: respect the resolver's choice. (For a + // nameless address, getHostString() falls back to the IP literal; no lookup either way.) + || !candidateInet.getHostString().equals(candidateIp.getHostAddress()) + // Rebuilding a scoped IPv6 address would silently drop its scope. + || (candidateIp instanceof Inet6Address + && (((Inet6Address) candidateIp).getScopeId() != 0 + || ((Inet6Address) candidateIp).getScopedInterface() != null))) { + return candidate; + } + try { + return new InetSocketAddress( + InetAddress.getByAddress(originalInet.getHostString(), candidateIp.getAddress()), + candidateInet.getPort()); + } catch (UnknownHostException impossible) { + // getByAddress only rejects illegal byte lengths, and these bytes come from a real + // InetAddress; keep the raw candidate rather than failing the connect over a cosmetic step. + return candidate; + } + } + /** * Rotates the expanded address list so that successive connections to the same name do not all * start at the same address. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 5de46167042..56363e8af37 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -32,6 +32,8 @@ import io.netty.bootstrap.Bootstrap; import io.netty.channel.local.LocalAddress; import io.netty.resolver.AddressResolverGroup; +import java.net.Inet6Address; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Arrays; @@ -114,6 +116,89 @@ public void should_leave_a_single_address_alone() { .containsExactly(UNREACHABLE_1); } + // ---- reattachHostname() --------------------------------------------------- + + @Test + public void should_reattach_queried_hostname_to_nameless_resolved_address() throws Exception { + // A custom resolver may build its results from raw address bytes; the queried name must be + // re-attached so TLS hostname validation checks the configured name (not the IP or a PTR + // record) and reading the host name never triggers a reverse lookup on the event loop. + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9999); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.isUnresolved()).isFalse(); + // getHostString() never looks anything up; getHostName() reverse-resolves a *nameless* + // address, so it returning the queried name proves the name is embedded, not looked up. + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getHostName()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + // The candidate's port wins over the original's: a resolver may remap ports too. + assertThat(result.getPort()).isEqualTo(9999); + // Equality is unchanged (a resolved InetSocketAddress compares IP bytes + port only), so + // pinning and the pin-equality shortcuts behave exactly as with the raw candidate. + assertThat(result).isEqualTo(candidate); + } + + @Test + public void should_keep_resolver_provided_hostname() throws Exception { + // A resolver that attaches its own name (e.g. a CNAME target) wins over the queried one -- + // and this is also why the transform is a no-op for the JDK and Netty-DNS resolvers, which + // attach the queried name themselves. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("cname.example.fake", new byte[] {10, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(HOSTNAME, candidate)).isSameAs(candidate); + } + + @Test + public void should_pass_non_inet_candidate_through() { + // The local-transport addresses these unit tests connect over must never be touched. + assertThat(ChannelFactory.reattachHostname(HOSTNAME, UNREACHABLE_1)).isSameAs(UNREACHABLE_1); + } + + @Test + public void should_pass_candidate_through_when_original_is_resolved() throws Exception { + // A resolved original never went through the resolver, so there is no queried name to + // re-attach. + InetSocketAddress original = new InetSocketAddress("127.0.0.1", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {127, 0, 0, 1}), 9042); + + assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + } + + @Test + public void should_reattach_hostname_to_nameless_ipv6_address() throws Exception { + byte[] loopback = new byte[16]; + loopback[15] = 1; // ::1 + InetSocketAddress candidate = new InetSocketAddress(InetAddress.getByAddress(loopback), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isEqualTo(candidate.getAddress()); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_not_touch_scoped_ipv6_address() throws Exception { + // InetAddress.getByAddress(host, bytes) cannot carry a scope id, so rebuilding a link-local + // address would silently drop the scope and could break the actual connect. + byte[] linkLocal = new byte[16]; + linkLocal[0] = (byte) 0xfe; + linkLocal[1] = (byte) 0x80; + linkLocal[15] = 1; + InetSocketAddress candidate = + new InetSocketAddress(Inet6Address.getByAddress(null, linkLocal, 3), 9042); + + assertThat(ChannelFactory.reattachHostname(HOSTNAME, candidate)).isSameAs(candidate); + } + @Test public void should_fail_future_when_endpoint_resolve_throws() { // ChannelFactory calls EndPoint.resolve() directly on the caller thread, so a third-party From 092d492c3357b809612bcf5259c59226b6fa643b Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 15:49:00 +0200 Subject: [PATCH 21/33] fix: stop trying further addresses after protocol-version rejection (DRIVER-201) tryNextCandidate()'s javadoc promised that protocol-version negotiation exhaustion does not advance to the next candidate, but the code advanced on any error. A protocol-version rejection -- negotiation exhausting every downgrade, or the server refusing a forced version -- is a property of the node, not of the address the connection happened to use, so replaying the whole negotiation ladder against every remaining IP of the same name bought nothing and stretched the worst-case failure time from the documented N x connect-timeout to N x versions x connect-timeout. Make UnsupportedProtocolVersionException terminal in the candidate loop, matching both the javadoc and the pre-multi-address behaviour of a single-address connect. The javadoc now also spells out the corner this deliberately does not rescue (a heterogeneous rolling upgrade where IPs behind one name support different protocol versions) and that TCP/init/auth failures still advance, since those may well be address-specific. The new test expands a name to the same live server twice (sidestepping rotation nondeterminism), exhausts negotiation on the first candidate, and asserts the second is never attempted plus that the propagated UnsupportedProtocolVersionException carries no suppressed connect errors. The no-second-attempt check uses a new non-failing tryReadOutboundFrame() base helper and runs before the future assertion, so a regression drains the stray frame and fails cleanly instead of deadlocking the server-side exchanger in tearDown(). Also hoist the installResolver() helper, duplicated across two test classes and inlined in a third, into ChannelFactoryTestBase. Co-Authored-By: Claude Fable 5 --- .../internal/core/channel/ChannelFactory.java | 32 +++++++-- .../ChannelFactoryMultiAddressTest.java | 16 ----- .../ChannelFactoryNettyResolverTest.java | 12 ---- .../ChannelFactoryPinnedEndPointTest.java | 23 +------ ...ChannelFactoryProtocolNegotiationTest.java | 69 +++++++++++++++++++ .../core/channel/ChannelFactoryTestBase.java | 37 ++++++++++ 6 files changed, 133 insertions(+), 56 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 6943510eb0b..aca391a3110 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -507,9 +507,18 @@ static List rotate(List addresses) { /** * Iterates through the candidate addresses produced by {@link #resolveCandidates}. Tries each one - * in sequence; if an address fails for a reason other than protocol-version negotiation - * exhaustion, the next candidate is tried. Only when all candidates are exhausted is the overall - * {@code resultFuture} failed. + * in sequence; when an address fails, the next candidate is tried, and only when all candidates + * are exhausted is the overall {@code resultFuture} failed. + * + *

The one exception is a {@link UnsupportedProtocolVersionException}: a protocol-version + * rejection -- whether negotiation exhausted every downgrade or the server refused a forced + * version -- is a property of the node, not of the address the connection happened to use, so it + * fails the attempt immediately instead of replaying the whole negotiation ladder against every + * remaining IP (worst case {@code N × versions × connect-timeout} for nothing). This matches the + * pre-multi-address behaviour of a single-address connect. The corner it deliberately does not + * rescue: a heterogeneous rolling upgrade where different IPs behind one name genuinely support + * different protocol versions. Other failures -- TCP, init, authentication -- do advance to the + * next candidate, since with a multi-record name they may well be address-specific. * *

Timeout note: addresses are tried serially, so the worst-case time before failure is * {@code N × connect-timeout} where N is the number of candidates. This is an intentional @@ -569,7 +578,8 @@ private void tryNextCandidate( try { if (error == null) { resultFuture.complete(channel); - } else if (index + 1 < candidates.size()) { + } else if (!(error instanceof UnsupportedProtocolVersionException) + && index + 1 < candidates.size()) { LOG.debug( "[{}] Failed to connect to {} ({}), trying next address", logPrefix, @@ -592,9 +602,17 @@ private void tryNextCandidate( index + 1, priorErrors); } else { - // All candidates exhausted. Surface the last error, carrying the earlier failures - // as suppressed exceptions so they are not lost (they were only logged at DEBUG - // above). + if (index + 1 < candidates.size()) { + // Only reachable for an UnsupportedProtocolVersionException (see the javadoc). + LOG.debug( + "[{}] Not trying the remaining addresses of {}: a protocol-version rejection" + + " is a property of the node, not of the address ({})", + logPrefix, + endPoint, + error.getMessage()); + } + // Surface the last error, carrying the earlier failures as suppressed exceptions + // so they are not lost (they were only logged at DEBUG above). for (Throwable priorError : priorErrors) { if (priorError != error) { error.addSuppressed(priorError); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 56363e8af37..390df274604 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -19,8 +19,6 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.DefaultProtocolVersion; @@ -29,9 +27,7 @@ import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import edu.umd.cs.findbugs.annotations.NonNull; -import io.netty.bootstrap.Bootstrap; import io.netty.channel.local.LocalAddress; -import io.netty.resolver.AddressResolverGroup; import java.net.Inet6Address; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -267,18 +263,6 @@ public void should_fail_future_when_event_loop_group_is_rejecting_tasks() .isFailed(e -> assertThat(e).isInstanceOf(RejectedExecutionException.class)); } - /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ - private void installResolver(AddressResolverGroup group) { - doAnswer( - invocation -> { - Bootstrap bootstrap = invocation.getArgument(0); - bootstrap.resolver(group); - return null; - }) - .when(nettyOptions) - .afterBootstrapInitialized(any(Bootstrap.class)); - } - /** An endpoint whose {@link EndPoint#resolve()} throws, standing in for a broken third party. */ private static class ThrowingEndPoint implements EndPoint { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java index 535dd8b6e27..6197d45ee46 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -258,16 +258,4 @@ public void close() { }; } } - - /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ - private void installResolver(AddressResolverGroup group) { - doAnswer( - invocation -> { - Bootstrap bootstrap = invocation.getArgument(0); - bootstrap.resolver(group); - return null; - }) - .when(nettyOptions) - .afterBootstrapInitialized(any(Bootstrap.class)); - } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java index a3b1c0c8352..4bf21472a79 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryPinnedEndPointTest.java @@ -19,8 +19,6 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.DefaultProtocolVersion; @@ -29,7 +27,6 @@ import com.datastax.oss.driver.internal.core.metadata.PinnableEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import edu.umd.cs.findbugs.annotations.NonNull; -import io.netty.bootstrap.Bootstrap; import io.netty.channel.local.LocalAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -67,15 +64,7 @@ public void should_pin_channel_endpoint_to_the_address_that_connected() { when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); SocketAddress reachable = SERVER_ADDRESS.resolve(); - doAnswer( - invocation -> { - Bootstrap bootstrap = invocation.getArgument(0); - bootstrap.resolver( - new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, reachable))); - return null; - }) - .when(nettyOptions) - .afterBootstrapInitialized(any(Bootstrap.class)); + installResolver(new TestAddressResolverGroup(Arrays.asList(UNREACHABLE, reachable))); ChannelFactory factory = newChannelFactory(); TestPinnableEndPoint endPoint = new TestPinnableEndPoint(HOSTNAME); @@ -129,15 +118,7 @@ public void should_fail_future_when_pin_to_throws() { when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); SocketAddress reachable = SERVER_ADDRESS.resolve(); - doAnswer( - invocation -> { - Bootstrap bootstrap = invocation.getArgument(0); - bootstrap.resolver( - new TestAddressResolverGroup(Collections.singletonList(reachable))); - return null; - }) - .when(nettyOptions) - .afterBootstrapInitialized(any(Bootstrap.class)); + installResolver(new TestAddressResolverGroup(Collections.singletonList(reachable))); ChannelFactory factory = newChannelFactory(); RuntimeException failure = new IllegalStateException("pinTo blew up"); TestPinnableEndPoint endPoint = diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java index 610db0744cf..d56d3b04c38 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.UnsupportedProtocolVersionException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.internal.core.TestResponses; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import com.datastax.oss.protocol.internal.Frame; import com.datastax.oss.protocol.internal.ProtocolConstants; @@ -33,6 +34,9 @@ import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletionStage; import org.junit.Test; @@ -280,6 +284,71 @@ public void should_fail_if_negotiation_finds_no_matching_version(int errorCode) }); } + @Test + public void should_not_try_next_address_when_negotiation_exhausts_versions() { + // Given – a name that expands to two candidates (the same live server twice, so whichever the + // rotation picks first is irrelevant), and a server that rejects every protocol version. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)) + .thenReturn(Optional.of(DefaultProtocolVersion.V3)); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V3)).thenReturn(Optional.empty()); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + + // First candidate: V4 rejected, downgrade retry with V3 rejected -> negotiation exhausted + Frame requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V4.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.message).isInstanceOf(Options.class); + writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); + + requestFrame = readOutboundFrame(); + assertThat(requestFrame.protocolVersion).isEqualTo(DefaultProtocolVersion.V3.getCode()); + writeInboundFrame( + requestFrame, + new Error( + ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); + + // Then – the second candidate must not be attempted: a protocol-version rejection is a + // property of the node, not of the address, so replaying the negotiation ladder against the + // remaining IPs would buy nothing. Checked before the future assertion so that on regression + // the stray frame is drained; leaving it unread would block the server's exchanger and hang + // the whole suite in tearDown() instead of failing this test. + assertThat(tryReadOutboundFrame(200)) + .as("second candidate must not be attempted after negotiation exhaustion") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("no other candidate should have been tried, so nothing to suppress") + .isEmpty(); + }); + } + @Test public void should_fail_future_when_downgrade_lookup_throws_in_connect_listener() { // Given – a version registry that throws when the factory looks up the downgrade. The lookup diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java index 91da31f0606..d77d235a141 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java @@ -20,6 +20,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.ProtocolVersion; @@ -42,6 +44,7 @@ import com.datastax.oss.protocol.internal.request.Startup; import com.datastax.oss.protocol.internal.response.Ready; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -53,6 +56,8 @@ import io.netty.channel.DefaultEventLoopGroup; import io.netty.channel.local.LocalChannel; import io.netty.channel.local.LocalServerChannel; +import io.netty.resolver.AddressResolverGroup; +import java.net.SocketAddress; import java.time.Duration; import java.util.Collections; import java.util.Optional; @@ -191,6 +196,38 @@ protected Frame readOutboundFrame() { return null; // never reached } + /** + * Like {@link #readOutboundFrame()}, but returns {@code null} instead of failing the test when no + * frame arrives within {@code timeoutMillis}. + * + *

Use this to assert that the client did not send another request. Unlike asserting via + * a failing read, it also drains a frame that does arrive: the server-side exchange in {@link + * ServerInitializer} has no timeout, so a stray unread frame would block the server event loop + * and hang the whole suite in {@link #tearDown()} instead of failing just the test. + */ + protected Frame tryReadOutboundFrame(long timeoutMillis) { + try { + return requestFrameExchanger.exchange(null, timeoutMillis, MILLISECONDS); + } catch (InterruptedException e) { + fail("unexpected interruption while waiting for outbound frame", e); + return null; // never reached + } catch (TimeoutException e) { + return null; + } + } + + /** Installs {@code group} the way a user would, through the {@code NettyOptions} hook. */ + protected void installResolver(AddressResolverGroup group) { + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.resolver(group); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + } + protected void writeInboundFrame(Frame requestFrame, Message response) { writeInboundFrame(requestFrame, response, requestFrame.protocolVersion); } From 73f9342019da395595d44fb549d24a5a1106199b Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 15:54:59 +0200 Subject: [PATCH 22/33] fix: resolve and connect on a single event loop per connection attempt (DRIVER-201) resolveCandidates() took one event loop from the I/O group for name resolution, and Bootstrap.connect() then advanced the group's chooser again when registering the channel. Every unresolved-address connect -- which is all cloud/SNI pool connections, client routes, and contact points -- therefore advanced the round-robin chooser by exactly two, and with the default power-of-two chooser that parks every channel on loops of a single parity: half the I/O threads carry all the traffic. Pick the event loop once per logical connect, run resolution on it, and bind the per-attempt bootstrap clones to it with clone(EventLoop): the chooser now advances exactly once per connect on both the resolved and unresolved paths, and resolution runs on the connecting channel's own loop -- which is precisely what Netty's Bootstrap does with an unresolved address. The base bootstrap keeps the full group, so the afterBootstrapInitialized() hook observes the same group as before. The new test registers which executor the resolver was created for and asserts the connected channel's event loop is that same object, using a two-thread group: with the base's single-thread group the assertion would be vacuous, while with two threads the old code deterministically split resolution and registration across different loops. Co-Authored-By: Claude Fable 5 --- .../internal/core/channel/ChannelFactory.java | 50 +++++++++++++------ .../ChannelFactoryNettyResolverTest.java | 39 +++++++++++++++ .../channel/TestAddressResolverGroup.java | 4 ++ 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index aca391a3110..0782a988312 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -55,9 +55,9 @@ import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoop; import io.netty.resolver.AddressResolver; import io.netty.resolver.AddressResolverGroup; -import io.netty.util.concurrent.EventExecutor; import io.netty.util.concurrent.Future; import java.io.IOException; import java.net.Inet6Address; @@ -241,9 +241,19 @@ private void connect( // AddressResolverGroup (see resolveCandidates()), and it means the user's // afterBootstrapInitialized() hook runs once per logical connection instead of once per address // attempt. Each attempt gets its own clone() with its own handler. + // + // The event loop is likewise picked once per connect() and shared by name resolution and the + // channel itself (the per-attempt clones are bound to it, see connectToAddress()). Advancing + // the group's round-robin chooser exactly once per connect keeps channels evenly distributed: + // taking one loop for resolution and letting Bootstrap.connect() take another would advance + // the chooser twice per connect, parking all channels on half the loops with the default + // power-of-two chooser. It also mirrors what Netty itself does with an unresolved address: + // Bootstrap resolves on the connecting channel's own event loop. Bootstrap baseBootstrap; + EventLoop eventLoop; try { baseBootstrap = newBootstrap(); + eventLoop = context.getNettyOptions().ioEventLoopGroup().next(); } catch (Exception e) { resultFuture.completeExceptionally(e); return; @@ -267,7 +277,7 @@ private void connect( return; } - resolveCandidates(baseBootstrap, address) + resolveCandidates(baseBootstrap, address, eventLoop) .whenComplete( (candidates, error) -> { if (error != null) { @@ -280,6 +290,7 @@ private void connect( } tryNextCandidate( baseBootstrap, + eventLoop, endPoint, shardingInfo, shardId, @@ -297,8 +308,10 @@ private void connect( /** * Builds the {@link Bootstrap} shared by every connection attempt of a single {@code connect()} * call, including the user's {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} hook. Per - * attempt, {@link #connectToAddress} takes a {@link Bootstrap#clone()} of it and installs its own - * handler; the copy carries the resolver configuration over. + * attempt, {@link #connectToAddress} takes a {@link + * Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it bound to the event loop the connect + * picked, and installs its own handler; the copy carries the resolver configuration over. The + * base bootstrap itself keeps the full I/O group, so the hook observes the same group as always. */ private Bootstrap newBootstrap() { NettyOptions nettyOptions = context.getNettyOptions(); @@ -337,7 +350,7 @@ private Bootstrap newBootstrap() { * effect. */ private CompletionStage> resolveCandidates( - Bootstrap bootstrap, SocketAddress address) { + Bootstrap bootstrap, SocketAddress address, EventLoop eventLoop) { // Nothing to expand, and nothing for a resolver to contribute: an already-resolved address is // exactly what we will connect to, and the resolver would pass it through untouched anyway. @@ -354,11 +367,11 @@ private CompletionStage> resolveCandidates( return CompletableFuture.completedFuture(Collections.singletonList(address)); } - // The resolver must be obtained for -- and used from -- an event executor whose transport - // matches the channel class, since DnsAddressResolverGroup registers a datagram channel on it. - // An I/O event loop is also what Netty itself uses here: Bootstrap resolves on the connecting - // channel's own event loop. - EventExecutor eventExecutor = context.getNettyOptions().ioEventLoopGroup().next(); + // The supplied event loop is the same one the channel will be registered on (see connect()), + // which is what Netty itself does with an unresolved address: Bootstrap resolves on the + // connecting channel's own event loop. Its transport also matches the channel class, which + // matters because DnsAddressResolverGroup registers a datagram channel on the executor it + // resolves for. CompletableFuture> result = new CompletableFuture<>(); // Every path below must complete `result`: nothing at this stage has a timeout, so a task or // listener that dies with the future still pending (Netty swallows their throwables, it only @@ -366,11 +379,11 @@ private CompletionStage> resolveCandidates( // reconnect -- forever. Hence the blanket catches around the task body, the listener body, and // the execute() call itself (which throws RejectedExecutionException while shutting down). try { - eventExecutor.execute( + eventLoop.execute( () -> { try { AddressResolver resolver = - resolverGroup.getResolver(eventExecutor); + resolverGroup.getResolver(eventLoop); if (!resolver.isSupported(address) || resolver.isResolved(address)) { // Nothing for the resolver to do; same short-circuit as // Bootstrap#doResolveAndConnect0. @@ -532,6 +545,7 @@ static List rotate(List addresses) { */ private void tryNextCandidate( Bootstrap baseBootstrap, + EventLoop eventLoop, EndPoint endPoint, NodeShardingInfo shardingInfo, Integer shardId, @@ -562,6 +576,7 @@ private void tryNextCandidate( List attemptedVersions = new CopyOnWriteArrayList<>(); connectToAddress( baseBootstrap, + eventLoop, pinnedEndPoint, shardingInfo, shardId, @@ -588,6 +603,7 @@ private void tryNextCandidate( priorErrors.add(error); tryNextCandidate( baseBootstrap, + eventLoop, // Deliberately the original, not the pinned copy: the next candidate must be // pinned from the unpinned endpoint. endPoint, @@ -638,6 +654,7 @@ private void tryNextCandidate( */ private void connectToAddress( Bootstrap baseBootstrap, + EventLoop eventLoop, EndPoint endPoint, NodeShardingInfo shardingInfo, Integer shardId, @@ -654,11 +671,13 @@ private void connectToAddress( // callback that swallows throwables and contains the downgrade recursion, the version-registry // lookup and the config overrides, any of which throwing would otherwise hang the attempt. try { - // clone() so each attempt gets its own handler while sharing the group, options and resolver - // configuration (including anything afterBootstrapInitialized() set). + // clone(eventLoop) so each attempt gets its own handler while sharing the options and + // resolver configuration (including anything afterBootstrapInitialized() set), and is + // registered on the event loop the connect() picked -- the same one resolution ran on, so + // the group's chooser advances exactly once per logical connect (see connect()). Bootstrap bootstrap = baseBootstrap - .clone() + .clone(eventLoop) .handler( initializer( endPoint, currentVersion, options, nodeMetricUpdater, perAddressFuture)); @@ -736,6 +755,7 @@ private void connectToAddress( // Stay on the same address for protocol-version downgrade retries. connectToAddress( baseBootstrap, + eventLoop, endPoint, shardingInfo, shardId, diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java index 6197d45ee46..243638f48db 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -28,6 +28,7 @@ import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; import io.netty.bootstrap.Bootstrap; +import io.netty.channel.DefaultEventLoopGroup; import io.netty.channel.local.LocalAddress; import io.netty.resolver.AddressResolver; import io.netty.resolver.AddressResolverGroup; @@ -40,6 +41,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; import org.junit.Test; /** @@ -181,6 +183,43 @@ public void should_pass_already_resolved_address_through_untouched() { assertThat(resolverGroup.queried).isEmpty(); } + @Test + public void should_resolve_and_connect_on_the_same_event_loop() throws InterruptedException { + // Resolution and channel registration must share the loop picked once per connect. Taking one + // loop for resolution and letting the registration pick another would advance the group's + // round-robin chooser twice per connect, parking every channel on half the loops with the + // default power-of-two chooser. The base's single-thread group would make this assertion + // vacuous, so use two loops -- on which the split behavior was deterministic. + DefaultEventLoopGroup twoLoops = new DefaultEventLoopGroup(2); + try { + when(nettyOptions.ioEventLoopGroup()).thenReturn(twoLoops); + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup(Collections.singletonList(SERVER_ADDRESS.resolve())); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(HOSTNAME), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + assertThatStage(channelFuture) + .isSuccess( + channel -> + assertThat((Object) channel.eventLoop()) + .as("the channel must be registered on the loop resolution ran on") + .isSameAs(resolverGroup.resolverExecutor)); + } finally { + twoLoops.shutdownGracefully(0, 100, TimeUnit.MILLISECONDS).sync(); + } + } + @Test public void should_fail_future_when_resolver_throws_synchronously() { // Given – a broken custom resolver that throws instead of returning a failed future. The throw diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java index abf69508a85..fe544936376 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java @@ -50,6 +50,9 @@ class TestAddressResolverGroup extends AddressResolverGroup { /** Whether a resolver was ever obtained from this group at all. */ volatile boolean resolverRequested; + /** The executor the last resolver was created for, i.e. the loop resolution runs on. */ + @Nullable volatile EventExecutor resolverExecutor; + /** The addresses to answer with, or {@code null} to fail every lookup. */ @Nullable private final List answer; @@ -60,6 +63,7 @@ class TestAddressResolverGroup extends AddressResolverGroup { @Override protected AddressResolver newResolver(EventExecutor executor) { resolverRequested = true; + resolverExecutor = executor; return new AddressResolver() { @Override From 41f401d2eefc1a8ba5a31955f50f06ff6a947f99 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 15:58:09 +0200 Subject: [PATCH 23/33] fix: rotate a hostname's addresses with a per-name counter (DRIVER-201) The rotation offset was one global counter shared by every name the driver expands. Names whose expansions interleave in lockstep -- for example two hostname contact points expanded in sequence on every control-connection reconnection round -- each only ever saw one offset parity, pinning every name with an even record count to a fixed starting address and defeating the rotation entirely. This is the same failure mode that once collapsed SniEndPoint's rotation when SSL engine setup shared its counter (fixed then by splitting the counters), now across names instead of across methods. Track one counter per name, keyed by the queried address's lowercased host string, with a single fallback counter for the rare non-name-based original. The map is never evicted; its keys are the distinct names the driver ever expands (contact points, the SNI proxy name, client-route hostnames), each holding one AtomicInteger, so growth is bounded by configuration and topology. The single-address short-circuit now also documents (and the test asserts) that no counter is created or advanced for it. The new independence test interleaves two fresh names and asserts each rotates on its own and neither perturbs the other -- it fails with a shared global counter. Co-Authored-By: Claude Fable 5 --- .../internal/core/channel/ChannelFactory.java | 47 +++++++++++++---- .../ChannelFactoryMultiAddressTest.java | 51 ++++++++++++++++--- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 0782a988312..0342b21ae73 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -70,11 +70,13 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -120,12 +122,26 @@ public class ChannelFactory { protected final InternalDriverContext context; /** - * Round-robin counter used by {@link #rotate} to vary which of a name's addresses a connection - * tries first. Static so the rotation spans every endpoint and session in the JVM, which is all - * the spreading it needs to do; {@code SniEndPoint} used to hold an equivalent counter of its - * own, before resolution moved here. + * Round-robin counters used by {@link #rotate} to vary which of a name's addresses a connection + * tries first, one counter per name. Static so the rotation spans every endpoint and session in + * the JVM, which is all the spreading it needs to do; {@code SniEndPoint} used to hold an + * equivalent (single) counter of its own, before resolution moved here. + * + *

Per name rather than one global counter, because names whose expansions interleave in + * lockstep -- say two hostname contact points tried in sequence on every reconnection round -- + * would each only ever see one offset parity, pinning every name with an even record count to a + * fixed starting address. (The same failure mode once collapsed {@code SniEndPoint}'s rotation, + * when SSL engine setup shared its counter.) + * + *

Entries are never evicted: the keys are the distinct names the driver ever expands -- + * contact points, the SNI proxy name, client-route hostnames -- each holding a single {@code + * AtomicInteger}, so growth is bounded by configuration and topology in practice. */ - private static final AtomicInteger ROTATION_OFFSET = new AtomicInteger(); + private static final ConcurrentHashMap ROTATION_OFFSETS = + new ConcurrentHashMap<>(); + + /** Fallback rotation counter for the odd original address that is not name-based. */ + private static final AtomicInteger FALLBACK_ROTATION_OFFSET = new AtomicInteger(); /** either set from the configuration, or null and will be negotiated */ @VisibleForTesting volatile ProtocolVersion protocolVersion; @@ -408,7 +424,7 @@ private CompletionStage> resolveCandidates( "Resolver returned no address for " + address)); return; } - result.complete(rotate(reattachHostnames(address, addresses))); + result.complete(rotate(address, reattachHostnames(address, addresses))); } catch (Throwable t) { result.completeExceptionally(t); } @@ -500,17 +516,21 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand *

The list is sorted first so that a given rotation offset maps to the same address on every * call, regardless of the order the resolver happened to return. Only that determinism matters * here, not the ordering itself, so a plain string comparison is enough. + * + *

The offset is tracked per name -- {@code original} is the address the expansion was queried + * for -- so that different names rotate independently (see {@link #ROTATION_OFFSETS}). */ @VisibleForTesting - static List rotate(List addresses) { + static List rotate( + SocketAddress original, List addresses) { int size = addresses.size(); if (size == 1) { - // Nothing to rotate, and don't burn a rotation offset on it. + // Nothing to rotate, and don't burn a rotation offset (or create a counter) for it. return new ArrayList<>(addresses); } List sorted = new ArrayList<>(addresses); sorted.sort(Comparator.comparing(SocketAddress::toString)); - int start = Math.floorMod(ROTATION_OFFSET.getAndIncrement(), size); + int start = Math.floorMod(rotationOffsetFor(original).getAndIncrement(), size); List result = new ArrayList<>(size); for (int i = 0; i < size; i++) { result.add(sorted.get((start + i) % size)); @@ -518,6 +538,15 @@ static List rotate(List addresses) { return result; } + private static AtomicInteger rotationOffsetFor(SocketAddress original) { + if (original instanceof InetSocketAddress) { + // DNS names are case-insensitive; normalize so the same name shares one counter. + String name = ((InetSocketAddress) original).getHostString().toLowerCase(Locale.ROOT); + return ROTATION_OFFSETS.computeIfAbsent(name, k -> new AtomicInteger()); + } + return FALLBACK_ROTATION_OFFSET; + } + /** * Iterates through the candidate addresses produced by {@link #resolveCandidates}. Tries each one * in sequence; when an address fails, the next candidate is tried, and only when all candidates diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 390df274604..2fee70fa6f2 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -90,13 +90,13 @@ public void should_fail_with_suppressed_causes_when_all_addresses_are_unreachabl @Test public void should_rotate_the_starting_address_across_successive_expansions() { // Every address must still be offered -- a single attempt has to be able to fall back across - // all - // of them -- but the one tried *first* has to move, otherwise every connection piles onto the - // resolver's first record and a multi-record name buys no spreading at all. + // all of them -- but the one tried *first* has to move, otherwise every connection piles onto + // the resolver's first record and a multi-record name buys no spreading at all. + InetSocketAddress name = InetSocketAddress.createUnresolved("successive.rotate.fake", 9042); List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2); - List first = ChannelFactory.rotate(addresses); - List second = ChannelFactory.rotate(addresses); + List first = ChannelFactory.rotate(name, addresses); + List second = ChannelFactory.rotate(name, addresses); assertThat(first).containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2); assertThat(second).containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2); @@ -107,9 +107,46 @@ public void should_rotate_the_starting_address_across_successive_expansions() { @Test public void should_leave_a_single_address_alone() { - // Nothing to spread, and the rotation offset must not advance for it either. - assertThat(ChannelFactory.rotate(Collections.singletonList(UNREACHABLE_1))) + // A name fresh to this test: the rotation counters are static, so a name shared with other + // tests could have been advanced before we get here. + InetSocketAddress name = InetSocketAddress.createUnresolved("single.rotate.fake", 9042); + + // Nothing to spread... + assertThat(ChannelFactory.rotate(name, Collections.singletonList(UNREACHABLE_1))) .containsExactly(UNREACHABLE_1); + + // ...and no rotation offset may be burned for it either: the name's first multi-address + // expansion still starts at the toString-sorted first element ("...-1" sorts before "...-2"). + List next = + ChannelFactory.rotate(name, Arrays.asList(UNREACHABLE_2, UNREACHABLE_1)); + assertThat(next.get(0)).isEqualTo(UNREACHABLE_1); + } + + @Test + public void should_rotate_names_independently() { + // Names fresh to this test: the rotation counters are static and shared across the JVM. + InetSocketAddress nameA = InetSocketAddress.createUnresolved("a.independent.fake", 9042); + InetSocketAddress nameB = InetSocketAddress.createUnresolved("b.independent.fake", 9042); + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2); + + // Interleave the two names the way two hostname contact points are expanded in sequence on + // every reconnection round. With one global counter each name would only ever see one offset + // parity, pinning both to a fixed starting address forever. + List a1 = ChannelFactory.rotate(nameA, addresses); + List b1 = ChannelFactory.rotate(nameB, addresses); + List a2 = ChannelFactory.rotate(nameA, addresses); + List b2 = ChannelFactory.rotate(nameB, addresses); + + // Each name still rotates on its own... + assertThat(a2.get(0)) + .as("name A must rotate despite interleaved expansions of name B") + .isNotEqualTo(a1.get(0)); + assertThat(b2.get(0)) + .as("name B must rotate despite interleaved expansions of name A") + .isNotEqualTo(b1.get(0)); + // ...and is not perturbed by the other: both fresh names start at the same (sorted-first) + // element instead of B starting wherever A's expansions left a shared counter. + assertThat(b1.get(0)).isEqualTo(a1.get(0)); } // ---- reattachHostname() --------------------------------------------------- From 426d9508c6ef9530ab3650b188945fb715f0ea87 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 16:01:23 +0200 Subject: [PATCH 24/33] docs: pin down the afterBootstrapInitialized contract and warn on stray handlers (DRIVER-201) Moving name resolution into ChannelFactory changed the hook's contract in two ways that were previously implicit: - it now runs once per logical connection to a node, instead of once per attempt (which included protocol-version downgrade retries) -- the per-address attempts and downgrade retries share the bootstrap through clone(EventLoop); - it receives the bootstrap before the driver installs its channel handler, and a handler set by the hook is replaced by the driver's own on each per-attempt copy. Previously the hook ran after .handler(...), so replacing the driver's handler was technically possible, though never a supported extension point. Spell both out in the NettyOptions.afterBootstrapInitialized() javadoc (options, attributes and Bootstrap.resolver() are what the hook is for; pipeline customization belongs in afterChannelInitialized()), log a one-time warning when the hook is detected installing a handler -- following the LOGGED_ORPHAN_WARNING pattern -- and document the change in the upgrade guide. The new test installs a dummy handler from the hook and asserts the connection still completes its protocol handshake, proving the driver's handler is the one that ends up on the channel. Co-Authored-By: Claude Fable 5 --- .../internal/core/channel/ChannelFactory.java | 8 +++ .../internal/core/context/NettyOptions.java | 16 ++++- .../ChannelFactoryBootstrapHookTest.java | 71 +++++++++++++++++++ upgrade_guide/README.md | 7 ++ 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 0342b21ae73..876aa3fc682 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -97,6 +97,7 @@ public class ChannelFactory { private static final String DATASTAX_CLOUD_PRODUCT_TYPE = "DATASTAX_APOLLO"; private static final AtomicBoolean LOGGED_ORPHAN_WARNING = new AtomicBoolean(); + private static final AtomicBoolean LOGGED_HANDLER_WARNING = new AtomicBoolean(); /** * A value for {@link #productType} that indicates that the server does not report any product @@ -337,6 +338,13 @@ private Bootstrap newBootstrap() { .channel(nettyOptions.channelClass()) .option(ChannelOption.ALLOCATOR, nettyOptions.allocator()); nettyOptions.afterBootstrapInitialized(bootstrap); + if (bootstrap.config().handler() != null && LOGGED_HANDLER_WARNING.compareAndSet(false, true)) { + LOG.warn( + "[{}] NettyOptions.afterBootstrapInitialized() installed a channel handler on the" + + " bootstrap; it will be replaced by the driver's own handler. Use" + + " NettyOptions.afterChannelInitialized() to customize the pipeline instead.", + logPrefix); + } return bootstrap; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java index 5b4ff4dcec8..b319186910d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/NettyOptions.java @@ -66,7 +66,21 @@ public interface NettyOptions { /** * A hook invoked each time the driver creates a client bootstrap in order to open a channel. This - * is a good place to configure any custom option on the bootstrap. + * is a good place to configure any custom option, attribute, or {@link + * Bootstrap#resolver(io.netty.resolver.AddressResolverGroup)} on the bootstrap. + * + *

The hook runs once per logical connection to a node. When a hostname expands to several IP + * addresses, the same bootstrap is shared by every per-address attempt (each attempt uses a + * {@link Bootstrap#clone(io.netty.channel.EventLoopGroup)} of it); likewise, protocol-version + * downgrade retries reuse it. Before multi-address support the hook ran once per attempt, + * including once per downgrade retry. + * + *

The bootstrap does not carry the driver's channel handler yet, and a handler + * installed by this hook is not honoured: the driver sets its own handler on each + * per-attempt copy afterwards (and logs a one-time warning if it overwrites one). To customize + * the pipeline, use {@link #afterChannelInitialized(Channel)} instead. (Before multi-address + * support the hook ran after the driver's handler was installed, so replacing it was technically + * possible; that was never a supported extension point.) */ void afterBootstrapInitialized(Bootstrap bootstrap); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java new file mode 100644 index 00000000000..dbe60ae4432 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryBootstrapHookTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.channel; + +import static com.datastax.oss.driver.Assertions.assertThatStage; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.DefaultProtocolVersion; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.internal.core.context.NettyOptions; +import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelInboundHandlerAdapter; +import java.util.concurrent.CompletionStage; +import org.junit.Test; + +/** + * Verifies the {@link NettyOptions#afterBootstrapInitialized(Bootstrap)} contract: the hook runs on + * a handler-less bootstrap, and a handler it installs is replaced by the driver's own. + */ +public class ChannelFactoryBootstrapHookTest extends ChannelFactoryTestBase { + + @Test + public void should_replace_handler_installed_by_bootstrap_hook() { + // Given – a hook that (incorrectly) installs its own channel handler. The driver sets its own + // handler on each per-attempt copy afterwards, logging a one-time warning; if the dummy + // handler below survived instead, the protocol handshake would never happen and this connect + // would fail on the init timeout. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + doAnswer( + invocation -> { + Bootstrap bootstrap = invocation.getArgument(0); + bootstrap.handler(new ChannelInboundHandlerAdapter()); + return null; + }) + .when(nettyOptions) + .afterBootstrapInitialized(any(Bootstrap.class)); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + SERVER_ADDRESS, + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then + assertThatStage(channelFuture).isSuccess(); + } +} diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index 53cbb2e2f5b..3e016fbb5fe 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -56,6 +56,13 @@ As part of this change: hold an already-resolved endpoint that is never re-resolved, so on control-connection reconnect the driver falls back to the original contact points to pick up current DNS records once the live-node plan is exhausted. Set it to `false` to restore the previous behavior. +- For advanced deployments that provide a custom `NettyOptions`: the + `afterBootstrapInitialized()` hook now runs once per logical connection to a node (previously + once per attempt, including protocol-version downgrade retries), and it receives the bootstrap + *before* the driver installs its channel handler — a handler set by the hook is replaced, and + the driver logs a one-time warning if it detects one. Use the hook for channel options, + attributes and `Bootstrap.resolver(...)`; use `afterChannelInitialized()` for pipeline + customization. ### 4.19.0.7 From f3b0703a4d71cd45aff57f3ea57f23b16fd5b9d5 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 16:02:34 +0200 Subject: [PATCH 25/33] test: drop the stale NETTY_DAEMON stub from ChannelFactoryTestBase (DRIVER-201) The stub and its comment referred to ChannelFactory's name-resolver thread pool, which was removed when resolution moved to Netty's AddressResolverGroup; the factory no longer reads advanced.netty.daemon at all (only DefaultNettyOptions does, and these tests mock NettyOptions). Harmless today only because the base class uses lenient initMocks(), but a misleading breadcrumb for the next reader. Co-Authored-By: Claude Fable 5 --- .../driver/internal/core/channel/ChannelFactoryTestBase.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java index d77d235a141..12ab96c59b9 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryTestBase.java @@ -132,9 +132,6 @@ public void setup() throws InterruptedException { .thenReturn(Duration.ofSeconds(30)); when(defaultProfile.getDuration(DefaultDriverOption.CONNECTION_CONNECT_TIMEOUT)) .thenReturn(Duration.ofSeconds(5)); - // The factory's name-resolver threads follow this setting; daemon here so a test that does not - // close the factory cannot hold the surefire JVM open. - when(defaultProfile.getBoolean(DefaultDriverOption.NETTY_DAEMON)).thenReturn(true); when(context.getProtocolVersionRegistry()).thenReturn(protocolVersionRegistry); when(context.getNettyOptions()).thenReturn(nettyOptions); From 00e2785b4c63204892b1e2a4bafd4066a872070a Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Fri, 31 Jul 2026 16:05:55 +0200 Subject: [PATCH 26/33] fix: align pinTo type checks and no-op shortcuts across endpoint implementations (DRIVER-201) ClientRoutesEndPoint accepted and stored any SocketAddress as its pin, while DefaultEndPoint and SniEndPoint reject non-InetSocketAddress pins; downstream readers of a pinned endpoint's resolve() (the GSSAPI authenticator's cast, DefaultTopologyMonitor's instanceof guards) expect Inet addresses. Tighten the field and guard to match the siblings: a non-Inet address skips pinning instead of being stored. SniEndPoint gains DefaultEndPoint's remaining shortcut: pinning to the very address the endpoint already holds returns the same instance, sparing the copy and its redundant "proxy(proxy)" toString suffix. Only reachable when the proxy address was supplied already resolved -- Cloud supplies a hostname, for which a resolved pin never compares equal. (The earlier reason for skipping this shortcut -- that SniEndPoint's unpinned resolve() used to look the proxy up lazily, making even a same-address pin meaningful -- no longer holds now that resolve() is a field read.) The stale toString() comment claiming channels always carry a pinned copy is updated to match. Co-Authored-By: Claude Fable 5 --- .../core/metadata/ClientRoutesEndPoint.java | 15 +++++++++---- .../internal/core/metadata/SniEndPoint.java | 14 ++++++++---- .../metadata/ClientRoutesEndPointTest.java | 22 +++++++++++++++++++ .../core/metadata/SniEndPointTest.java | 12 ++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java index 571c5146702..4d495ec5303 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -39,7 +39,7 @@ public class ClientRoutesEndPoint implements PinnableEndPoint { * null} if it is not pinned. Deliberately excluded from {@link #equals} and {@link #hashCode}, * which key off the host id alone. */ - @Nullable private final SocketAddress pinnedAddress; + @Nullable private final InetSocketAddress pinnedAddress; /** * @param topologyMonitor the topology monitor used to resolve the endpoint address on demand. @@ -64,7 +64,7 @@ private ClientRoutesEndPoint( @NonNull UUID hostId, @Nullable InetAddress broadcastInetAddress, @NonNull EndPoint fallbackEndPoint, - @Nullable SocketAddress pinnedAddress) { + @Nullable InetSocketAddress pinnedAddress) { this.topologyMonitor = Objects.requireNonNull(topologyMonitor, "Topology monitor cannot be null"); this.hostId = Objects.requireNonNull(hostId, "HOST uuid cannot be null"); @@ -110,11 +110,18 @@ public SocketAddress resolve() { @Override public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); - if (resolvedAddress.equals(this.pinnedAddress)) { + // Mirror DefaultEndPoint: an address we cannot hold in an InetSocketAddress field skips + // pinning rather than failing the connection. + if (!(resolvedAddress instanceof InetSocketAddress) + || resolvedAddress.equals(this.pinnedAddress)) { return this; } return new ClientRoutesEndPoint( - topologyMonitor, hostId, broadcastInetAddress, fallbackEndPoint, resolvedAddress); + topologyMonitor, + hostId, + broadcastInetAddress, + fallbackEndPoint, + (InetSocketAddress) resolvedAddress); } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index b70de4498e0..a9cb6c8f422 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -86,7 +86,13 @@ public InetSocketAddress resolve() { public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { Objects.requireNonNull(resolvedAddress, "resolvedAddress cannot be null"); if (!(resolvedAddress instanceof InetSocketAddress) - || resolvedAddress.equals(this.pinnedAddress)) { + || resolvedAddress.equals(this.pinnedAddress) + // The address we already hold: pinning to it changes nothing -- resolve() and toString() + // keep yielding what they already do -- so spare the copy (and its redundant + // "proxy(proxy)" toString suffix). Only reachable when the proxy address was supplied + // already resolved: for the usual unresolved Cloud proxy hostname, a resolved pin never + // compares equal to it. + || resolvedAddress.equals(this.proxyAddress)) { return this; } return new SniEndPoint(proxyAddress, serverName, (InetSocketAddress) resolvedAddress); @@ -112,9 +118,9 @@ public int hashCode() { @Override public String toString() { // An unpinned endpoint prints the original proxy address, so with multiple A-records it does - // not - // say which one a given connection selected. A pinned copy does -- and that is what channels - // carry, so connection-level logs identify the actual proxy IP. + // not say which one a given connection selected. A pinned copy does -- and channels carry a + // pinned copy whenever the connected IP differs from the stored proxy address (see pinTo()) -- + // so connection-level logs identify the actual proxy IP. return pinnedAddress == null ? proxyAddress + ":" + serverName : proxyAddress + "(" + pinnedAddress + "):" + serverName; diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java index a6374d73a36..296390c53e7 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPointTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import io.netty.channel.local.LocalAddress; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -120,6 +121,27 @@ public void pin_to_should_stop_consulting_the_topology_monitor() { assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); } + @Test + public void pin_to_should_return_same_instance_when_already_pinned_to_that_address() { + ClientRoutesEndPoint original = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + InetSocketAddress pinnedTo = new InetSocketAddress("127.0.0.1", 9042); + + EndPoint pinned = original.pinTo(pinnedTo); + + assertThat(((ClientRoutesEndPoint) pinned).pinTo(pinnedTo)).isSameAs(pinned); + } + + @Test + public void pin_to_should_be_a_no_op_for_a_non_inet_address() { + // Mirror DefaultEndPoint: an address that cannot be held in an InetSocketAddress field (e.g. + // the local transport used by unit tests) skips pinning rather than failing the connection. + ClientRoutesEndPoint endPoint = + new ClientRoutesEndPoint(topologyMonitor, UUID.randomUUID(), null, fallbackEndPoint); + + assertThat(endPoint.pinTo(new LocalAddress("some-id"))).isSameAs(endPoint); + } + // ---- equals / hashCode -------------------------------------------------- @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java index 28da6e59ddd..5213222cfe4 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -85,4 +85,16 @@ public void pin_to_should_return_same_instance_when_already_pinned_to_that_addre assertThat(pinned.pinTo(pinnedTo)).isSameAs(pinned); } + + @Test + public void pin_to_should_return_same_instance_when_address_is_the_proxy_address_itself() { + // Only reachable when the proxy address was supplied already resolved (Cloud supplies a + // hostname): pinning to the very address the endpoint holds is a no-op, so the copy -- and its + // redundant "proxy(proxy)" toString suffix -- is spared. + InetSocketAddress resolvedProxy = new InetSocketAddress("127.0.0.1", 9042); + SniEndPoint endPoint = new SniEndPoint(resolvedProxy, "test-server-name"); + + assertThat(endPoint.pinTo(new InetSocketAddress("127.0.0.1", 9042))).isSameAs(endPoint); + assertThat(endPoint.toString()).doesNotContain("("); + } } From 1b3aea181255df880693d12571a7d504accdf602 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Mon, 3 Aug 2026 13:47:17 +0200 Subject: [PATCH 27/33] fix: keep trying addresses of an unidentified endpoint after a version rejection (DRIVER-201) tryNextCandidate() treated every UnsupportedProtocolVersionException as terminal. That is right for a node we have already identified -- all of its addresses are that same node, so replaying the negotiation ladder against each one buys nothing -- but wrong for a contact point: the addresses one name expands to may belong to different nodes, and a rejection by the first says nothing about the rest. It was also a regression. With advanced.resolve-contact-points = true each resolved address used to be a separate Node, and ControlConnection.SingleThreaded.connect() advances to the next node in its query plan on any error, this one included. Collapsing a name into a single Node moved that responsibility into the candidate loop, so the loop has to honour it. Thread node identity down from the connect(Node, ...) entry points: Node.getHostId() is null only for an initial contact point, until host ids have been read from system.local and system.peers for the first time, which is exactly the "we do not know which node this is" case. The shortcut now applies only to identified nodes. The @VisibleForTesting connect(EndPoint, ...) overload keeps its signature and passes "unidentified", since a bare endpoint carries no host id either; a new overload takes the flag explicitly. The existing terminal-shortcut test now drives the identified path, and a mirror test covers the unidentified one: the second candidate is tried, replays the ladder from the top, and the propagated UnsupportedProtocolVersionException carries the first candidate's failure as a suppressed exception. The negotiation-ladder mocking and the server-side exchange they share are now helpers. Co-Authored-By: Claude Opus 5 (1M context) --- .../internal/core/channel/ChannelFactory.java | 80 ++++++++++-- ...ChannelFactoryProtocolNegotiationTest.java | 119 +++++++++++++----- 2 files changed, 157 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 876aa3fc682..ed573507ab5 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -199,7 +199,7 @@ public CompletionStage connect(Node node, DriverChannelOptions op } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater); + return connect(node.getEndPoint(), null, null, options, nodeMetricUpdater, isIdentified(node)); } public CompletionStage connect( @@ -210,7 +210,24 @@ public CompletionStage connect( } else { nodeMetricUpdater = NoopNodeMetricUpdater.INSTANCE; } - return connect(node.getEndPoint(), node.getShardingInfo(), shardId, options, nodeMetricUpdater); + return connect( + node.getEndPoint(), + node.getShardingInfo(), + shardId, + options, + nodeMetricUpdater, + isIdentified(node)); + } + + /** + * Whether we know which node we are connecting to, as opposed to merely which address to + * try. {@link Node#getHostId()} is null only for an initial contact point, until the driver has + * read host ids from {@code system.local} and {@code system.peers} for the first time; every node + * discovered from those rows has one. {@link #tryNextCandidate} needs the distinction because an + * unidentified contact-point name may expand to addresses of different nodes. + */ + private static boolean isIdentified(Node node) { + return node.getHostId() != null; } @VisibleForTesting @@ -220,6 +237,19 @@ CompletionStage connect( Integer shardId, DriverChannelOptions options, NodeMetricUpdater nodeMetricUpdater) { + // A bare endpoint carries no host id, so this matches the contact-point case (see + // isIdentified()). + return connect(endPoint, shardingInfo, shardId, options, nodeMetricUpdater, false); + } + + @VisibleForTesting + CompletionStage connect( + EndPoint endPoint, + NodeShardingInfo shardingInfo, + Integer shardId, + DriverChannelOptions options, + NodeMetricUpdater nodeMetricUpdater, + boolean nodeIsIdentified) { CompletableFuture resultFuture = new CompletableFuture<>(); ProtocolVersion currentVersion; @@ -240,6 +270,7 @@ CompletionStage connect( nodeMetricUpdater, currentVersion, isNegotiating, + nodeIsIdentified, resultFuture); return resultFuture; } @@ -252,6 +283,7 @@ private void connect( NodeMetricUpdater nodeMetricUpdater, ProtocolVersion currentVersion, boolean isNegotiating, + boolean nodeIsIdentified, CompletableFuture resultFuture) { // Built once per connect() rather than once per candidate: it is the only handle on the Netty @@ -315,6 +347,7 @@ private void connect( nodeMetricUpdater, currentVersion, isNegotiating, + nodeIsIdentified, resultFuture, candidates, 0, @@ -560,15 +593,25 @@ private static AtomicInteger rotationOffsetFor(SocketAddress original) { * in sequence; when an address fails, the next candidate is tried, and only when all candidates * are exhausted is the overall {@code resultFuture} failed. * - *

The one exception is a {@link UnsupportedProtocolVersionException}: a protocol-version - * rejection -- whether negotiation exhausted every downgrade or the server refused a forced - * version -- is a property of the node, not of the address the connection happened to use, so it - * fails the attempt immediately instead of replaying the whole negotiation ladder against every - * remaining IP (worst case {@code N × versions × connect-timeout} for nothing). This matches the - * pre-multi-address behaviour of a single-address connect. The corner it deliberately does not - * rescue: a heterogeneous rolling upgrade where different IPs behind one name genuinely support - * different protocol versions. Other failures -- TCP, init, authentication -- do advance to the - * next candidate, since with a multi-record name they may well be address-specific. + *

The one exception is a {@link UnsupportedProtocolVersionException} against an + * identified node ({@code nodeIsIdentified}, see {@link #isIdentified(Node)}): a + * protocol-version rejection -- whether negotiation exhausted every downgrade or the server + * refused a forced version -- is a property of the node, and every address of a node we have + * already identified is that same node, so the attempt fails immediately instead of replaying the + * whole negotiation ladder against every remaining IP (worst case {@code N × versions × + * connect-timeout} for nothing). This matches the pre-multi-address behaviour of a single-address + * connect. The corner it deliberately does not rescue: a heterogeneous rolling upgrade where + * different IPs of one identified node genuinely support different protocol versions. + * + *

For an unidentified endpoint -- a contact point, before the driver has read host ids + * -- the addresses a name expands to may well belong to different nodes, so a rejection by the + * first of them says nothing about the rest and the loop keeps going. That also preserves the + * behaviour this PR would otherwise have removed: with {@code advanced.resolve-contact-points = + * true} each resolved address used to be a separate {@code Node}, and {@code ControlConnection} + * advances to the next node in its query plan on any error, including this one. + * + *

Other failures -- TCP, init, authentication -- always advance to the next candidate, since + * with a multi-record name they may well be address-specific. * *

Timeout note: addresses are tried serially, so the worst-case time before failure is * {@code N × connect-timeout} where N is the number of candidates. This is an intentional @@ -590,6 +633,7 @@ private void tryNextCandidate( NodeMetricUpdater nodeMetricUpdater, ProtocolVersion currentVersion, boolean isNegotiating, + boolean nodeIsIdentified, CompletableFuture resultFuture, List candidates, int index, @@ -630,7 +674,7 @@ private void tryNextCandidate( try { if (error == null) { resultFuture.complete(channel); - } else if (!(error instanceof UnsupportedProtocolVersionException) + } else if (!isNodeWideFailure(error, nodeIsIdentified) && index + 1 < candidates.size()) { LOG.debug( "[{}] Failed to connect to {} ({}), trying next address", @@ -650,13 +694,14 @@ private void tryNextCandidate( nodeMetricUpdater, currentVersion, isNegotiating, + nodeIsIdentified, resultFuture, candidates, index + 1, priorErrors); } else { if (index + 1 < candidates.size()) { - // Only reachable for an UnsupportedProtocolVersionException (see the javadoc). + // Only reachable for a node-wide failure (see the javadoc). LOG.debug( "[{}] Not trying the remaining addresses of {}: a protocol-version rejection" + " is a property of the node, not of the address ({})", @@ -683,6 +728,15 @@ private void tryNextCandidate( } } + /** + * Whether {@code error} dooms every remaining address of the endpoint, making it pointless for + * {@link #tryNextCandidate} to try them. See its javadoc for why this is limited to a + * protocol-version rejection against an identified node. + */ + private static boolean isNodeWideFailure(Throwable error, boolean nodeIsIdentified) { + return nodeIsIdentified && error instanceof UnsupportedProtocolVersionException; + } + /** * Performs a Netty bootstrap connect to a single, already-resolved address. Handles * protocol-version negotiation (downgrade retries) internally, staying on the same address. Uses diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java index d56d3b04c38..6e868e5cc0b 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryProtocolNegotiationTest.java @@ -285,14 +285,11 @@ public void should_fail_if_negotiation_finds_no_matching_version(int errorCode) } @Test - public void should_not_try_next_address_when_negotiation_exhausts_versions() { - // Given – a name that expands to two candidates (the same live server twice, so whichever the - // rotation picks first is irrelevant), and a server that rejects every protocol version. - when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); - when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); - when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)) - .thenReturn(Optional.of(DefaultProtocolVersion.V3)); - when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V3)).thenReturn(Optional.empty()); + public void should_not_try_next_address_of_identified_node_when_negotiation_exhausts_versions() { + // Given – an *identified* node (its host id is known, so every address its name expands to is + // that same node) whose name expands to two candidates: the same live server twice, so + // whichever the rotation picks first is irrelevant. The server rejects every protocol version. + mockNegotiationLadderDownToV3(); SocketAddress serverAddress = SERVER_ADDRESS.resolve(); installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); ChannelFactory factory = newChannelFactory(); @@ -304,9 +301,92 @@ public void should_not_try_next_address_when_negotiation_exhausts_versions() { null, null, DriverChannelOptions.DEFAULT, - NoopNodeMetricUpdater.INSTANCE); + NoopNodeMetricUpdater.INSTANCE, + true); + + exhaustNegotiationLadder(); + + // Then – the second candidate must not be attempted: for a node we have already identified, a + // protocol-version rejection is a property of the node, not of the address, so replaying the + // negotiation ladder against the remaining IPs would buy nothing. Checked before the future + // assertion so that on regression the stray frame is drained; leaving it unread would block the + // server's exchanger and hang the whole suite in tearDown() instead of failing this test. + assertThat(tryReadOutboundFrame(200)) + .as("second candidate must not be attempted after negotiation exhaustion") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("no other candidate should have been tried, so nothing to suppress") + .isEmpty(); + }); + } + + @Test + public void + should_try_next_address_of_unidentified_endpoint_when_negotiation_exhausts_versions() { + // Given – the same setup, but for an endpoint the driver has not identified yet: a contact + // point, before host ids have been read. Its name may well expand to addresses of *different* + // nodes, so a version rejection by the first says nothing about the second. + mockNegotiationLadderDownToV3(); + SocketAddress serverAddress = SERVER_ADDRESS.resolve(); + installResolver(new TestAddressResolverGroup(Arrays.asList(serverAddress, serverAddress))); + ChannelFactory factory = newChannelFactory(); + + // When + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(InetSocketAddress.createUnresolved("test.cluster.fake", 9042)), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE, + false); + + // The first candidate exhausts the ladder... + exhaustNegotiationLadder(); + // ...and the second is tried all the same, replaying the ladder from the top. Before this, + // resolve-contact-points=true made each address a separate node and ControlConnection advanced + // to the next one on exactly this error; collapsing a name into one node must not lose that. + exhaustNegotiationLadder(); - // First candidate: V4 rejected, downgrade retry with V3 rejected -> negotiation exhausted + // Then + assertThat(tryReadOutboundFrame(200)) + .as("the name expands to two addresses, so there is no third attempt") + .isNull(); + assertThatStage(channelFuture) + .isFailed( + e -> { + assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); + assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) + .as("each candidate negotiates on its own, so this is the last one's ladder") + .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); + assertThat(e.getSuppressed()) + .as("the first candidate's failure must still be reported") + .hasSize(1); + assertThat(e.getSuppressed()[0]) + .isInstanceOf(UnsupportedProtocolVersionException.class); + }); + } + + /** Negotiation starts at V4 and has exactly one downgrade available, to V3. */ + private void mockNegotiationLadderDownToV3() { + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V4)) + .thenReturn(Optional.of(DefaultProtocolVersion.V3)); + when(protocolVersionRegistry.downgrade(DefaultProtocolVersion.V3)).thenReturn(Optional.empty()); + } + + /** + * Plays the server side of a full negotiation ladder against one candidate address: V4 rejected, + * downgrade retry with V3 rejected, i.e. no version left to try on that address. + */ + private void exhaustNegotiationLadder() { Frame requestFrame = readOutboundFrame(); assertThat(requestFrame.message).isInstanceOf(Options.class); writeInboundFrame(requestFrame, TestResponses.supportedResponse("mock_key", "mock_value")); @@ -328,25 +408,6 @@ public void should_not_try_next_address_when_negotiation_exhausts_versions() { requestFrame, new Error( ProtocolConstants.ErrorCode.PROTOCOL_ERROR, "Invalid or unsupported protocol version")); - - // Then – the second candidate must not be attempted: a protocol-version rejection is a - // property of the node, not of the address, so replaying the negotiation ladder against the - // remaining IPs would buy nothing. Checked before the future assertion so that on regression - // the stray frame is drained; leaving it unread would block the server's exchanger and hang - // the whole suite in tearDown() instead of failing this test. - assertThat(tryReadOutboundFrame(200)) - .as("second candidate must not be attempted after negotiation exhaustion") - .isNull(); - assertThatStage(channelFuture) - .isFailed( - e -> { - assertThat(e).isInstanceOf(UnsupportedProtocolVersionException.class); - assertThat(((UnsupportedProtocolVersionException) e).getAttemptedVersions()) - .containsExactly(DefaultProtocolVersion.V4, DefaultProtocolVersion.V3); - assertThat(e.getSuppressed()) - .as("no other candidate should have been tried, so nothing to suppress") - .isEmpty(); - }); } @Test From f1abf32287ed17e4e33a3093c3c23ed144022c8c Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Mon, 3 Aug 2026 13:49:05 +0200 Subject: [PATCH 28/33] fix: always preserve the queried hostname on resolver-returned addresses (DRIVER-201) reattachHostname() only re-attached the queried name to a candidate that carried no name of its own, deferring to a resolver that labelled its results with a canonical or CNAME name. But that label is not cosmetic: the candidate is pinned onto the channel endpoint, and DefaultSslEngineFactory / SniSslEngineFactory derive the SSL peer host from it inside the channel initializer. So the resolver's label became the name TLS hostname verification checked the server certificate against -- a name the operator never configured. Before multi-address support the initializer kept the original endpoint and Netty resolved only the TCP destination, so the configured name was always the one validated. Make the queried name win unconditionally. The bail-out is now "the candidate already carries the queried name", which is the common case (the JDK and Netty-DNS resolvers attach it themselves) and keeps the no-op cheap; the scoped-IPv6 exception stays, since a rebuild would drop the scope id. Uniform names across an expansion also make rotate()'s toString() sort depend only on the IP and port, so ordering gets more deterministic, not less. The resolver-name test now asserts the queried name replaces the CNAME label, and a new test covers the already-has-the-name pass-through that used to be implied by it. Co-Authored-By: Claude Opus 5 (1M context) --- .../internal/core/channel/ChannelFactory.java | 38 +++++++++++-------- .../ChannelFactoryMultiAddressTest.java | 25 ++++++++++-- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index ed573507ab5..5d065ef4f94 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -496,24 +496,29 @@ private static List reattachHostnames( } /** - * Re-attaches the {@code original} (unresolved) address's host name to a resolved candidate that - * carries no name of its own. + * Re-attaches the {@code original} (unresolved) address's host name to one of the resolved + * candidates it expanded to, whatever name that candidate carries. * *

The JDK and Netty-DNS resolvers already attach the queried name to the {@link InetAddress}es * they return, so this is a no-op for them. A custom resolver, however, may build its results - * from raw address bytes. The channel's pinned endpoint is built from the candidate (see {@link - * PinnableEndPoint}), and it is what {@code DefaultSslEngineFactory} and {@code - * SniSslEngineFactory} derive the SSL peer host from, inside the channel initializer: with a - * nameless address, {@code InetSocketAddress#getHostName()} triggers a blocking reverse-DNS - * lookup on the event loop, and TLS hostname validation ends up checking the certificate against - * the IP or the PTR record instead of the name the user configured. + * from raw address bytes, or label them with a canonical/CNAME name of its own. The channel's + * pinned endpoint is built from the candidate (see {@link PinnableEndPoint}), and it is what + * {@code DefaultSslEngineFactory} and {@code SniSslEngineFactory} derive the SSL peer host from, + * inside the channel initializer. So whatever name the candidate carries is the name TLS hostname + * verification checks the server certificate against, and the only name that may be is the one + * the user configured: with a nameless address, {@code InetSocketAddress#getHostName()} + * additionally triggers a blocking reverse-DNS lookup on the event loop and validation falls back + * to the IP or the PTR record, and with a resolver-supplied label it validates a name the + * operator never chose. Hence the queried name always wins here; before multi-address support the + * initializer kept the original endpoint and Netty resolved only the TCP destination, which had + * the same effect. * *

Re-attaching changes nothing else: {@code InetAddress.getByAddress(host, bytes)} performs no - * lookup, the TCP connect target is the same IP, a resolved {@link InetSocketAddress}'s equality - * ignores host names (so pinning and pin-equality shortcuts are unaffected), and {@link #rotate} - * stays deterministic because the transform is applied identically on every expansion. A - * candidate that already carries a real name (e.g. from a resolver that returns CNAME targets) is - * respected, and a scoped IPv6 address is left alone, since rebuilding it would drop the scope. + * lookup, the TCP connect target is the same IP, and a resolved {@link InetSocketAddress}'s + * equality ignores host names, so pinning and the pin-equality shortcuts are unaffected. It also + * makes {@link #rotate} more deterministic, not less: with one uniform name across an expansion, + * its {@code toString()} sort depends only on the IP and port. The single exception is a scoped + * IPv6 address, left alone because rebuilding it would drop the scope. */ @VisibleForTesting static SocketAddress reattachHostname(SocketAddress original, SocketAddress candidate) { @@ -525,9 +530,10 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand InetAddress candidateIp = candidateInet.getAddress(); if (!originalInet.isUnresolved() || candidateIp == null - // The candidate already carries a real name: respect the resolver's choice. (For a - // nameless address, getHostString() falls back to the IP literal; no lookup either way.) - || !candidateInet.getHostString().equals(candidateIp.getHostAddress()) + // Nothing to change: the candidate already carries the queried name, which is the common + // case (the JDK and Netty-DNS resolvers attach it themselves). getHostString() never looks + // anything up -- for a nameless address it falls back to the IP literal. + || candidateInet.getHostString().equals(originalInet.getHostString()) // Rebuilding a scoped IPv6 address would silently drop its scope. || (candidateIp instanceof Inet6Address && (((Inet6Address) candidateIp).getScopeId() != 0 diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 2fee70fa6f2..6a49ba63f34 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -176,14 +176,31 @@ public void should_reattach_queried_hostname_to_nameless_resolved_address() thro } @Test - public void should_keep_resolver_provided_hostname() throws Exception { - // A resolver that attaches its own name (e.g. a CNAME target) wins over the queried one -- - // and this is also why the transform is a no-op for the JDK and Netty-DNS resolvers, which - // attach the queried name themselves. + public void should_override_resolver_provided_hostname_with_queried_name() throws Exception { + // A resolver may label its results with a canonical/CNAME name of its own. That name would end + // up on the pinned endpoint and hence be the one TLS hostname verification checks the server + // certificate against, so the name the user configured has to win over it. InetSocketAddress candidate = new InetSocketAddress( InetAddress.getByAddress("cname.example.fake", new byte[] {10, 0, 0, 1}), 9042); + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_pass_candidate_through_when_it_already_carries_the_queried_name() + throws Exception { + // The common case: the JDK and Netty-DNS resolvers attach the queried name themselves, so + // there is nothing to rebuild. + InetSocketAddress candidate = + new InetSocketAddress( + InetAddress.getByAddress("test.cluster.fake", new byte[] {10, 0, 0, 1}), 9042); + assertThat(ChannelFactory.reattachHostname(HOSTNAME, candidate)).isSameAs(candidate); } From 0f6d91af9f703356c0a99e8d4e6282a45f767902 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Mon, 3 Aug 2026 13:54:40 +0200 Subject: [PATCH 29/33] fix: scope the address rotation counters to the session and bound them (DRIVER-201) The per-name rotation offsets lived in a static map, so every multi-address name the driver ever expanded stayed in it for the lifetime of the JVM. Its keys are not bounded by the current configuration or topology: client routes can hand out different hostnames on every refresh, topology churns, and successive sessions in the same JVM can use entirely unrelated names. Move the counters onto the ChannelFactory -- one per session, so they go away with it -- and bound them with an evicting cache on top, since the churn within a single long-lived session is unbounded too. Spreading connections only ever matters among the names a session is currently using, so an evicted counter costs that name nothing but a rotation restart. The cache uses the shaded-Guava idiom already used for the codec and prepared-statement caches. rotate() and rotationOffsetFor() become instance methods; the rotation tests now go through a factory, which also makes them self-contained -- they no longer need names unique across the whole class to avoid inheriting another test's offset. Two new tests cover what the change is for: separate factories do not share offsets, and the tracked-name count stays bounded when a session churns through many names. Co-Authored-By: Claude Opus 5 (1M context) --- .../internal/core/channel/ChannelFactory.java | 44 +++++++++------ .../ChannelFactoryMultiAddressTest.java | 54 ++++++++++++++----- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 5d065ef4f94..78116675b58 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -47,6 +47,9 @@ import com.datastax.oss.driver.internal.core.protocol.FrameEncoder; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; +import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; +import com.datastax.oss.driver.shaded.guava.common.cache.CacheLoader; +import com.datastax.oss.driver.shaded.guava.common.cache.LoadingCache; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.datastax.oss.protocol.internal.ProtocolFeatures; import io.netty.bootstrap.Bootstrap; @@ -76,7 +79,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -105,6 +107,13 @@ public class ChannelFactory { */ private static final String UNKNOWN_PRODUCT_TYPE = "UNKNOWN"; + /** + * How many names {@link #rotationOffsets} tracks before it starts evicting. Generous next to the + * handful of names a session actually expands, and an evicted counter only costs that name a + * rotation restart. + */ + @VisibleForTesting static final int MAX_ROTATION_OFFSETS = 256; + // The names of the handlers on the pipeline: public static final String SSL_HANDLER_NAME = "ssl"; public static final String INBOUND_TRAFFIC_METER_NAME = "inboundTrafficMeter"; @@ -124,9 +133,8 @@ public class ChannelFactory { /** * Round-robin counters used by {@link #rotate} to vary which of a name's addresses a connection - * tries first, one counter per name. Static so the rotation spans every endpoint and session in - * the JVM, which is all the spreading it needs to do; {@code SniEndPoint} used to hold an - * equivalent (single) counter of its own, before resolution moved here. + * tries first, one counter per name. {@code SniEndPoint} used to hold an equivalent (single) + * counter of its own, before resolution moved here. * *

Per name rather than one global counter, because names whose expansions interleave in * lockstep -- say two hostname contact points tried in sequence on every reconnection round -- @@ -134,15 +142,20 @@ public class ChannelFactory { * fixed starting address. (The same failure mode once collapsed {@code SniEndPoint}'s rotation, * when SSL engine setup shared its counter.) * - *

Entries are never evicted: the keys are the distinct names the driver ever expands -- - * contact points, the SNI proxy name, client-route hostnames -- each holding a single {@code - * AtomicInteger}, so growth is bounded by configuration and topology in practice. + *

Per factory, i.e. per session, and bounded on top of that: the names that reach here -- + * contact points, the SNI proxy name, client-route hostnames -- are not fixed for the lifetime of + * a JVM, or even of a session, since client routes can hand out different hostnames on every + * refresh. Spreading connections is only ever needed among the names a session is currently + * using, so nothing is lost by letting the rest go. */ - private static final ConcurrentHashMap ROTATION_OFFSETS = - new ConcurrentHashMap<>(); + @VisibleForTesting + final LoadingCache rotationOffsets = + CacheBuilder.newBuilder() + .maximumSize(MAX_ROTATION_OFFSETS) + .build(CacheLoader.from(name -> new AtomicInteger())); /** Fallback rotation counter for the odd original address that is not name-based. */ - private static final AtomicInteger FALLBACK_ROTATION_OFFSET = new AtomicInteger(); + private final AtomicInteger fallbackRotationOffset = new AtomicInteger(); /** either set from the configuration, or null and will be negotiated */ @VisibleForTesting volatile ProtocolVersion protocolVersion; @@ -565,11 +578,10 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand * here, not the ordering itself, so a plain string comparison is enough. * *

The offset is tracked per name -- {@code original} is the address the expansion was queried - * for -- so that different names rotate independently (see {@link #ROTATION_OFFSETS}). + * for -- so that different names rotate independently (see {@link #rotationOffsets}). */ @VisibleForTesting - static List rotate( - SocketAddress original, List addresses) { + List rotate(SocketAddress original, List addresses) { int size = addresses.size(); if (size == 1) { // Nothing to rotate, and don't burn a rotation offset (or create a counter) for it. @@ -585,13 +597,13 @@ static List rotate( return result; } - private static AtomicInteger rotationOffsetFor(SocketAddress original) { + private AtomicInteger rotationOffsetFor(SocketAddress original) { if (original instanceof InetSocketAddress) { // DNS names are case-insensitive; normalize so the same name shares one counter. String name = ((InetSocketAddress) original).getHostString().toLowerCase(Locale.ROOT); - return ROTATION_OFFSETS.computeIfAbsent(name, k -> new AtomicInteger()); + return rotationOffsets.getUnchecked(name); } - return FALLBACK_ROTATION_OFFSET; + return fallbackRotationOffset; } /** diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 6a49ba63f34..64ead82a293 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -94,9 +94,10 @@ public void should_rotate_the_starting_address_across_successive_expansions() { // the resolver's first record and a multi-record name buys no spreading at all. InetSocketAddress name = InetSocketAddress.createUnresolved("successive.rotate.fake", 9042); List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2); + ChannelFactory factory = newChannelFactory(); - List first = ChannelFactory.rotate(name, addresses); - List second = ChannelFactory.rotate(name, addresses); + List first = factory.rotate(name, addresses); + List second = factory.rotate(name, addresses); assertThat(first).containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2); assertThat(second).containsExactlyInAnyOrder(UNREACHABLE_1, UNREACHABLE_2); @@ -107,35 +108,33 @@ public void should_rotate_the_starting_address_across_successive_expansions() { @Test public void should_leave_a_single_address_alone() { - // A name fresh to this test: the rotation counters are static, so a name shared with other - // tests could have been advanced before we get here. InetSocketAddress name = InetSocketAddress.createUnresolved("single.rotate.fake", 9042); + ChannelFactory factory = newChannelFactory(); // Nothing to spread... - assertThat(ChannelFactory.rotate(name, Collections.singletonList(UNREACHABLE_1))) + assertThat(factory.rotate(name, Collections.singletonList(UNREACHABLE_1))) .containsExactly(UNREACHABLE_1); // ...and no rotation offset may be burned for it either: the name's first multi-address // expansion still starts at the toString-sorted first element ("...-1" sorts before "...-2"). - List next = - ChannelFactory.rotate(name, Arrays.asList(UNREACHABLE_2, UNREACHABLE_1)); + List next = factory.rotate(name, Arrays.asList(UNREACHABLE_2, UNREACHABLE_1)); assertThat(next.get(0)).isEqualTo(UNREACHABLE_1); } @Test public void should_rotate_names_independently() { - // Names fresh to this test: the rotation counters are static and shared across the JVM. InetSocketAddress nameA = InetSocketAddress.createUnresolved("a.independent.fake", 9042); InetSocketAddress nameB = InetSocketAddress.createUnresolved("b.independent.fake", 9042); List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2); + ChannelFactory factory = newChannelFactory(); // Interleave the two names the way two hostname contact points are expanded in sequence on // every reconnection round. With one global counter each name would only ever see one offset // parity, pinning both to a fixed starting address forever. - List a1 = ChannelFactory.rotate(nameA, addresses); - List b1 = ChannelFactory.rotate(nameB, addresses); - List a2 = ChannelFactory.rotate(nameA, addresses); - List b2 = ChannelFactory.rotate(nameB, addresses); + List a1 = factory.rotate(nameA, addresses); + List b1 = factory.rotate(nameB, addresses); + List a2 = factory.rotate(nameA, addresses); + List b2 = factory.rotate(nameB, addresses); // Each name still rotates on its own... assertThat(a2.get(0)) @@ -149,6 +148,37 @@ public void should_rotate_names_independently() { assertThat(b1.get(0)).isEqualTo(a1.get(0)); } + @Test + public void should_not_share_rotation_offsets_across_factories() { + // The counters belong to the factory, i.e. to the session: a name an earlier session expanded + // must not leave an offset behind for the next one, and nothing outlives the session. + InetSocketAddress name = InetSocketAddress.createUnresolved("per.session.rotate.fake", 9042); + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2); + + List first = newChannelFactory().rotate(name, addresses); + List other = newChannelFactory().rotate(name, addresses); + + assertThat(other.get(0)) + .as("a fresh factory rotates from its own start, not from where another one left off") + .isEqualTo(first.get(0)); + } + + @Test + public void should_bound_the_number_of_tracked_names() { + // Client routes can hand out different hostnames on every refresh, so even within one session + // the set of names is not bounded by the configuration or the topology. + ChannelFactory factory = newChannelFactory(); + List addresses = Arrays.asList(UNREACHABLE_1, UNREACHABLE_2); + + for (int i = 0; i < ChannelFactory.MAX_ROTATION_OFFSETS * 4; i++) { + factory.rotate(InetSocketAddress.createUnresolved("churn-" + i + ".fake", 9042), addresses); + } + + assertThat(factory.rotationOffsets.size()) + .as("stale names must be evicted rather than retained for the session's lifetime") + .isLessThanOrEqualTo(ChannelFactory.MAX_ROTATION_OFFSETS); + } + // ---- reattachHostname() --------------------------------------------------- @Test From 9d495e89c135990b8e07f344303796521b85d4f6 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Tue, 4 Aug 2026 00:19:11 +0200 Subject: [PATCH 30/33] fix: let the resolver decide whether an address needs resolving (DRIVER-201) resolveCandidates() short-circuited on InetSocketAddress#isUnresolved() before ever obtaining a resolver, so an already-resolved address never reached the configured AddressResolverGroup. Netty does not work that way: Bootstrap#doResolveAndConnect0 asks the resolver -- !isSupported(a) || isResolved(a) -- and both are overridable. A custom resolver may well report an address that already carries an IP as unresolved in order to redirect it, and Netty consulted it either way, so the shortcut silently took that configuration point away for every connect to an already-resolved node, which is to say for almost every connect. The in-loop check that mirrors Netty's was already there; only the pre-check and its helper are gone. The cost is one eventLoop.execute() hop on a path that was previously synchronous. Connects are not a hot path, and the thread that completes the connect future is unchanged -- the Netty connect listener always ran on the loop. Consequence for reattachHostname(): resolved originals now reach it too, and a redirecting resolver would otherwise hand TLS the substituted IP instead of the name the operator configured -- before multi-address support Netty resolved only the TCP destination and the channel kept the original endpoint, so the name won. It now applies to any original that carries a name, resolved or not. An original written as an IP literal is still passed through: a resolver may redirect it, and labelling the substitute with the literal form of a different address would invent a name for it. should_pass_already_resolved_address_through_untouched asserted that the resolver was not consulted, i.e. exactly the behaviour being removed; it now asserts that the resolver was obtained and reached that verdict itself, and a new test covers a resolver that redirects a resolved address. --- .../internal/core/channel/ChannelFactory.java | 62 ++++++++++++------- .../ChannelFactoryMultiAddressTest.java | 30 +++++++-- .../ChannelFactoryNettyResolverTest.java | 51 +++++++++++++-- .../channel/TestAddressResolverGroup.java | 15 +++++ 4 files changed, 127 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 78116675b58..56fe14f4bd8 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -51,6 +51,7 @@ import com.datastax.oss.driver.shaded.guava.common.cache.CacheLoader; import com.datastax.oss.driver.shaded.guava.common.cache.LoadingCache; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import com.datastax.oss.driver.shaded.guava.common.net.InetAddresses; import com.datastax.oss.protocol.internal.ProtocolFeatures; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -407,10 +408,16 @@ private Bootstrap newBootstrap() { * the only way to keep that configuration point working, and the only way to keep {@code * resolve()} non-blocking. * - *

An address the resolver does not support (e.g. {@link io.netty.channel.local.LocalAddress}) - * or that is already resolved is passed through untouched, mirroring {@code - * Bootstrap#doResolveAndConnect0}. A null group means the user called {@link - * Bootstrap#disableResolver()}, which is likewise respected. + *

Whether an address needs resolving at all is the resolver's decision, not ours: exactly as + * in {@code Bootstrap#doResolveAndConnect0}, the address is passed through untouched only when + * the resolver says it does not {@linkplain AddressResolver#isSupported support} it (e.g. {@link + * io.netty.channel.local.LocalAddress}) or that it {@linkplain AddressResolver#isResolved is + * already resolved}. Both are overridable, and a custom resolver may well report an + * already-resolved address as unresolved in order to redirect it — Netty consulted it either way, + * so a pre-check here on {@code InetSocketAddress#isUnresolved()} would silently take that + * configuration point away for every connect to an already-resolved node, which is to say for + * almost every connect. A null group means the user called {@link Bootstrap#disableResolver()}, + * which is likewise respected. * *

Note that with Netty's default resolver the lookup blocks the event loop it runs on, * because {@code DefaultNameResolver} performs {@code InetAddress.getAllByName()} inline. That is @@ -422,15 +429,6 @@ private Bootstrap newBootstrap() { private CompletionStage> resolveCandidates( Bootstrap bootstrap, SocketAddress address, EventLoop eventLoop) { - // Nothing to expand, and nothing for a resolver to contribute: an already-resolved address is - // exactly what we will connect to, and the resolver would pass it through untouched anyway. - // Worth - // short-circuiting because this is the common case -- every node discovered from the peers rows - // holds a resolved address, so this is every pool refill and every reconnect. - if (isResolved(address)) { - return CompletableFuture.completedFuture(Collections.singletonList(address)); - } - AddressResolverGroup resolverGroup = bootstrap.config().resolver(); if (resolverGroup == null) { // Bootstrap.disableResolver(): the user wants the address passed through as-is. @@ -493,11 +491,6 @@ private CompletionStage> resolveCandidates( return result; } - /** Whether {@code address} is already connectable, i.e. needs no resolver at all. */ - private static boolean isResolved(SocketAddress address) { - return address instanceof InetSocketAddress && !((InetSocketAddress) address).isUnresolved(); - } - /** Applies {@link #reattachHostname} to every expanded candidate. */ private static List reattachHostnames( SocketAddress original, List candidates) { @@ -509,8 +502,8 @@ private static List reattachHostnames( } /** - * Re-attaches the {@code original} (unresolved) address's host name to one of the resolved - * candidates it expanded to, whatever name that candidate carries. + * Re-attaches the {@code original} address's host name to one of the resolved candidates it + * expanded to, whatever name that candidate carries. * *

The JDK and Netty-DNS resolvers already attach the queried name to the {@link InetAddress}es * they return, so this is a no-op for them. A custom resolver, however, may build its results @@ -532,6 +525,10 @@ private static List reattachHostnames( * makes {@link #rotate} more deterministic, not less: with one uniform name across an expansion, * its {@code toString()} sort depends only on the IP and port. The single exception is a scoped * IPv6 address, left alone because rebuilding it would drop the scope. + * + *

An original that carries no name of its own is left alone (see {@link #carriesName}): a + * resolver is free to redirect it to a different IP, and labelling that IP with the literal form + * of the one we asked for would invent a name that resolves to something else. */ @VisibleForTesting static SocketAddress reattachHostname(SocketAddress original, SocketAddress candidate) { @@ -541,7 +538,7 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand InetSocketAddress originalInet = (InetSocketAddress) original; InetSocketAddress candidateInet = (InetSocketAddress) candidate; InetAddress candidateIp = candidateInet.getAddress(); - if (!originalInet.isUnresolved() + if (!carriesName(originalInet) || candidateIp == null // Nothing to change: the candidate already carries the queried name, which is the common // case (the JDK and Netty-DNS resolvers attach it themselves). getHostString() never looks @@ -564,6 +561,29 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand } } + /** + * Whether {@code address} denotes a host name, as opposed to an IP address written out in + * literal form. + * + *

A name is what the operator configured and therefore the only thing worth carrying over to a + * candidate (see {@link #reattachHostname}); the literal form of an IP says nothing that the + * candidate's own bytes do not already say. An unresolved address holds whatever string it was + * built from, so it too has to be tested rather than assumed to be a name. + */ + @VisibleForTesting + static boolean carriesName(InetSocketAddress address) { + String hostString = address.getHostString(); + if (hostString == null) { + return false; + } + // Cheap and lookup-free either way: for a resolved address compare against the literal its own + // bytes produce, for an unresolved one (whose getAddress() is null) parse the string. + InetAddress ip = address.getAddress(); + return ip != null + ? !hostString.equals(ip.getHostAddress()) + : !InetAddresses.isInetAddress(hostString); + } + /** * Rotates the expanded address list so that successive connections to the same name do not all * start at the same address. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 64ead82a293..9ebb6752537 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -241,14 +241,36 @@ public void should_pass_non_inet_candidate_through() { } @Test - public void should_pass_candidate_through_when_original_is_resolved() throws Exception { - // A resolved original never went through the resolver, so there is no queried name to - // re-attach. + public void should_pass_candidate_through_when_original_carries_no_name() throws Exception { + // An original written as an IP literal has no name to carry over, and inventing one from the + // literal would be worse than leaving the candidate alone: a resolver is free to redirect it to + // a different IP, which would then be labelled with the literal form of a *different* address. InetSocketAddress original = new InetSocketAddress("127.0.0.1", 9042); InetSocketAddress candidate = - new InetSocketAddress(InetAddress.getByAddress(new byte[] {127, 0, 0, 1}), 9042); + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); + assertThat(ChannelFactory.carriesName(original)).isFalse(); + assertThat(ChannelFactory.carriesName(InetSocketAddress.createUnresolved("10.0.0.1", 9042))) + .isFalse(); + } + + @Test + public void should_reattach_name_of_a_resolved_original() throws Exception { + // A resolved original still reaches the resolver -- whether an address needs resolving is the + // resolver's call, and a custom one may redirect it. Its name is the one the operator + // configured, so it must survive onto whatever the resolver substitutes, exactly as it did when + // Netty resolved the TCP destination and the channel kept the original endpoint for TLS. + InetSocketAddress original = new InetSocketAddress("localhost", 9042); + InetSocketAddress candidate = + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); + + assertThat(ChannelFactory.carriesName(original)).isTrue(); + assertThat(result.getHostString()).isEqualTo("localhost"); + assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java index 243638f48db..92d092ee107 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryNettyResolverTest.java @@ -156,10 +156,9 @@ public void should_not_resolve_at_all_when_the_user_disabled_the_resolver() { @Test public void should_pass_already_resolved_address_through_untouched() { // Given – an endpoint whose address is already resolved, which is the common case: metadata - // nodes - // hold resolved addresses from the peers rows, so this is every pool refill and every - // reconnect. - // Mirrors Bootstrap#doResolveAndConnect0, which skips resolution in that case. + // nodes hold resolved addresses from the peers rows, so this is every pool refill and every + // reconnect. A resolver with the usual semantics reports it as resolved and there is nothing + // to expand. when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); TestAddressResolverGroup resolverGroup = @@ -177,10 +176,50 @@ public void should_pass_already_resolved_address_through_untouched() { NoopNodeMetricUpdater.INSTANCE); completeSimpleChannelInit(); - // Then – had the resolver been consulted it would have redirected us to UNREACHABLE and the - // connection would have failed. + // Then – no lookup was performed: had one been, it would have redirected us to UNREACHABLE and + // the connection would have failed. The decision was the resolver's own, though -- see + // should_let_the_resolver_redirect_an_already_resolved_address. assertThatStage(channelFuture).isSuccess(); assertThat(resolverGroup.queried).isEmpty(); + assertThat(resolverGroup.resolverRequested) + .as("whether an address needs resolving must be the resolver's decision") + .isTrue(); + } + + @Test + public void should_let_the_resolver_redirect_an_already_resolved_address() { + // Given – a resolver that reports even an address carrying an IP as still needing resolution, + // and redirects it. Netty consulted the resolver for every connect, resolved address or not + // (Bootstrap#doResolveAndConnect0 calls isSupported()/isResolved() on it rather than testing + // the address itself), so short-circuiting on InetSocketAddress#isUnresolved() here would take + // that away for every connect to an already-resolved node -- which is nearly all of them. + when(defaultProfile.isDefined(DefaultDriverOption.PROTOCOL_VERSION)).thenReturn(false); + when(protocolVersionRegistry.highestNonBeta()).thenReturn(DefaultProtocolVersion.V4); + TestAddressResolverGroup resolverGroup = + new TestAddressResolverGroup( + Collections.singletonList(SERVER_ADDRESS.resolve()), + /* claimNothingIsResolved = */ true); + installResolver(resolverGroup); + ChannelFactory factory = newChannelFactory(); + + // When – the endpoint holds a resolved address that nothing is listening on. + InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); + CompletionStage channelFuture = + factory.connect( + new DefaultEndPoint(resolved), + null, + null, + DriverChannelOptions.DEFAULT, + NoopNodeMetricUpdater.INSTANCE); + completeSimpleChannelInit(); + + // Then – the connect landed on the address the resolver substituted, which it could only do by + // having been asked about an address that already carried an IP. (Netty then asks again about + // the substitute, from Bootstrap.connect(); that second lookup is its own business.) + assertThatStage(channelFuture).isSuccess(); + assertThat(resolverGroup.queried) + .as("the resolver must get a say on an address that already carries an IP") + .startsWith(resolved); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java index fe544936376..06f5f636cba 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/TestAddressResolverGroup.java @@ -56,8 +56,20 @@ class TestAddressResolverGroup extends AddressResolverGroup { /** The addresses to answer with, or {@code null} to fail every lookup. */ @Nullable private final List answer; + /** + * Whether to claim that every address still needs resolving, even one that already carries an IP. + * A real resolver may do this to redirect traffic, and Netty honours it: {@code + * Bootstrap#doResolveAndConnect0} asks the resolver rather than testing the address itself. + */ + private final boolean claimNothingIsResolved; + TestAddressResolverGroup(@Nullable List answer) { + this(answer, false); + } + + TestAddressResolverGroup(@Nullable List answer, boolean claimNothingIsResolved) { this.answer = answer; + this.claimNothingIsResolved = claimNothingIsResolved; } @Override @@ -73,6 +85,9 @@ public boolean isSupported(SocketAddress address) { @Override public boolean isResolved(SocketAddress address) { + if (claimNothingIsResolved) { + return false; + } // Only hostnames need resolving; anything else (including the local-transport addresses we // hand back) is already usable. return !(address instanceof InetSocketAddress) From a05ddd09f036a071c2507f0a9afa384a062b5f57 Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Tue, 4 Aug 2026 00:23:39 +0200 Subject: [PATCH 31/33] fix: keep the zone when reattaching a hostname to a scoped IPv6 address (DRIVER-201) reattachHostname() passed scoped IPv6 candidates through untouched, on the grounds that InetAddress.getByAddress(host, bytes) cannot carry a scope -- true of that overload, but Inet6Address.getByAddress(host, bytes, scopeId) exists precisely for this. So the queried name was dropped for exactly the addresses that need it as much as any other, and the special case is gone rather than narrowed: a scope id of 0 means "unscoped", so plain IPv6 goes through the same path. The NetworkInterface-taking overload is deliberately not used. It re-derives the numeric scope by searching the interface for an address of the same local type and throws UnknownHostException("no scope_id found") when it finds none, so it can fail for an address that was legitimately built from an interface -- as it does on a host whose loopback carries ::1 but no fe80:: address. The numeric scope is what the connect goes on; only the interface name is lost, and that surfaces in toString() and nowhere else. --- .../internal/core/channel/ChannelFactory.java | 38 ++++++++++---- .../ChannelFactoryMultiAddressTest.java | 49 +++++++++++++++++-- 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 56fe14f4bd8..65210fb91ca 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -67,6 +67,7 @@ import java.net.Inet6Address; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.NetworkInterface; import java.net.ServerSocket; import java.net.SocketAddress; import java.net.UnknownHostException; @@ -523,8 +524,8 @@ private static List reattachHostnames( * lookup, the TCP connect target is the same IP, and a resolved {@link InetSocketAddress}'s * equality ignores host names, so pinning and the pin-equality shortcuts are unaffected. It also * makes {@link #rotate} more deterministic, not less: with one uniform name across an expansion, - * its {@code toString()} sort depends only on the IP and port. The single exception is a scoped - * IPv6 address, left alone because rebuilding it would drop the scope. + * its {@code toString()} sort depends only on the IP and port. A scoped IPv6 candidate keeps its + * scope, since {@link Inet6Address} has {@code getByAddress} overloads that carry one. * *

An original that carries no name of its own is left alone (see {@link #carriesName}): a * resolver is free to redirect it to a different IP, and labelling that IP with the literal form @@ -543,17 +544,12 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand // Nothing to change: the candidate already carries the queried name, which is the common // case (the JDK and Netty-DNS resolvers attach it themselves). getHostString() never looks // anything up -- for a nameless address it falls back to the IP literal. - || candidateInet.getHostString().equals(originalInet.getHostString()) - // Rebuilding a scoped IPv6 address would silently drop its scope. - || (candidateIp instanceof Inet6Address - && (((Inet6Address) candidateIp).getScopeId() != 0 - || ((Inet6Address) candidateIp).getScopedInterface() != null))) { + || candidateInet.getHostString().equals(originalInet.getHostString())) { return candidate; } try { return new InetSocketAddress( - InetAddress.getByAddress(originalInet.getHostString(), candidateIp.getAddress()), - candidateInet.getPort()); + withHostName(originalInet.getHostString(), candidateIp), candidateInet.getPort()); } catch (UnknownHostException impossible) { // getByAddress only rejects illegal byte lengths, and these bytes come from a real // InetAddress; keep the raw candidate rather than failing the connect over a cosmetic step. @@ -561,6 +557,30 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand } } + /** + * Returns a copy of {@code ip} labelled with {@code hostName}, preserving an IPv6 scope if there + * is one. + * + *

{@link InetAddress#getByAddress(String, byte[])} cannot carry a scope, and dropping one + * would change where the address actually points — a link-local address is only meaningful + * together with its zone. {@link Inet6Address#getByAddress(String, byte[], int)} carries the zone + * as its numeric id, which is what the connect itself goes on; a scope id of 0 means "unscoped" + * and is accepted, so this needs no special case for a plain IPv6 address. + * + *

The sibling overload taking a {@link NetworkInterface} is deliberately not used: it + * re-derives the numeric scope by searching that interface for an address of the same local type, + * and throws {@code UnknownHostException("no scope_id found")} when it finds none — so it can + * fail for an address that was legitimately built from an interface in the first place. All that + * is lost by going numeric is the interface name, which surfaces in {@code toString()} and + * nowhere else. + */ + private static InetAddress withHostName(String hostName, InetAddress ip) + throws UnknownHostException { + return ip instanceof Inet6Address + ? Inet6Address.getByAddress(hostName, ip.getAddress(), ((Inet6Address) ip).getScopeId()) + : InetAddress.getByAddress(hostName, ip.getAddress()); + } + /** * Whether {@code address} denotes a host name, as opposed to an IP address written out in * literal form. diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 9ebb6752537..1df30a4ccd9 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -19,6 +19,7 @@ import static com.datastax.oss.driver.Assertions.assertThatStage; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.DefaultProtocolVersion; @@ -31,6 +32,7 @@ import java.net.Inet6Address; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.NetworkInterface; import java.net.SocketAddress; import java.util.Arrays; import java.util.Collections; @@ -288,9 +290,10 @@ public void should_reattach_hostname_to_nameless_ipv6_address() throws Exception } @Test - public void should_not_touch_scoped_ipv6_address() throws Exception { - // InetAddress.getByAddress(host, bytes) cannot carry a scope id, so rebuilding a link-local - // address would silently drop the scope and could break the actual connect. + public void should_keep_the_scope_when_reattaching_to_a_scoped_ipv6_address() throws Exception { + // A link-local address only points anywhere together with its zone, so the queried name has to + // be re-attached without dropping the scope. InetAddress.getByAddress(host, bytes) cannot carry + // one, but Inet6Address.getByAddress(host, bytes, scopeId) can. byte[] linkLocal = new byte[16]; linkLocal[0] = (byte) 0xfe; linkLocal[1] = (byte) 0x80; @@ -298,7 +301,45 @@ public void should_not_touch_scoped_ipv6_address() throws Exception { InetSocketAddress candidate = new InetSocketAddress(Inet6Address.getByAddress(null, linkLocal, 3), 9042); - assertThat(ChannelFactory.reattachHostname(HOSTNAME, candidate)).isSameAs(candidate); + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(result.getAddress()).isInstanceOf(Inet6Address.class); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(3); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_keep_the_zone_of_an_interface_scoped_ipv6_address() throws Exception { + // An address built from a NetworkInterface rather than from an index must keep pointing into + // the + // same zone. The numeric scope the JDK derived at construction is what the connect goes on, so + // carrying that over is enough; only the interface name, a toString() detail, is not. + Inet6Address linkLocal = firstInterfaceScopedIpv6Address(); + assumeThat(linkLocal).as("no interface-scoped IPv6 address on this host").isNotNull(); + InetSocketAddress candidate = new InetSocketAddress(linkLocal, 9042); + + InetSocketAddress result = + (InetSocketAddress) ChannelFactory.reattachHostname(HOSTNAME, candidate); + + assertThat(result.getHostString()).isEqualTo("test.cluster.fake"); + assertThat(((Inet6Address) result.getAddress()).getScopeId()).isEqualTo(linkLocal.getScopeId()); + assertThat(result.getAddress().getAddress()).isEqualTo(linkLocal.getAddress()); + } + + /** An interface-scoped IPv6 address of this host, or null if it has none. */ + private static Inet6Address firstInterfaceScopedIpv6Address() throws Exception { + for (NetworkInterface nif : Collections.list(NetworkInterface.getNetworkInterfaces())) { + for (InetAddress address : Collections.list(nif.getInetAddresses())) { + if (address instanceof Inet6Address + && ((Inet6Address) address).getScopedInterface() != null) { + return (Inet6Address) address; + } + } + } + return null; } @Test From e56428425d20ae0d25f2719911afc43a085fae4b Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Tue, 4 Aug 2026 00:27:55 +0200 Subject: [PATCH 32/33] fix: re-resolve a cloud proxy hostname supplied already resolved (DRIVER-201) SniEndPoint.resolve() used to re-resolve the proxy hostname on every call; since resolution moved to the connection layer it returns the stored address as-is, and only an *unresolved* address gets expanded there. So a proxy hostname that arrived already resolved stayed bound to whichever single IP its lookup happened to return, for the life of the session: no spreading across the proxy's A-records, no fallback when that IP stopped answering, no pick-up of a DNS change. CloudConfigFactory, the usual path, builds an unresolved address, so this only bit withCloudProxyAddress() -- where the ordinary InetSocketAddress(String, int) constructor resolves eagerly, which is the natural way to write it. The constructor now keeps a proxy *hostname* unresolved whichever form it arrived in; a proxy given as an IP address is stored as-is, since there is nothing there to re-resolve and sending a literal through the resolver on every connect would buy nothing. Doing it in the constructor rather than at the call site keeps endpoints built from either form of the same proxy equal (equals() keys on that field) and covers CloudTopologyMonitor, which builds one per node. Contact points deliberately keep the opposite policy, already documented on addContactPoints(): a programmatically supplied resolved address is used as provided. ContactPoints.merge() only ever applied its resolve flag to config-file entries, so that is what they did before this PR too -- it is only the SNI path that re-resolved and therefore only the SNI path that has something to restore. The "does this denote a name" test is shared with ChannelFactory's hostname re-attachment, which needs exactly the same distinction, so it moves to AddressUtils rather than being written twice. --- .../api/core/session/SessionBuilder.java | 6 ++ .../internal/core/channel/ChannelFactory.java | 34 ++--------- .../internal/core/metadata/SniEndPoint.java | 47 ++++++++++++--- .../internal/core/util/AddressUtils.java | 26 ++++++++ .../ChannelFactoryMultiAddressTest.java | 7 ++- .../core/metadata/SniEndPointTest.java | 42 ++++++++++++- .../internal/core/util/AddressUtilsTest.java | 60 +++++++++++++++++++ 7 files changed, 179 insertions(+), 43 deletions(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index bcb54fa81dd..720e3233fd1 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -745,6 +745,12 @@ public SelfT withCloudSecureConnectBundle(@NonNull InputStream cloudConfigInputS * *

For more information, please refer to the DataStax Astra documentation. * + *

A proxy given as a hostname is resolved at connection time, to all of its addresses, + * and each is tried in turn. That holds however the {@link InetSocketAddress} was built: the + * driver keeps a proxy hostname unresolved internally, so passing one that the ordinary {@code + * InetSocketAddress(String, int)} constructor already resolved does not bind the session to that + * single address. + * * @param cloudProxyAddress The address of the Cloud proxy to use. * @see Server Name Indication */ diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java index 65210fb91ca..a6d5db48d3f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ChannelFactory.java @@ -45,13 +45,13 @@ import com.datastax.oss.driver.internal.core.metrics.SessionMetricUpdater; import com.datastax.oss.driver.internal.core.protocol.FrameDecoder; import com.datastax.oss.driver.internal.core.protocol.FrameEncoder; +import com.datastax.oss.driver.internal.core.util.AddressUtils; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; import com.datastax.oss.driver.shaded.guava.common.cache.CacheLoader; import com.datastax.oss.driver.shaded.guava.common.cache.LoadingCache; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; -import com.datastax.oss.driver.shaded.guava.common.net.InetAddresses; import com.datastax.oss.protocol.internal.ProtocolFeatures; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -527,9 +527,10 @@ private static List reattachHostnames( * its {@code toString()} sort depends only on the IP and port. A scoped IPv6 candidate keeps its * scope, since {@link Inet6Address} has {@code getByAddress} overloads that carry one. * - *

An original that carries no name of its own is left alone (see {@link #carriesName}): a - * resolver is free to redirect it to a different IP, and labelling that IP with the literal form - * of the one we asked for would invent a name that resolves to something else. + *

An original that carries no name of its own is left alone (see {@link + * AddressUtils#carriesName}): a resolver is free to redirect it to a different IP, and labelling + * that IP with the literal form of the one we asked for would invent a name that resolves to + * something else. */ @VisibleForTesting static SocketAddress reattachHostname(SocketAddress original, SocketAddress candidate) { @@ -539,7 +540,7 @@ static SocketAddress reattachHostname(SocketAddress original, SocketAddress cand InetSocketAddress originalInet = (InetSocketAddress) original; InetSocketAddress candidateInet = (InetSocketAddress) candidate; InetAddress candidateIp = candidateInet.getAddress(); - if (!carriesName(originalInet) + if (!AddressUtils.carriesName(originalInet) || candidateIp == null // Nothing to change: the candidate already carries the queried name, which is the common // case (the JDK and Netty-DNS resolvers attach it themselves). getHostString() never looks @@ -581,29 +582,6 @@ private static InetAddress withHostName(String hostName, InetAddress ip) : InetAddress.getByAddress(hostName, ip.getAddress()); } - /** - * Whether {@code address} denotes a host name, as opposed to an IP address written out in - * literal form. - * - *

A name is what the operator configured and therefore the only thing worth carrying over to a - * candidate (see {@link #reattachHostname}); the literal form of an IP says nothing that the - * candidate's own bytes do not already say. An unresolved address holds whatever string it was - * built from, so it too has to be tested rather than assumed to be a name. - */ - @VisibleForTesting - static boolean carriesName(InetSocketAddress address) { - String hostString = address.getHostString(); - if (hostString == null) { - return false; - } - // Cheap and lookup-free either way: for a resolved address compare against the literal its own - // bytes produce, for an unresolved one (whose getAddress() is null) parse the string. - InetAddress ip = address.getAddress(); - return ip != null - ? !hostString.equals(ip.getHostAddress()) - : !InetAddresses.isInetAddress(hostString); - } - /** * Rotates the expanded address list so that successive connections to the same name do not all * start at the same address. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index a9cb6c8f422..7c7c087b6fe 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.util.AddressUtils; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetSocketAddress; @@ -37,9 +38,11 @@ public class SniEndPoint implements PinnableEndPoint { @Nullable private final InetSocketAddress pinnedAddress; /** - * @param proxyAddress the address of the proxy. It is returned by {@link #resolve()} as-is, so if - * it is {@linkplain InetSocketAddress#isUnresolved() unresolved} the driver expands it to all - * of the proxy's A-records at connection time and tries each of them. + * @param proxyAddress the address of the proxy. A proxy hostname is stored {@linkplain + * InetSocketAddress#isUnresolved() unresolved}, whether or not it was supplied that way, so + * that the driver expands it to all of the proxy's A-records at connection time and tries + * each of them — see {@link #keepHostnameUnresolved}. A proxy given as an IP address is + * stored as-is. * @param serverName the SNI server name. In the context of Cloud, this is the string * representation of the host id. */ @@ -51,11 +54,37 @@ private SniEndPoint( InetSocketAddress proxyAddress, String serverName, @Nullable InetSocketAddress pinnedAddress) { - this.proxyAddress = Objects.requireNonNull(proxyAddress, "SNI address cannot be null"); + this.proxyAddress = + keepHostnameUnresolved(Objects.requireNonNull(proxyAddress, "SNI address cannot be null")); this.serverName = Objects.requireNonNull(serverName, "SNI Server name cannot be null"); this.pinnedAddress = pinnedAddress; } + /** + * Turns a proxy address that names a host into an unresolved one, leaving anything else + * untouched. + * + *

{@link #resolve()} hands the stored address to the connection layer as-is, and only an + * unresolved one gets expanded and re-expanded there. A proxy hostname supplied already resolved + * would therefore stay bound to whichever single IP its lookup happened to return, for the life + * of the session: no spreading across the proxy's A-records, no fallback when that one IP stops + * answering, and no pick-up of a DNS change. That is a real possibility for a hostname handed to + * {@link + * com.datastax.oss.driver.api.core.session.SessionBuilder#withCloudProxyAddress(InetSocketAddress)}, + * because the ordinary {@code InetSocketAddress(String, int)} constructor resolves eagerly. + * ({@code CloudConfigFactory}, the usual path, already builds an unresolved address.) + * + *

Normalizing here rather than at the call site keeps every {@code SniEndPoint} built from the + * same proxy comparable — {@link #equals} keys on this field — and matches what this endpoint did + * before resolution moved to the connection layer, when it re-resolved the proxy hostname on + * every {@code resolve()} call. + */ + private static InetSocketAddress keepHostnameUnresolved(InetSocketAddress proxyAddress) { + return proxyAddress.isUnresolved() || !AddressUtils.carriesName(proxyAddress) + ? proxyAddress + : InetSocketAddress.createUnresolved(proxyAddress.getHostString(), proxyAddress.getPort()); + } + public String getServerName() { return serverName; } @@ -63,8 +92,8 @@ public String getServerName() { /** * Returns the proxy address connections should be opened to. * - *

Unpinned, this is the configured proxy address as-is. For Cloud that is a hostname (see - * {@code CloudConfigFactory}), which {@link + *

Unpinned, this is the stored proxy address as-is. For Cloud that is a hostname, kept + * unresolved (see {@link #keepHostnameUnresolved}), which {@link * com.datastax.oss.driver.internal.core.channel.ChannelFactory} expands to every proxy A-record, * trying each in turn — so a single unreachable proxy IP no longer fails the connection. * Re-resolving here instead would block whichever event loop called us, and would bypass a custom @@ -89,9 +118,9 @@ public EndPoint pinTo(@NonNull SocketAddress resolvedAddress) { || resolvedAddress.equals(this.pinnedAddress) // The address we already hold: pinning to it changes nothing -- resolve() and toString() // keep yielding what they already do -- so spare the copy (and its redundant - // "proxy(proxy)" toString suffix). Only reachable when the proxy address was supplied - // already resolved: for the usual unresolved Cloud proxy hostname, a resolved pin never - // compares equal to it. + // "proxy(proxy)" toString suffix). Only reachable when the proxy was given as an IP + // address: a proxy hostname is stored unresolved, and a resolved pin never compares equal + // to that. || resolvedAddress.equals(this.proxyAddress)) { return this; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java index 8905edb9192..bc599910ac7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/AddressUtils.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.internal.core.util; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet; +import com.datastax.oss.driver.shaded.guava.common.net.InetAddresses; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -56,4 +57,29 @@ public static Set extract(String address, boolean resolve) { return result; } } + + /** + * Whether {@code address} denotes a host name, as opposed to an IP address written out in + * literal form. + * + *

The distinction matters wherever a name is treated as something that can be resolved — and + * re-resolved — while a literal is taken as the final answer. Both forms can appear resolved or + * unresolved, so neither {@link InetSocketAddress#isUnresolved()} nor the presence of an {@link + * InetAddress} tells them apart. + * + *

Performs no lookup of any kind. + */ + public static boolean carriesName(InetSocketAddress address) { + String hostString = address.getHostString(); + if (hostString == null) { + return false; + } + // A resolved address is compared against the literal its own bytes produce, which is cheaper + // and + // stricter than parsing; an unresolved one has no bytes, so its string has to be parsed. + InetAddress ip = address.getAddress(); + return ip != null + ? !hostString.equals(ip.getHostAddress()) + : !InetAddresses.isInetAddress(hostString); + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java index 1df30a4ccd9..fc6be0af645 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/ChannelFactoryMultiAddressTest.java @@ -27,6 +27,7 @@ import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import com.datastax.oss.driver.internal.core.metrics.NoopNodeMetricUpdater; +import com.datastax.oss.driver.internal.core.util.AddressUtils; import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.channel.local.LocalAddress; import java.net.Inet6Address; @@ -252,8 +253,8 @@ public void should_pass_candidate_through_when_original_carries_no_name() throws new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042); assertThat(ChannelFactory.reattachHostname(original, candidate)).isSameAs(candidate); - assertThat(ChannelFactory.carriesName(original)).isFalse(); - assertThat(ChannelFactory.carriesName(InetSocketAddress.createUnresolved("10.0.0.1", 9042))) + assertThat(AddressUtils.carriesName(original)).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("10.0.0.1", 9042))) .isFalse(); } @@ -270,7 +271,7 @@ public void should_reattach_name_of_a_resolved_original() throws Exception { InetSocketAddress result = (InetSocketAddress) ChannelFactory.reattachHostname(original, candidate); - assertThat(ChannelFactory.carriesName(original)).isTrue(); + assertThat(AddressUtils.carriesName(original)).isTrue(); assertThat(result.getHostString()).isEqualTo("localhost"); assertThat(result.getAddress().getHostAddress()).isEqualTo("10.0.0.1"); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java index 5213222cfe4..36c6e0cf714 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -38,6 +38,42 @@ public void resolve_returns_the_proxy_address_as_is_without_looking_it_up() { assertThat(endPoint.resolve().isUnresolved()).isTrue(); } + @Test + public void should_keep_a_resolved_proxy_hostname_unresolved() { + // InetSocketAddress(String, int) resolves eagerly, so a hostname passed to + // withCloudProxyAddress() arrives here already bound to one of its IPs. Storing it that way + // would freeze every Cloud connection on that IP for the life of the session: resolve() hands + // the stored address straight to the connection layer, which only expands unresolved ones. + InetSocketAddress resolvedProxy = new InetSocketAddress("localhost", 9042); + assertThat(resolvedProxy.isUnresolved()).isFalse(); + + SniEndPoint endPoint = new SniEndPoint(resolvedProxy, "test-server-name"); + + assertThat(endPoint.resolve().isUnresolved()).isTrue(); + assertThat(endPoint.resolve().getHostString()).isEqualTo("localhost"); + assertThat(endPoint.resolve().getPort()).isEqualTo(9042); + // Normalization is unconditional, so endpoints built from either form of the same proxy still + // denote the same node -- equals() keys on the stored address. + assertThat(endPoint) + .isEqualTo( + new SniEndPoint( + InetSocketAddress.createUnresolved("localhost", 9042), "test-server-name")); + // The metric prefix is unaffected either way: it was already built from the host string. + assertThat(endPoint.asMetricPrefix()).isEqualTo("localhost:9042_test-server-name"); + } + + @Test + public void should_keep_a_proxy_given_as_an_ip_address_as_is() { + // Nothing to re-resolve: an IP address is the final answer, and turning it into an unresolved + // "hostname" would only send a literal through the resolver on every connect. + InetSocketAddress ipProxy = new InetSocketAddress("127.0.0.1", 9042); + + SniEndPoint endPoint = new SniEndPoint(ipProxy, "test-server-name"); + + assertThat(endPoint.resolve()).isSameAs(ipProxy); + assertThat(endPoint.resolve().isUnresolved()).isFalse(); + } + @Test public void resolve_does_not_throw_for_unresolvable_proxy_hostname() { // No lookup happens here, so an unresolvable name only fails later, at connect time. @@ -88,9 +124,9 @@ public void pin_to_should_return_same_instance_when_already_pinned_to_that_addre @Test public void pin_to_should_return_same_instance_when_address_is_the_proxy_address_itself() { - // Only reachable when the proxy address was supplied already resolved (Cloud supplies a - // hostname): pinning to the very address the endpoint holds is a no-op, so the copy -- and its - // redundant "proxy(proxy)" toString suffix -- is spared. + // Only reachable when the proxy was given as an IP address (Cloud supplies a hostname, which is + // stored unresolved): pinning to the very address the endpoint holds is a no-op, so the copy -- + // and its redundant "proxy(proxy)" toString suffix -- is spared. InetSocketAddress resolvedProxy = new InetSocketAddress("127.0.0.1", 9042); SniEndPoint endPoint = new SniEndPoint(resolvedProxy, "test-server-name"); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java new file mode 100644 index 00000000000..f48d70ef838 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/AddressUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import org.junit.Test; + +public class AddressUtilsTest { + + @Test + public void should_recognize_a_hostname_whether_resolved_or_not() { + assertThat( + AddressUtils.carriesName(InetSocketAddress.createUnresolved("host.example.com", 9042))) + .isTrue(); + // Eagerly resolved by the constructor, but still a name. + assertThat(AddressUtils.carriesName(new InetSocketAddress("localhost", 9042))).isTrue(); + } + + @Test + public void should_not_mistake_an_ip_literal_for_a_hostname() throws Exception { + assertThat(AddressUtils.carriesName(new InetSocketAddress("127.0.0.1", 9042))).isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("127.0.0.1", 9042))) + .isFalse(); + assertThat(AddressUtils.carriesName(InetSocketAddress.createUnresolved("::1", 9042))).isFalse(); + // Built from raw bytes, so it carries no name at all and getHostString() falls back to the + // literal -- without triggering the reverse lookup that getHostName() would. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress(InetAddress.getByAddress(new byte[] {10, 0, 0, 1}), 9042))) + .isFalse(); + } + + @Test + public void should_report_an_explicitly_named_address_as_a_name() throws Exception { + // A resolver may label its results with a name of its own; that is still a name. + assertThat( + AddressUtils.carriesName( + new InetSocketAddress( + InetAddress.getByAddress("cname.example.com", new byte[] {10, 0, 0, 1}), 9042))) + .isTrue(); + } +} From f2a262cac1e14f0151d4fd51544201355b1a73fb Mon Sep 17 00:00:00 2001 From: Mikita Hradovich Date: Tue, 4 Aug 2026 00:35:10 +0200 Subject: [PATCH 33/33] fix: keep a node's metric identity stable across endpoint pinning (DRIVER-201) Nodes adopt pinned endpoint copies -- DefaultTopologyMonitor#buildNodeEndPoint returns the control channel's endpoint verbatim for the control node's own system.local row, and that endpoint is pinned -- so anything derived from a node's endpoint had better not depend on the pin. PinnableEndPoint said as much for equals(), hashCode() and asMetricPrefix(), but not for toString(), and TaggingMetricIdGenerator tags every node metric with node.getEndPoint().toString(). Ids are computed per call, so the control node's tag silently changed to "addr(pinned)" the first time the topology refreshed, and again on every move to another address, leaving the earlier series registered and never cleared. A third-party MetricIdGenerator is just as free to use toString(), and cannot be fixed from here, so the pin is now invisible in toString() too and the contract says so. Nothing is lost for diagnostics: the address a channel actually connected to is in the channel's own toString(), which Netty builds from its remote address, and ChannelFactory logs each candidate as it tries it. The other half is the reverse mistake. setEndPoint() rebuilt the updater when the endpoints were unequal, but DefaultEndPoint#equals resolves an unresolved address before comparing, so an unresolved hostname and the address it maps to compare *equal* while their metric prefixes differ -- which is precisely what happens when a contact-point node adopts the endpoint built from its system.local row. The rebuild is now keyed on the metric identity itself, which is both narrower (a pin-only change is not one) and wider (an equal endpoint that renames the metrics is) than node equality. Both halves of that identity are compared, since the default generator names metrics after asMetricPrefix() and the tagging one tags them with toString(). The pin-only test asserted this against a context whose metrics factory returns NoopNodeMetricUpdater -- for which the rebuild is skipped unconditionally -- so it could not fail. It and its new counterpart now stub the factory, and the new one was checked to fail against the old condition. --- .../core/metadata/DefaultEndPoint.java | 7 ++- .../internal/core/metadata/DefaultNode.java | 36 ++++++++---- .../core/metadata/PinnableEndPoint.java | 20 +++++-- .../internal/core/metadata/SniEndPoint.java | 11 ++-- .../core/metadata/DefaultEndPointTest.java | 8 ++- .../core/metadata/DefaultNodeTest.java | 57 +++++++++++++++++-- .../core/metadata/SniEndPointTest.java | 11 ++-- 7 files changed, 113 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java index 9ff35594377..bff230917b6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPoint.java @@ -113,9 +113,10 @@ public int hashCode() { @Override public String toString() { - // Show both when pinned: the original identifies the node, the pinned address tells you which - // IP a connection actually landed on -- which is the useful bit in connection-level logs. - return pinnedAddress == null ? address.toString() : address + "(" + pinnedAddress + ")"; + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which IP a given connection + // landed on is in the channel's own toString(), which Netty builds from the actual remote + // address. + return address.toString(); } @NonNull diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java index e42c0b0d3cd..15b943313bf 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultNode.java @@ -102,17 +102,33 @@ public EndPoint getEndPoint() { } public void setEndPoint(@NonNull EndPoint newEndPoint, @NonNull InternalDriverContext context) { - boolean differentNode = !newEndPoint.equals(endPoint); - // Adopt the newest instance even when it compares equal. A PinnableEndPoint copy differs from - // the original only by the address it is pinned to, and equals() ignores that by contract (see - // PinnableEndPoint) -- but it is the address every subsequent connection to this node will use, - // so refusing to adopt it would freeze the node on the first address it ever connected to, even - // after the control connection has moved to another one and told us about it. + // Metrics are registered under names derived from the endpoint, so they have to be + // re-registered + // whenever those names change -- which is not the same question as whether this is a different + // node. It is narrower in one direction: a PinnableEndPoint copy differs from the original only + // by the address it is pinned to, and both equals() and the metric identity ignore that by + // contract (see PinnableEndPoint). And it is wider in the other: an unresolved hostname and the + // resolved address it maps to compare *equal* (see DefaultEndPoint#equals) while their metric + // prefixes differ, which is exactly what happens when a contact-point node adopts the endpoint + // built from its system.local row. + // + // Both halves of that identity are compared, because both are in use: the default + // MetricIdGenerator names node metrics after asMetricPrefix(), the tagging one tags them with + // the endpoint's toString(). Comparing toString() does mean that an endpoint whose string form + // is not stable across equal instances re-registers this node's metrics on every topology + // refresh; that is the intended reading, since the alternative is reporting under a name the + // endpoint no longer answers to. + boolean differentMetricIdentity = + !newEndPoint.asMetricPrefix().equals(endPoint.asMetricPrefix()) + || !newEndPoint.toString().equals(endPoint.toString()); + // Adopt the newest instance even when it compares equal: a pinned copy carries the address + // every + // subsequent connection to this node will use, so refusing it would freeze the node on the + // first + // address it ever connected to, even after the control connection moved to another one and told + // us about it. endPoint = newEndPoint; - if (differentNode) { - // Only a genuine address change may rebuild the updater: asMetricPrefix() is pin-independent, - // so doing it for a pin-only change would clear and re-register metrics under identical - // names. + if (differentMetricIdentity) { // metricUpdater is transient, so it can be null on deserialized nodes. NodeMetricUpdater previousMetricUpdater = metricUpdater; if (previousMetricUpdater != null diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java index fcfbb43d9ab..aee8630e09f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/PinnableEndPoint.java @@ -47,11 +47,21 @@ * and leaves any other implementation untouched, so third-party {@link EndPoint}s keep working * exactly as before. * - *

Implementations must keep {@link Object#equals}, {@link Object#hashCode} and {@link - * EndPoint#asMetricPrefix()} identical to the unpinned original — a pinned copy denotes the same - * node, and metric names must not change depending on which IP a connection happened to land on. - * Equality must stay symmetric: {@code original.equals(pinned)} and {@code pinned.equals(original)} - * must agree, since endpoints are used as set and map keys. + *

Implementations must keep {@link Object#equals}, {@link Object#hashCode}, {@link + * EndPoint#asMetricPrefix()} and {@link Object#toString()} identical to the unpinned + * original: a pinned copy denotes the same node, and every one of those is part of how the node is + * identified from the outside. Metric names in particular must not change depending on which IP a + * connection happened to land on — and that includes {@code toString()}, which is what {@code + * TaggingMetricIdGenerator} tags node metrics with, and what any third-party {@code + * MetricIdGenerator} is equally free to use. Nodes do adopt pinned copies (see {@code + * DefaultNode#setEndPoint}), so an identity that varied with the pin would silently re-tag a node's + * metrics mid-session. Equality must also stay symmetric: {@code original.equals(pinned)} and + * {@code pinned.equals(original)} must agree, since endpoints are used as set and map keys. + * + *

The pinned address is therefore observable only through {@link EndPoint#resolve()}. That is no + * loss for diagnostics: the address a channel is actually connected to appears in the channel's own + * {@code toString()}, which Netty builds from its remote address, and {@code ChannelFactory} logs + * each candidate as it tries it. */ public interface PinnableEndPoint extends EndPoint { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java index 7c7c087b6fe..69c0c1151e7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/SniEndPoint.java @@ -146,13 +146,10 @@ public int hashCode() { @Override public String toString() { - // An unpinned endpoint prints the original proxy address, so with multiple A-records it does - // not say which one a given connection selected. A pinned copy does -- and channels carry a - // pinned copy whenever the connected IP differs from the stored proxy address (see pinTo()) -- - // so connection-level logs identify the actual proxy IP. - return pinnedAddress == null - ? proxyAddress + ":" + serverName - : proxyAddress + "(" + pinnedAddress + "):" + serverName; + // Deliberately identical for a pinned copy: see PinnableEndPoint. Which proxy IP a given + // connection landed on is in the channel's own toString(), which Netty builds from the actual + // remote address. + return proxyAddress + ":" + serverName; } @NonNull diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java index e716bf19371..c92c7b8e20c 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultEndPointTest.java @@ -108,8 +108,10 @@ public void pin_to_should_override_resolution_but_preserve_identity() { // Resolution now yields exactly the pinned address... assertThat(pinned.resolve()).isEqualTo(pinnedTo); // ...but the copy still denotes the same node, and metric names must not change depending on - // which IP a connection happened to land on. + // which IP a connection happened to land on -- including through toString(), which is what + // TaggingMetricIdGenerator tags node metrics with, and nodes do adopt pinned copies. assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); assertThat(pinned).isEqualTo(original); assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); // Equality has to hold in both directions: endpoints are used as set and map keys. @@ -133,13 +135,13 @@ public void pin_to_should_return_same_instance_when_already_pinned_to_that_addre public void pin_to_should_return_same_instance_when_address_is_already_the_endpoints_own() { // An already-resolved endpoint expands to exactly one candidate -- itself -- so ChannelFactory // pins it to the address it already holds. That copy would be indistinguishable from the - // original, and would only add a redundant suffix to toString(). Every node discovered from the + // original in every respect, so there is no point allocating it. Every node discovered from the // peers rows takes this path. InetSocketAddress resolved = new InetSocketAddress("127.0.0.1", 9042); DefaultEndPoint endPoint = new DefaultEndPoint(resolved); assertThat(endPoint.pinTo(new InetSocketAddress("127.0.0.1", 9042))).isSameAs(endPoint); - assertThat(endPoint.toString()).isEqualTo(resolved.toString()).doesNotContain("("); + assertThat(endPoint.toString()).isEqualTo(resolved.toString()); } @Test diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java index 18a85bcfa3d..6f9f6916769 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultNodeTest.java @@ -18,10 +18,16 @@ package com.datastax.oss.driver.internal.core.metadata; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.MockedDriverContextFactory; +import com.datastax.oss.driver.internal.core.metrics.MetricsFactory; import com.datastax.oss.driver.internal.core.metrics.NodeMetricUpdater; import java.net.InetSocketAddress; import java.util.UUID; @@ -86,15 +92,56 @@ public void should_adopt_a_newer_endpoint_that_only_differs_by_its_pinned_addres @Test public void should_not_rebuild_the_metric_updater_for_a_pin_only_change() { - // asMetricPrefix() is pin-independent, so rebuilding would clear and re-register metrics under - // identical names for no reason. - InternalDriverContext context = MockedDriverContextFactory.defaultDriverContext(); + // A pinned copy is identified exactly like the original -- same asMetricPrefix(), same + // toString() -- so rebuilding would clear and re-register metrics under identical names, and + // reset their values along the way. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); DefaultNode node = new DefaultNode(endPoint, context); - NodeMetricUpdater before = node.getMetricUpdater(); + assertThat(node.getMetricUpdater()).isSameAs(first); node.setEndPoint( ((PinnableEndPoint) endPoint).pinTo(new InetSocketAddress("127.0.0.2", 9042)), context); - assertThat(node.getMetricUpdater()).isSameAs(before); + assertThat(node.getMetricUpdater()).isSameAs(first); + verify(first, never()).clearMetrics(); + } + + @Test + public void should_rebuild_the_metric_updater_when_an_equal_endpoint_renames_the_metrics() { + // An unresolved hostname and the address it resolves to compare *equal* (see + // DefaultEndPoint#equals) but do not produce the same metric prefix. That is exactly what + // happens when a contact-point node adopts the endpoint built from its system.local row, so + // deciding on equals() alone would leave the node's metrics registered under the hostname while + // asMetricPrefix() had moved on to the IP. + MetricsFactory metricsFactory = mock(MetricsFactory.class); + InternalDriverContext context = contextWith(metricsFactory); + NodeMetricUpdater first = mock(NodeMetricUpdater.class); + NodeMetricUpdater second = mock(NodeMetricUpdater.class); + when(metricsFactory.newNodeUpdater(any())).thenReturn(first, second); + + EndPoint asHostname = + new DefaultEndPoint(InetSocketAddress.createUnresolved("localhost", 9042)); + EndPoint asAddress = new DefaultEndPoint(new InetSocketAddress("127.0.0.1", 9042)); + assertThat(asHostname).isEqualTo(asAddress); + assertThat(asHostname.asMetricPrefix()).isNotEqualTo(asAddress.asMetricPrefix()); + + DefaultNode node = new DefaultNode(asHostname, context); + assertThat(node.getMetricUpdater()).isSameAs(first); + + node.setEndPoint(asAddress, context); + + assertThat(node.getMetricUpdater()).isSameAs(second); + verify(first).clearMetrics(); + } + + /** A context whose only stubbed behaviour is the metrics factory {@code DefaultNode} asks for. */ + private static InternalDriverContext contextWith(MetricsFactory metricsFactory) { + InternalDriverContext context = mock(InternalDriverContext.class); + when(context.getMetricsFactory()).thenReturn(metricsFactory); + return context; } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java index 36c6e0cf714..63484ec21d9 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/SniEndPointTest.java @@ -102,9 +102,13 @@ public void pin_to_should_make_resolve_return_the_connected_proxy_ip_and_preserv // The original is untouched. assertThat(original.resolve().isUnresolved()).isTrue(); - // The pinned copy still denotes the same node. + // The pinned copy still denotes the same node, down to every string it is identified by: the + // tagging MetricIdGenerator tags node metrics with the endpoint's toString(), and nodes do + // adopt + // pinned copies. assertThat(pinned.getServerName()).isEqualTo(original.getServerName()); assertThat(pinned.asMetricPrefix()).isEqualTo(original.asMetricPrefix()); + assertThat(pinned.toString()).isEqualTo(original.toString()); assertThat(pinned).isEqualTo(original); assertThat(original).isEqualTo(pinned); assertThat(pinned.hashCode()).isEqualTo(original.hashCode()); @@ -125,12 +129,11 @@ public void pin_to_should_return_same_instance_when_already_pinned_to_that_addre @Test public void pin_to_should_return_same_instance_when_address_is_the_proxy_address_itself() { // Only reachable when the proxy was given as an IP address (Cloud supplies a hostname, which is - // stored unresolved): pinning to the very address the endpoint holds is a no-op, so the copy -- - // and its redundant "proxy(proxy)" toString suffix -- is spared. + // stored unresolved): pinning to the very address the endpoint holds is a no-op, so there is no + // point allocating a copy that would be indistinguishable from it. InetSocketAddress resolvedProxy = new InetSocketAddress("127.0.0.1", 9042); SniEndPoint endPoint = new SniEndPoint(resolvedProxy, "test-server-name"); assertThat(endPoint.pinTo(new InetSocketAddress("127.0.0.1", 9042))).isSameAs(endPoint); - assertThat(endPoint.toString()).doesNotContain("("); } }