From 57d27fef04cd82394d5d5bded08c02464fce7504 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Tue, 27 Oct 2026 10:55:45 +0530 Subject: [PATCH 1/2] failed accept() should pause before retrying which caused errors to be flooded --- .../server/XceiverServerDomainSocket.java | 8 +++++ .../hdds/TestXceiverServerDomainSocket.java | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java index 6207d8cbe6eb..6fc98cc08152 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java @@ -230,6 +230,14 @@ public void run() { // usually when the xceiver count limit is hit. LOG.warn("Got an exception. Peer {}", peer, ie); IOUtils.closeQuietly(peer); + if (peer == null) { + // accept() itself failed, for example with no file descriptor left, and would fail again right away + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(1L)); + } catch (InterruptedException e) { + // not restored: stop() clears isRunning before interrupting, and a set flag would make the pause a spin + } + } } catch (OutOfMemoryError ie) { IOUtils.closeQuietly(peer); // DataNode can run out of memory if there is too many transfers. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java index fc134b0c7c14..8fb3439b15c7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java @@ -27,7 +27,10 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import com.google.common.collect.Maps; @@ -51,6 +54,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; @@ -94,6 +98,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.MockedStatic; /** * Tests the XceiverServerDomainSocket class. @@ -629,6 +634,36 @@ public void testMaxXceiverCount() throws IOException, InterruptedException { } } + /** + * A failed accept(), for example with no file descriptor left, is retried after a pause instead of at once. + */ + @Test + public void testAcceptFailureIsRetriedAfterPause() throws Exception { + conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, dir.getAbsolutePath()); + AtomicInteger accepts = new AtomicInteger(); + DomainSocket listener = mock(DomainSocket.class, RETURNS_DEEP_STUBS); + when(listener.accept()).thenAnswer(invocation -> { + accepts.incrementAndGet(); + throw new SocketException("accept(2) error: Too many open files"); + }); + DomainSocketFactory factory = mock(DomainSocketFactory.class); + when(factory.isServiceEnabled()).thenReturn(true); + when(factory.isServiceReady()).thenReturn(true); + XceiverServerDomainSocket server; + try (MockedStatic domainSocket = mockStatic(DomainSocket.class)) { + domainSocket.when(() -> DomainSocket.bindAndListen(any())).thenReturn(listener); + server = new XceiverServerDomainSocket(randomDatanodeDetails(), conf, null, readExecutors, metrics, factory); + } + try { + server.start(); + GenericTestUtils.waitFor(() -> accepts.get() > 0, 10, 5000); + Thread.sleep(500); + assertTrue(accepts.get() <= 2, "accept() was called " + accepts.get() + " times"); + } finally { + server.stop(); + } + } + /** * When server receives any message which doesn't follow the version, request type, request body sequence, server * will treat it as a critical error, close the connection. From ebc692e7fca2ecc4e85bf20b9cb5895c39af1d06 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Wed, 28 Oct 2026 15:40:55 +0530 Subject: [PATCH 2/2] add log and correct the comment --- .../transport/server/XceiverServerDomainSocket.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java index 7ca2955bdfd7..c1826d38d2cd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java @@ -232,16 +232,18 @@ public void run() { // another thread closed our listener socket - that's expected during shutdown, but not in other circumstances LOG.info("XceiverServerDomainSocket is closed", ace); } catch (IOException ie) { - // usually when the xceiver count limit is hit. - LOG.warn("Got an exception. Peer {}", peer, ie); - IOUtils.closeQuietly(peer); if (peer == null) { // accept() itself failed, for example with no file descriptor left, and would fail again right away + LOG.warn("accept() failed. Will retry after 1 second.", ie); try { Thread.sleep(TimeUnit.SECONDS.toMillis(1L)); } catch (InterruptedException e) { - // not restored: stop() clears isRunning before interrupting, and a set flag would make the pause a spin + Thread.currentThread().interrupt(); } + } else { + // usually when the xceiver count limit is hit. + LOG.warn("Got an exception. Peer {}", peer, ie); + IOUtils.closeQuietly(peer); } } catch (OutOfMemoryError ie) { IOUtils.closeQuietly(peer);