From db51efa849dfadd3a5340072f5da848fd983ffc6 Mon Sep 17 00:00:00 2001 From: Oleksandr Lazarenko Date: Wed, 24 Jun 2026 14:43:32 +0200 Subject: [PATCH] FIX: Resolve nightly web port-bind flake and MetadataOnPrem OOM --- .../examples/console/MetadataOnPrem.java | 17 ++++++--- .../examples/web/GettingStartedApiTest.java | 11 +++--- .../web/GettingStartedWebCloudMixedTest.java | 12 +++--- .../web/GettingStartedWebCloudTest.java | 12 +++--- .../web/GettingStartedWebMixedTest.java | 12 +++--- .../web/GettingStartedWebOnPremTest.java | 10 +++-- .../examples/web/EmbedJetty.java | 37 +++++++++++++++++++ 7 files changed, 82 insertions(+), 29 deletions(-) diff --git a/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index a69a3a3..2ab25e4 100644 --- a/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -27,7 +27,6 @@ import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; -import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; import fiftyone.pipeline.engines.fiftyone.data.ValueMetaData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,12 +145,18 @@ private static void outputEvidenceKeyDetails(IPIntelligenceOnPremiseEngine engin private static void outputProfileDetails(IPIntelligenceOnPremiseEngine engine, PrintWriter output) { - // Group the profiles by component and then output the number of profiles - // for each component. - Map> groups = + // Group the profiles by component and then output the number of profiles + // for each component. Count with a reducing collector rather than + // collecting every ProfileMetaData into a List: an on-premise IP + // Intelligence data file has an enormous number of profiles, so + // materializing them all just to call size() exhausts the heap + // (OutOfMemoryError). counting() retains only the per-group total. + Map groups = StreamSupport.stream(engine.getProfiles().spliterator(), false) - .collect(Collectors.groupingBy(p -> p.getComponent().getName())); - groups.forEach((k,v)->output.format("%s Profiles: %d\n", k , v.size())); + .collect(Collectors.groupingBy( + p -> p.getComponent().getName(), + Collectors.counting())); + groups.forEach((k,v)->output.format("%s Profiles: %d\n", k , v)); } // Output the component name as well as a list of all the associated properties. diff --git a/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiTest.java b/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiTest.java index ce78ede..97a76e7 100644 --- a/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiTest.java +++ b/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiTest.java @@ -44,7 +44,7 @@ public class GettingStartedApiTest { private static Server SERVER; - private static final int PORT = GettingStartedApi.DEFAULT_PORT; + private static int PORT; @BeforeClass public static void startJetty() throws Exception { @@ -56,7 +56,10 @@ public static void startJetty() throws Exception { Map initParams = new HashMap<>(); initParams.put(GettingStartedApi.DATA_FILE_INIT_PARAM, dataFilePath); - SERVER = EmbedJetty.startServlet("/*", PORT, GettingStartedApi.class, initParams); + // Bind an OS-assigned ephemeral port (0) to avoid intermittent + // "Address already in use" failures from a fixed port not yet released. + SERVER = EmbedJetty.startServlet("/*", 0, GettingStartedApi.class, initParams); + PORT = EmbedJetty.boundPort(SERVER); } private static String get(String path, int expectedCode) throws Exception { @@ -108,8 +111,6 @@ public void testUnknownPathReturns404() throws Exception { @AfterClass public static void stopJetty() throws Exception { - if (SERVER != null) { - SERVER.stop(); - } + EmbedJetty.stopAndJoin(SERVER); } } diff --git a/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java b/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java index fbbc686..ca1b1ec 100644 --- a/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java +++ b/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java @@ -40,6 +40,7 @@ public class GettingStartedWebCloudMixedTest { private static Server SERVER; + private static int PORT; @BeforeClass public static void startJetty() throws Exception { @@ -51,15 +52,18 @@ public static void startJetty() throws Exception { // Intelligence properties as this example uses both engines. System.setProperty(KeyHelper.TEST_RESOURCE_KEY, resourceKey); + // Bind an OS-assigned ephemeral port (0) to avoid intermittent + // "Address already in use" failures from a fixed port not yet released. SERVER = EmbedJetty.startWebApp( - GettingStartedWebCloudMixed.getResourceBase(), 8084); + GettingStartedWebCloudMixed.getResourceBase(), 0); + PORT = EmbedJetty.boundPort(SERVER); } @Test public void testWebCloudMixed() throws Exception { HttpURLConnection connection = - (HttpURLConnection) new URL("http://localhost:8084/").openConnection(); + (HttpURLConnection) new URL("http://localhost:" + PORT + "/").openConnection(); int code = connection.getResponseCode(); @@ -91,8 +95,6 @@ public void testWebCloudMixed() throws Exception { @AfterClass public static void stopJetty() throws Exception { - if (SERVER != null) { - SERVER.stop(); - } + EmbedJetty.stopAndJoin(SERVER); } } diff --git a/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java b/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java index 8bbc4fe..ea87c40 100644 --- a/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java +++ b/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java @@ -42,6 +42,7 @@ public class GettingStartedWebCloudTest { private static Server SERVER; + private static int PORT; @BeforeClass public static void startJetty() throws Exception { @@ -51,14 +52,17 @@ public static void startJetty() throws Exception { // Make the resource key available to the pipeline configuration file System.setProperty(KeyHelper.TEST_RESOURCE_KEY, resourceKey); - SERVER = EmbedJetty.startWebApp(getFilePath(getResourceBase()).getAbsolutePath(), 8083); + // Bind an OS-assigned ephemeral port (0) to avoid intermittent + // "Address already in use" failures from a fixed port not yet released. + SERVER = EmbedJetty.startWebApp(getFilePath(getResourceBase()).getAbsolutePath(), 0); + PORT = EmbedJetty.boundPort(SERVER); } @Test public void testWebCloud() throws Exception { HttpURLConnection connection = - (HttpURLConnection) new URL("http://localhost:8083/").openConnection(); + (HttpURLConnection) new URL("http://localhost:" + PORT + "/").openConnection(); int code = connection.getResponseCode(); @@ -93,8 +97,6 @@ public void testWebCloud() throws Exception { @AfterClass public static void stopJetty() throws Exception { - if (SERVER != null) { - SERVER.stop(); - } + EmbedJetty.stopAndJoin(SERVER); } } diff --git a/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java b/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java index a341c62..4aa023b 100644 --- a/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java +++ b/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java @@ -43,6 +43,7 @@ public class GettingStartedWebMixedTest { private static Server SERVER; + private static int PORT; /** * Resolve the device detection data file. It is not part of this @@ -77,15 +78,18 @@ public static void startJetty() throws Exception { ipiDataFilePath != null); System.setProperty("TestDataFile", ipiDataFilePath); + // Bind an OS-assigned ephemeral port (0) to avoid intermittent + // "Address already in use" failures from a fixed port not yet released. SERVER = EmbedJetty.startWebApp( - GettingStartedWebMixed.getResourceBase(), 8081); + GettingStartedWebMixed.getResourceBase(), 0); + PORT = EmbedJetty.boundPort(SERVER); } @Test public void testWebMixed() throws Exception { HttpURLConnection connection = - (HttpURLConnection) new URL("http://localhost:8081/").openConnection(); + (HttpURLConnection) new URL("http://localhost:" + PORT + "/").openConnection(); int code = connection.getResponseCode(); @@ -117,8 +121,6 @@ public void testWebMixed() throws Exception { @AfterClass public static void stopJetty() throws Exception { - if (SERVER != null) { - SERVER.stop(); - } + EmbedJetty.stopAndJoin(SERVER); } } diff --git a/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java b/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java index 6962eed..e1831b3 100644 --- a/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java +++ b/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java @@ -41,6 +41,7 @@ public class GettingStartedWebOnPremTest { private static Server SERVER; + private static int PORT; @BeforeClass public static void startJetty() throws Exception { @@ -51,14 +52,17 @@ public static void startJetty() throws Exception { dataFilePath != null); System.setProperty("TestDataFile", dataFilePath); - SERVER = EmbedJetty.startWebApp(getFilePath(getResourceBase()).getAbsolutePath(), 8081); + // Bind an OS-assigned ephemeral port (0) to avoid intermittent + // "Address already in use" failures from a fixed port not yet released. + SERVER = EmbedJetty.startWebApp(getFilePath(getResourceBase()).getAbsolutePath(), 0); + PORT = EmbedJetty.boundPort(SERVER); } @Test public void testWebOnPrem() throws Exception { HttpURLConnection connection = - (HttpURLConnection) new URL("http://localhost:8081/").openConnection(); + (HttpURLConnection) new URL("http://localhost:" + PORT + "/").openConnection(); int code = connection.getResponseCode(); @@ -96,6 +100,6 @@ public void testWebOnPrem() throws Exception { @AfterClass public static void stopJetty() throws Exception { - SERVER.stop(); + EmbedJetty.stopAndJoin(SERVER); } } \ No newline at end of file diff --git a/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/EmbedJetty.java b/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/EmbedJetty.java index 11f0675..382c6da 100644 --- a/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/EmbedJetty.java +++ b/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/EmbedJetty.java @@ -57,6 +57,12 @@ public static Server startWebApp(String resourceBase, int port) throws Exception // Link the context to the server. server.setHandler(context); + // Ensure the connector socket is released promptly and gracefully when + // the server is stopped, so a subsequent bind on the same port (e.g. + // when tests run back-to-back) does not hit "Address already in use". + server.setStopAtShutdown(true); + server.setStopTimeout(5000); + server.start(); // Wait for the server to be fully started and ready to accept connections @@ -66,6 +72,32 @@ public static Server startWebApp(String resourceBase, int port) throws Exception return server; } + /** + * Returns the local TCP port the server's first connector is actually + * listening on. When the server is started with port {@code 0} the OS + * assigns a free ephemeral port; tests should read it back via this method + * rather than assuming a fixed port. + * + * @param server a started Jetty server + * @return the bound local port + */ + public static int boundPort(Server server) { + return ((ServerConnector) server.getConnectors()[0]).getLocalPort(); + } + + /** + * Stops the server and blocks until it has fully terminated, ensuring the + * listening socket is released before the method returns. + * + * @param server the server to stop (ignored if null) + */ + public static void stopAndJoin(Server server) throws Exception { + if (server != null) { + server.stop(); + server.join(); + } + } + /** * Wait for the server to be fully started and ready to accept connections. * This includes waiting for all connectors to be started and the context to be available. @@ -141,6 +173,11 @@ public static Server startServlet(String contextPath, int port, // Link the context to the server. server.setHandler(context); + // Release the connector socket promptly and gracefully on stop (see + // startWebApp for rationale). + server.setStopAtShutdown(true); + server.setStopTimeout(5000); + server.start(); // Wait for the server to be fully started and ready to accept connections