From 8f5e81c288ed31f59213640f1025e5f85e9220ee Mon Sep 17 00:00:00 2001 From: Andres Felder <81707831+andyfelder16@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:48:07 -0300 Subject: [PATCH 1/4] wire neo4j heuristics into the search fitness function --- .../api/dto/ExtraHeuristicEntryDto.java | 4 +- .../client/java/controller/SutHandler.java | 8 + .../controller/internal/EMController.java | 1 + .../controller/internal/SutController.java | 44 +++++ .../db/neo4j/Neo4jCommandWithDistance.java | 24 +++ .../db/neo4j/Neo4jDistanceWithMetrics.java | 41 ++++ .../internal/db/neo4j/Neo4jHandler.java | 116 +++++++++++ .../internal/db/neo4j/Neo4jHandlerTest.java | 185 ++++++++++++++++++ .../kotlin/org/evomaster/core/EMConfig.kt | 5 + .../enterprise/service/EnterpriseFitness.kt | 37 ++++ .../core/search/service/Statistics.kt | 25 +++ .../search/service/StatisticsNeo4jTest.kt | 22 +++ docs/options.md | 1 + 13 files changed, 511 insertions(+), 2 deletions(-) create mode 100644 client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jCommandWithDistance.java create mode 100644 client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetrics.java create mode 100644 client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java create mode 100644 client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java create mode 100644 core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java index 2a089f3ff3..c01fc19a04 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java @@ -9,9 +9,9 @@ public class ExtraHeuristicEntryDto implements Serializable { /** * The type of extra heuristic. - * Note: for the moment, we only have heuristics on SQL, MONGO, OPENSEARCH and REDIS commands + * Note: for the moment, we only have heuristics on SQL, MONGO, OPENSEARCH, REDIS and NEO4J commands */ - public enum Type {SQL, MONGO, OPENSEARCH, REDIS} + public enum Type {SQL, MONGO, OPENSEARCH, REDIS, NEO4J} /** * Should we try to minimize or maximize the heuristic? diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java index 8758bfee09..2bf8236ece 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java @@ -179,6 +179,14 @@ default void extractRPCSchema(){} default Object getMongoConnection() {return null;} + /** + * @return the Neo4j {@code org.neo4j.driver.Driver} of the SUT, or {@code null} if the SUT does + * not use Neo4j. Returned as {@code Object} and accessed by reflection, so the driver does not + * hard-depend on a specific {@code neo4j-java-driver} version. Used both to read the live graph + * when computing Cypher heuristics and (later) to insert test data. + */ + default Object getNeo4jConnection() {return null;} + default Object getOpenSearchConnection() {return null;} default ReflectionBasedRedisClient getRedisConnection() {return null;} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java index 9d9a05683f..4e7736acc1 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java @@ -386,6 +386,7 @@ public Response runSut(SutRunDto dto, @Context HttpServletRequest httpServletReq noKillSwitch(() -> sutController.initSqlHandler()); noKillSwitch(() -> sutController.registerOrExecuteInitSqlCommandsIfNeeded(true)); noKillSwitch(() -> sutController.initMongoHandler()); + noKillSwitch(() -> sutController.initNeo4jHandler()); noKillSwitch(() -> sutController.initOpenSearchHandler()); noKillSwitch(() -> sutController.initRedisHandler()); } else { diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 16ef32b718..6927612d16 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -34,6 +34,7 @@ import org.evomaster.client.java.sql.SqlScriptRunnerCached; import org.evomaster.client.java.sql.DbSpecification; import org.evomaster.client.java.controller.internal.db.mongo.MongoHandler; +import org.evomaster.client.java.controller.internal.db.neo4j.Neo4jHandler; import org.evomaster.client.java.sql.DbInfoExtractor; import org.evomaster.client.java.sql.internal.SqlHandler; import org.evomaster.client.java.controller.mongo.MongoScriptRunner; @@ -88,6 +89,8 @@ public abstract class SutController implements SutHandler, CustomizationHandler private final MongoHandler mongoHandler = new MongoHandler(); + private final Neo4jHandler neo4jHandler = new Neo4jHandler(); + private final OpenSearchHandler openSearchHandler = new OpenSearchHandler(); private final RedisHandler redisHandler = new RedisHandler(); @@ -349,6 +352,10 @@ public final void initMongoHandler() { } } + public final void initNeo4jHandler() { + neo4jHandler.setNeo4jConnection(getNeo4jConnection()); + } + // TODO: Refactor this initialization methods once Redis and OpenSearch implementations are done public final void initOpenSearchHandler() { // This is needed because the replacement use to get this info occurs during the start of the SUT. @@ -388,6 +395,7 @@ public final boolean doEmploySmartDbClean(){ public final void resetExtraHeuristics() { sqlHandler.reset(); mongoHandler.reset(); + neo4jHandler.reset(); redisHandler.reset(); } @@ -411,6 +419,7 @@ public final ExtraHeuristicsDto computeExtraHeuristics(boolean queryFromDatabase ExtraHeuristicsDto dto = new ExtraHeuristicsDto(); if (isSQLHeuristicsComputationAllowed() || isMongoHeuristicsComputationAllowed() + || isNeo4jHeuristicsComputationAllowed() || isOpenSearchHeuristicsComputationAllowed() || isRedisHeuristicsComputationAllowed()) { List additionalInfoList = getAdditionalInfoList(); @@ -420,6 +429,9 @@ public final ExtraHeuristicsDto computeExtraHeuristics(boolean queryFromDatabase if (isMongoHeuristicsComputationAllowed()) { computeMongoHeuristics(dto, additionalInfoList); } + if (isNeo4jHeuristicsComputationAllowed()) { + computeNeo4jHeuristics(dto, additionalInfoList); + } if (isOpenSearchHeuristicsComputationAllowed()) { computeOpenSearchHeuristics(dto, additionalInfoList); } @@ -438,6 +450,10 @@ private boolean isMongoHeuristicsComputationAllowed() { return mongoHandler.isCalculateHeuristics() || mongoHandler.isExtractMongoExecution(); } + private boolean isNeo4jHeuristicsComputationAllowed() { + return neo4jHandler.isCalculateHeuristics(); + } + private boolean isOpenSearchHeuristicsComputationAllowed() { return openSearchHandler.isCalculateHeuristics(); } @@ -528,6 +544,34 @@ public final void computeMongoHeuristics(ExtraHeuristicsDto dto, List additionalInfoList){ + if(neo4jHandler.isCalculateHeuristics()){ + if(!additionalInfoList.isEmpty()) { + AdditionalInfo last = additionalInfoList.get(additionalInfoList.size() - 1); + last.getNeo4JInfoData().forEach(it -> { + try { + neo4jHandler.handle(it); + } catch (Exception e){ + SimpleLogger.error("FAILED TO HANDLE NEO4J COMMAND"); + assert false; + } + }); + } + + neo4jHandler.getEvaluatedCommands().stream() + .map(p -> + new ExtraHeuristicEntryDto( + ExtraHeuristicEntryDto.Type.NEO4J, + ExtraHeuristicEntryDto.Objective.MINIMIZE_TO_ZERO, + p.getNeo4jCommand(), + p.getNeo4jDistanceWithMetrics().getNeo4jDistance(), + p.getNeo4jDistanceWithMetrics().getNumberOfEvaluatedNodes(), + p.getNeo4jDistanceWithMetrics().isNeo4jDistanceEvaluationFailure() + )) + .forEach(h -> dto.heuristics.add(h)); + } + } + public final void computeOpenSearchHeuristics(ExtraHeuristicsDto dto, List additionalInfoList) { if (openSearchHandler.isCalculateHeuristics()) { if (!additionalInfoList.isEmpty()) { diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jCommandWithDistance.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jCommandWithDistance.java new file mode 100644 index 0000000000..f6e7f189c5 --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jCommandWithDistance.java @@ -0,0 +1,24 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +/** + * Pairs a captured Cypher query with its computed distance to being satisfied by the live graph. + */ +public class Neo4jCommandWithDistance { + + private final String neo4jCommand; + + private final Neo4jDistanceWithMetrics neo4jDistanceWithMetrics; + + public Neo4jCommandWithDistance(String neo4jCommand, Neo4jDistanceWithMetrics neo4jDistanceWithMetrics) { + this.neo4jCommand = neo4jCommand; + this.neo4jDistanceWithMetrics = neo4jDistanceWithMetrics; + } + + public String getNeo4jCommand() { + return neo4jCommand; + } + + public Neo4jDistanceWithMetrics getNeo4jDistanceWithMetrics() { + return neo4jDistanceWithMetrics; + } +} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetrics.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetrics.java new file mode 100644 index 0000000000..fc96a77269 --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetrics.java @@ -0,0 +1,41 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +/** + * The result of scoring one captured Cypher query against the live graph: the distance to satisfying + * it ({@code 1 - ofTrue}, in {@code [0,1]}, 0 meaning satisfied), how many graph nodes were available + * when scoring, and whether the evaluation failed (e.g. the query could not be parsed). + */ +public class Neo4jDistanceWithMetrics { + + private final double neo4jDistance; + + private final int numberOfEvaluatedNodes; + + private final boolean neo4jDistanceEvaluationFailure; + + public Neo4jDistanceWithMetrics(double neo4jDistance, int numberOfEvaluatedNodes, + boolean neo4jDistanceEvaluationFailure) { + if (neo4jDistance < 0) { + throw new IllegalArgumentException("neo4jDistance must be non-negative but value is " + neo4jDistance); + } + if (numberOfEvaluatedNodes < 0) { + throw new IllegalArgumentException( + "numberOfEvaluatedNodes must be non-negative but value is " + numberOfEvaluatedNodes); + } + this.neo4jDistance = neo4jDistance; + this.numberOfEvaluatedNodes = numberOfEvaluatedNodes; + this.neo4jDistanceEvaluationFailure = neo4jDistanceEvaluationFailure; + } + + public double getNeo4jDistance() { + return neo4jDistance; + } + + public int getNumberOfEvaluatedNodes() { + return numberOfEvaluatedNodes; + } + + public boolean isNeo4jDistanceEvaluationFailure() { + return neo4jDistanceEvaluationFailure; + } +} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java new file mode 100644 index 0000000000..0d45cc1352 --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java @@ -0,0 +1,116 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +import org.evomaster.client.java.controller.neo4j.data.Neo4jGraph; +import org.evomaster.client.java.controller.neo4j.heuristics.Neo4jHeuristicsCalculator; +import org.evomaster.client.java.controller.neo4j.operations.MatchOperation; +import org.evomaster.client.java.controller.neo4j.parser.CypherParser; +import org.evomaster.client.java.controller.neo4j.parser.CypherParserException; +import org.evomaster.client.java.controller.neo4j.parser.CypherParserFactory; +import org.evomaster.client.java.controller.internal.TaintHandlerExecutionTracer; +import org.evomaster.client.java.instrumentation.Neo4JRunCommand; +import org.evomaster.client.java.utils.SimpleLogger; + +import java.util.ArrayList; +import java.util.List; + +/** + * Acts upon Cypher queries executed by the SUT (captured as {@link Neo4JRunCommand}s): for each + * captured query it computes how close the live graph is to satisfying it, as a distance to minimize. + * Only MATCH queries are scored; a query that does not parse as a MATCH (e.g. a write) is skipped. + */ +public class Neo4jHandler { + + /** Cypher queries captured from {@code Session.run}, pending evaluation. */ + private final List operations; + + /** The computed heuristics, one per scored query. */ + private final List commandsWithDistances; + + /** Whether to compute heuristics based on execution or not. */ + private volatile boolean calculateHeuristics; + + /** + * The SUT's {@code org.neo4j.driver.Driver}, kept as an {@code Object} and used by reflection so + * we do not hard-depend on a specific driver version. {@code null} when the SUT does not use Neo4j. + */ + private Object neo4jConnection = null; + + private final CypherParser parser = CypherParserFactory.buildParser(); + + private final Neo4jHeuristicsCalculator calculator = + new Neo4jHeuristicsCalculator(new TaintHandlerExecutionTracer()); + private final Neo4jGraphReader graphReader = new Neo4jGraphReader(); + + public Neo4jHandler() { + operations = new ArrayList<>(); + commandsWithDistances = new ArrayList<>(); + calculateHeuristics = true; + } + + public void reset() { + operations.clear(); + commandsWithDistances.clear(); + } + + public boolean isCalculateHeuristics() { + return calculateHeuristics; + } + + public void setCalculateHeuristics(boolean calculateHeuristics) { + this.calculateHeuristics = calculateHeuristics; + } + + public void setNeo4jConnection(Object neo4jConnection) { + this.neo4jConnection = neo4jConnection; + } + + public void handle(Neo4JRunCommand info) { + if (calculateHeuristics && info.getQuery() != null) { + operations.add(info); + } + } + + public List getEvaluatedCommands() { + + if (!calculateHeuristics || neo4jConnection == null || operations.isEmpty()) { + operations.clear(); + return commandsWithDistances; + } + + Neo4jGraph graph; + try { + graph = graphReader.read(neo4jConnection); + } catch (Exception e) { + SimpleLogger.uniqueWarn("Failed to read the Neo4j graph to compute heuristics: " + e.getMessage()); + operations.clear(); + return commandsWithDistances; + } + + for (Neo4JRunCommand op : operations) { + String query = op.getQuery(); + if (query == null) { + continue; + } + final MatchOperation parsedQuery; + try { + parsedQuery = parser.parse(query); + } catch (CypherParserException e) { + SimpleLogger.uniqueWarn("Failed to parse Cypher query for Neo4j heuristics: " + e.getMessage()); + continue; + } + + Neo4jDistanceWithMetrics metrics; + try { + double distance = calculator.computeDistance(parsedQuery, graph); + metrics = new Neo4jDistanceWithMetrics(distance, graph.nodeCount(), false); + } catch (Exception e) { + SimpleLogger.uniqueWarn("Failed to compute Neo4j heuristic for query: " + query); + metrics = new Neo4jDistanceWithMetrics(1.0, graph.nodeCount(), true); + } + commandsWithDistances.add(new Neo4jCommandWithDistance(query, metrics)); + } + + operations.clear(); + return commandsWithDistances; + } +} diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java new file mode 100644 index 0000000000..ca0c877ef0 --- /dev/null +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java @@ -0,0 +1,185 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +import org.evomaster.client.java.instrumentation.Neo4JRunCommand; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Exercises {@link Neo4jHandler} and {@link Neo4jGraphReader} end-to-end against a hand-rolled fake + * driver that exposes the same method names the reader reflects over ({@code session}/{@code run}/ + * {@code list}/{@code get}/{@code asString}/{@code asList}/{@code asMap}/{@code close}). This validates + * the reflection plumbing and the parse→score→DTO pipeline without needing a live Neo4j instance. + *

+ * Integers are returned as {@code Long} to mimic the real driver's value mapping. + */ +class Neo4jHandlerTest { + + private static final String MATCH_QUERY = + "MATCH (a:Person {age: 25})-[r:KNOWS]->(b:Person) WHERE b.age > 30 RETURN b"; + + private FakeDriver example1Driver() { + List nodes = Arrays.asList( + nodeRecord("n1", labels("Person"), props("age", 25L, "name", "Ana")), + nodeRecord("n2", labels("Person"), props("age", 28L, "name", "Luis")), + nodeRecord("n3", labels("Animal"), props("age", 5L, "name", "Rex")), + nodeRecord("n4", labels("Person"), props("age", 40L, "name", "Carlos"))); + List rels = Arrays.asList( + relRecord("e1", "KNOWS", "n1", "n2"), + relRecord("e2", "LIKES", "n1", "n3"), + relRecord("e3", "KNOWS", "n3", "n4")); + return new FakeDriver(nodes, rels); + } + + @Test + void testScoresMatchQueryAgainstLiveGraph() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(example1Driver()); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + List evaluated = handler.getEvaluatedCommands(); + + assertEquals(1, evaluated.size()); + Neo4jCommandWithDistance result = evaluated.get(0); + assertEquals(MATCH_QUERY, result.getNeo4jCommand()); + assertFalse(result.getNeo4jDistanceWithMetrics().isNeo4jDistanceEvaluationFailure()); + assertEquals(4, result.getNeo4jDistanceWithMetrics().getNumberOfEvaluatedNodes()); + // distance = 1 - ofTrue; ofTrue ≈ 0.939 → distance ≈ 0.061. + assertEquals(0.061, result.getNeo4jDistanceWithMetrics().getNeo4jDistance(), 0.005); + } + + @Test + void testNonMatchQueryIsSkipped() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(example1Driver()); + handler.handle(new Neo4JRunCommand("CREATE (n:Person {name: 'Zoe'})", null, true, 1)); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + List evaluated = handler.getEvaluatedCommands(); + // The write query does not parse as a MATCH and is skipped; only the read query is scored. + assertEquals(1, evaluated.size()); + assertEquals(MATCH_QUERY, evaluated.get(0).getNeo4jCommand()); + } + + @Test + void testNoConnectionYieldsNoHeuristics() { + Neo4jHandler handler = new Neo4jHandler(); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + assertTrue(handler.getEvaluatedCommands().isEmpty()); + } + + // --- fake Neo4j driver (only the methods the reader reflects over) --------------------------- + + public static final class FakeDriver { + private final List nodes; + private final List rels; + + FakeDriver(List nodes, List rels) { + this.nodes = nodes; + this.rels = rels; + } + + public FakeSession session() { + return new FakeSession(nodes, rels); + } + } + + public static final class FakeSession { + private final List nodes; + private final List rels; + + FakeSession(List nodes, List rels) { + this.nodes = nodes; + this.rels = rels; + } + + public FakeResult run(String query) { + return new FakeResult(query.contains("labels(n)") ? nodes : rels); + } + + public void close() { + } + } + + public static final class FakeResult { + private final List records; + + FakeResult(List records) { + this.records = records; + } + + public List list() { + return records; + } + } + + public static final class FakeRecord { + private final Map fields; + + FakeRecord(Map fields) { + this.fields = fields; + } + + public FakeValue get(String key) { + return new FakeValue(fields.get(key)); + } + } + + public static final class FakeValue { + private final Object value; + + FakeValue(Object value) { + this.value = value; + } + + public String asString() { + return (String) value; + } + + @SuppressWarnings("unchecked") + public List asList() { + return (List) value; + } + + @SuppressWarnings("unchecked") + public Map asMap() { + return (Map) value; + } + } + + private static FakeRecord nodeRecord(String id, List labels, Map props) { + Map f = new LinkedHashMap<>(); + f.put("id", id); + f.put("labels", labels); + f.put("props", props); + return new FakeRecord(f); + } + + private static FakeRecord relRecord(String id, String type, String src, String tgt) { + Map f = new LinkedHashMap<>(); + f.put("id", id); + f.put("type", type); + f.put("src", src); + f.put("tgt", tgt); + f.put("props", new LinkedHashMap()); + return new FakeRecord(f); + } + + private static List labels(String... ls) { + return new ArrayList<>(Arrays.asList(ls)); + } + + private static Map props(Object... kv) { + Map m = new LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put((String) kv[i], kv[i + 1]); + } + return m; + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 651d797745..f1b5952b35 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1947,6 +1947,11 @@ class EMConfig { @DependsOnFalseFor("blackBox") var heuristicsForRedis = false + @Experimental + @Cfg("Tracking of Neo4j commands to improve test generation") + @DependsOnFalseFor("blackBox") + var heuristicsForNeo4j = false + @Cfg("Enable extracting SQL execution info") @DependsOnFalseFor("blackBox") var extractSqlExecutionInfo = true diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt index 93fd33c7c7..0d9ce2b759 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt @@ -378,6 +378,10 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual handleRedisHeuristics(dto, fv) } + if (configuration.heuristicsForNeo4j) { + handleNeo4jHeuristics(dto, fv) + } + if (configuration.extractRedisExecutionInfo) { for (i in 0 until dto.extraHeuristics.size) { val extra = dto.extraHeuristics[i] @@ -493,4 +497,37 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual } } } + + private fun handleNeo4jHeuristics(dto: TestResultsDto, fv: FitnessValue) { + for (i in 0 until dto.extraHeuristics.size) { + + val extra = dto.extraHeuristics[i] + + extraHeuristicsLogger.writeHeuristics(extra.heuristics, i) + + val toMinimize = extra.heuristics + .filter { + it != null + && it.objective == ExtraHeuristicEntryDto.Objective.MINIMIZE_TO_ZERO + && it.type == ExtraHeuristicEntryDto.Type.NEO4J + }.map { it.value } + .toList() + + if (toMinimize.isNotEmpty()) { + fv.setExtraToMinimize(i, toMinimize) + } + + extra.heuristics + .filterNotNull().forEach { + if (it.type == ExtraHeuristicEntryDto.Type.NEO4J) { + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(it.numberOfEvaluatedRecords) + if (it.extraHeuristicEvaluationFailure) { + statistics.reportNeo4jHeuristicEvaluationFailure() + } else { + statistics.reportNeo4jHeuristicEvaluationSuccess() + } + } + } + } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index b91b4846c5..97ce0d2642 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -114,6 +114,11 @@ class Statistics : SearchListener { private var redisHeuristicEvaluationFailureCount = 0 private val redisDocumentsAverageCalculator = IncrementalAverage() + // neo4j heuristic evaluation statistic + private var neo4jHeuristicEvaluationSuccessCount = 0 + private var neo4jHeuristicEvaluationFailureCount = 0 + private val neo4jNodesAverageCalculator = IncrementalAverage() + class Pair(val header: String, val element: String) @@ -210,6 +215,10 @@ class Statistics : SearchListener { redisDocumentsAverageCalculator.addValue(numberOfEvaluatedDocuments) } + fun reportNumberOfEvaluatedNodesForNeo4jHeuristic(numberOfEvaluatedNodes: Int) { + neo4jNodesAverageCalculator.addValue(numberOfEvaluatedNodes) + } + fun reportSqlParsingFailures(numberOfParsingFailures: Int) { if (numberOfParsingFailures<0) { throw IllegalArgumentException("Invalid number of parsing failures: $numberOfParsingFailures") @@ -301,6 +310,14 @@ class Statistics : SearchListener { internal fun getSqlZ3CacheHitCount() = sqlZ3CacheHitCount internal fun getSqlZ3CacheMissCount() = sqlZ3CacheMissCount + fun reportNeo4jHeuristicEvaluationSuccess() { + neo4jHeuristicEvaluationSuccessCount++ + } + + fun reportNeo4jHeuristicEvaluationFailure() { + neo4jHeuristicEvaluationFailureCount++ + } + fun getMongoHeuristicsEvaluationCount(): Int = mongoHeuristicEvaluationSuccessCount + mongoHeuristicEvaluationFailureCount fun getSqlHeuristicsEvaluationCount(): Int = sqlHeuristicEvaluationSuccessCount + sqlHeuristicEvaluationFailureCount @@ -313,6 +330,10 @@ class Statistics : SearchListener { fun averageNumberOfEvaluatedDocumentsForRedisHeuristics(): Double = redisDocumentsAverageCalculator.mean + fun getNeo4jHeuristicsEvaluationCount(): Int = neo4jHeuristicEvaluationSuccessCount + neo4jHeuristicEvaluationFailureCount + + fun averageNumberOfEvaluatedNodesForNeo4jHeuristics(): Double = neo4jNodesAverageCalculator.mean + override fun newActionsEvaluated(n: Int) { if(!epc.isInSearch()){ @@ -477,6 +498,10 @@ class Statistics : SearchListener { add(Pair("sqlZ3AvgSmtlibSizeBytes", "%.1f".format(sqlZ3SmtlibSizeBytes.mean))) } + // statistics info for Neo4j Heuristics + add(Pair("averageNumberOfEvaluatedNodesForNeo4jHeuristics","${averageNumberOfEvaluatedNodesForNeo4jHeuristics()}")) + add(Pair("neo4jHeuristicsEvaluationCount","${getNeo4jHeuristicsEvaluationCount()}")) + for(phase in ExecutionPhaseController.Phase.entries){ add(Pair("phase_${phase.name}", "${epc.getPhaseDurationInSeconds(phase)}")) } diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt new file mode 100644 index 0000000000..12abc78188 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt @@ -0,0 +1,22 @@ +package org.evomaster.core.search.service + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class StatisticsNeo4jTest { + + @Test + fun testNeo4jHeuristicsAverage() { + val statistics = Statistics() + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(10) + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(20) + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(30) + + repeat(3) { + statistics.reportNeo4jHeuristicEvaluationSuccess() + } + + assertEquals(3, statistics.getNeo4jHeuristicsEvaluationCount()) + assertEquals((10 + 20 + 30).toDouble() / 3, statistics.averageNumberOfEvaluatedNodesForNeo4jHeuristics()) + } +} diff --git a/docs/options.md b/docs/options.md index f51c573d3f..51530a1c15 100644 --- a/docs/options.md +++ b/docs/options.md @@ -303,6 +303,7 @@ There are 3 types of options: |`generateRedisData`| __Boolean__. Enable EvoMaster to generate Redis data with direct accesses to the database. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`generateSqlDataWithZ3`| __Boolean__. Enable EvoMaster to generate SQL data with direct accesses to the database. Use the Z3 SMT solver. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`handleFlakiness`| __Boolean__. Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing. Note that flakiness is now supported only for fuzzing REST APIs. *Default value*: `false`.| +|`heuristicsForNeo4j`| __Boolean__. Tracking of Neo4j commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`heuristicsForRedis`| __Boolean__. Tracking of Redis commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`heuristicsForSQLAdvanced`| __Boolean__. If using SQL heuristics, enable more advanced version. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`httpOracles`| __Boolean__. Extra checks on HTTP properties in returned responses, used as automated oracles to detect faults. *Default value*: `false`.| From f67a9f173953b831a6e91f294d00a14c66b0146b Mon Sep 17 00:00:00 2001 From: Andres Felder <81707831+andyfelder16@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:36:26 -0300 Subject: [PATCH 2/4] document the neo4j heuristics handler and drop the redundant prefixes from its result types --- .../controller/internal/SutController.java | 14 ++--- .../db/neo4j/Neo4jCommandWithDistance.java | 33 ++++++++---- .../db/neo4j/Neo4jDistanceWithMetrics.java | 52 +++++++++++-------- .../internal/db/neo4j/Neo4jHandler.java | 36 +++++++++++-- .../internal/db/neo4j/Neo4jHandlerTest.java | 52 +++++++++++++++---- .../core/search/service/Statistics.kt | 9 +++- .../search/service/StatisticsNeo4jTest.kt | 22 -------- .../core/search/service/StatisticsTest.kt | 16 ++++++ 8 files changed, 157 insertions(+), 77 deletions(-) delete mode 100644 core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index acff10e0d9..ac7563a2bf 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -575,15 +575,15 @@ public final void computeNeo4jHeuristics(ExtraHeuristicsDto dto, List new ExtraHeuristicEntryDto( ExtraHeuristicEntryDto.Type.NEO4J, ExtraHeuristicEntryDto.Objective.MINIMIZE_TO_ZERO, - p.getNeo4jCommand(), - p.getNeo4jDistanceWithMetrics().getNeo4jDistance(), - p.getNeo4jDistanceWithMetrics().getNumberOfEvaluatedNodes(), - p.getNeo4jDistanceWithMetrics().isNeo4jDistanceEvaluationFailure() + p.getCommand(), + p.getDistanceWithMetrics().getDistance(), + p.getDistanceWithMetrics().getNumberOfEvaluatedNodes(), + p.getDistanceWithMetrics().isEvaluationFailure() )) .forEach(h -> dto.heuristics.add(h)); } @@ -608,8 +608,8 @@ public final void computeOpenSearchHeuristics(ExtraHeuristicsDto dto, List 1.0d || Double.isNaN(distance)) { + throw new IllegalArgumentException("distance must be between 0 and 1, but was " + distance); } if (numberOfEvaluatedNodes < 0) { - throw new IllegalArgumentException( - "numberOfEvaluatedNodes must be non-negative but value is " + numberOfEvaluatedNodes); + throw new IllegalArgumentException("numberOfEvaluatedNodes must be non-negative"); } - this.neo4jDistance = neo4jDistance; + this.distance = distance; this.numberOfEvaluatedNodes = numberOfEvaluatedNodes; - this.neo4jDistanceEvaluationFailure = neo4jDistanceEvaluationFailure; + this.evaluationFailure = evaluationFailure; } - public double getNeo4jDistance() { - return neo4jDistance; + /** + * @return normalized distance to satisfying the query, 0 meaning satisfied + */ + public double getDistance() { + return distance; } + /** + * @return number of graph nodes considered + */ public int getNumberOfEvaluatedNodes() { return numberOfEvaluatedNodes; } - public boolean isNeo4jDistanceEvaluationFailure() { - return neo4jDistanceEvaluationFailure; + /** + * @return whether the evaluation failed + */ + public boolean isEvaluationFailure() { + return evaluationFailure; } } diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java index 0d45cc1352..e78747b574 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java @@ -41,36 +41,67 @@ public class Neo4jHandler { new Neo4jHeuristicsCalculator(new TaintHandlerExecutionTracer()); private final Neo4jGraphReader graphReader = new Neo4jGraphReader(); + /** + * Creates a handler with heuristic calculation enabled. + */ public Neo4jHandler() { operations = new ArrayList<>(); commandsWithDistances = new ArrayList<>(); calculateHeuristics = true; } + /** + * Clears data collected for the current action. + */ public void reset() { operations.clear(); commandsWithDistances.clear(); } + /** + * @return whether Neo4j heuristic calculation is enabled + */ public boolean isCalculateHeuristics() { return calculateHeuristics; } + /** + * Enables or disables Neo4j heuristic calculation. + * + * @param calculateHeuristics new calculation state + */ public void setCalculateHeuristics(boolean calculateHeuristics) { this.calculateHeuristics = calculateHeuristics; } + /** + * Sets the driver used to read the live graph. + * + * @param neo4jConnection the SUT's {@code org.neo4j.driver.Driver}, or {@code null} if it has none + */ public void setNeo4jConnection(Object neo4jConnection) { this.neo4jConnection = neo4jConnection; } + /** + * Registers one intercepted Cypher query. + * + * @param info intercepted query + */ public void handle(Neo4JRunCommand info) { if (calculateHeuristics && info.getQuery() != null) { operations.add(info); } } - public List getEvaluatedCommands() { + /** + * Evaluates all registered queries against a single snapshot of the graph, and consumes them. + * The snapshot is read once per action rather than per query, since the SUT is not running while + * the heuristics are computed. + * + * @return evaluated queries for the current action + */ + public List getEvaluatedNeo4jCommands() { if (!calculateHeuristics || neo4jConnection == null || operations.isEmpty()) { operations.clear(); @@ -88,9 +119,6 @@ public List getEvaluatedCommands() { for (Neo4JRunCommand op : operations) { String query = op.getQuery(); - if (query == null) { - continue; - } final MatchOperation parsedQuery; try { parsedQuery = parser.parse(query); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java index ca0c877ef0..ea53f4f1eb 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java @@ -15,7 +15,8 @@ * Exercises {@link Neo4jHandler} and {@link Neo4jGraphReader} end-to-end against a hand-rolled fake * driver that exposes the same method names the reader reflects over ({@code session}/{@code run}/ * {@code list}/{@code get}/{@code asString}/{@code asList}/{@code asMap}/{@code close}). This validates - * the reflection plumbing and the parse→score→DTO pipeline without needing a live Neo4j instance. + * the reflection plumbing and the whole parse, score and report pipeline without needing a live Neo4j + * instance. *

* Integers are returned as {@code Long} to mimic the real driver's value mapping. */ @@ -43,15 +44,15 @@ void testScoresMatchQueryAgainstLiveGraph() { handler.setNeo4jConnection(example1Driver()); handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); - List evaluated = handler.getEvaluatedCommands(); + List evaluated = handler.getEvaluatedNeo4jCommands(); assertEquals(1, evaluated.size()); Neo4jCommandWithDistance result = evaluated.get(0); - assertEquals(MATCH_QUERY, result.getNeo4jCommand()); - assertFalse(result.getNeo4jDistanceWithMetrics().isNeo4jDistanceEvaluationFailure()); - assertEquals(4, result.getNeo4jDistanceWithMetrics().getNumberOfEvaluatedNodes()); - // distance = 1 - ofTrue; ofTrue ≈ 0.939 → distance ≈ 0.061. - assertEquals(0.061, result.getNeo4jDistanceWithMetrics().getNeo4jDistance(), 0.005); + assertEquals(MATCH_QUERY, result.getCommand()); + assertFalse(result.getDistanceWithMetrics().isEvaluationFailure()); + assertEquals(4, result.getDistanceWithMetrics().getNumberOfEvaluatedNodes()); + // distance is 1 - ofTrue, and here ofTrue is about 0.939. + assertEquals(0.061, result.getDistanceWithMetrics().getDistance(), 0.005); } @Test @@ -61,20 +62,41 @@ void testNonMatchQueryIsSkipped() { handler.handle(new Neo4JRunCommand("CREATE (n:Person {name: 'Zoe'})", null, true, 1)); handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); - List evaluated = handler.getEvaluatedCommands(); + List evaluated = handler.getEvaluatedNeo4jCommands(); // The write query does not parse as a MATCH and is skipped; only the read query is scored. assertEquals(1, evaluated.size()); - assertEquals(MATCH_QUERY, evaluated.get(0).getNeo4jCommand()); + assertEquals(MATCH_QUERY, evaluated.get(0).getCommand()); } @Test void testNoConnectionYieldsNoHeuristics() { Neo4jHandler handler = new Neo4jHandler(); handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); - assertTrue(handler.getEvaluatedCommands().isEmpty()); + assertTrue(handler.getEvaluatedNeo4jCommands().isEmpty()); } - // --- fake Neo4j driver (only the methods the reader reflects over) --------------------------- + @Test + void testHeuristicsAreNotComputedWhenDisabled() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(example1Driver()); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + handler.setCalculateHeuristics(false); + + // Registered while enabled, so it is the check before evaluating that has to drop the query. + assertTrue(handler.getEvaluatedNeo4jCommands().isEmpty()); + } + + @Test + void testAnUnreadableGraphYieldsNoHeuristicsInsteadOfFailing() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(new BrokenDriver()); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + // The SUT must keep running even if its driver cannot be queried. + assertTrue(handler.getEvaluatedNeo4jCommands().isEmpty()); + } + + // Fake Neo4j driver, exposing only the methods the reader reflects over. public static final class FakeDriver { private final List nodes; @@ -90,6 +112,14 @@ public FakeSession session() { } } + /** A driver whose session cannot be opened, standing in for a Neo4j that is down. */ + public static final class BrokenDriver { + + public Object session() { + throw new IllegalStateException("no connection to the database"); + } + } + public static final class FakeSession { private final List nodes; private final List rels; diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index d27c3fc3cf..fc034570de 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -252,6 +252,7 @@ class Statistics : SearchListener { dynamoDbItemsAverageCalculator.addValue(numberOfEvaluatedItems) } + /** Records the number of nodes inspected by one Neo4j heuristic evaluation. */ fun reportNumberOfEvaluatedNodesForNeo4jHeuristic(numberOfEvaluatedNodes: Int) { neo4jNodesAverageCalculator.addValue(numberOfEvaluatedNodes) } @@ -381,16 +382,19 @@ class Statistics : SearchListener { fun reportSqlZ3CacheHit() { sqlZ3CacheHitCount++ } + // Exposed for tests: verify the memoization accounting invariant // (seen == cacheHits + cacheMisses). internal fun getSqlZ3QueriesSeenCount() = sqlZ3QueriesSeenCount internal fun getSqlZ3CacheHitCount() = sqlZ3CacheHitCount internal fun getSqlZ3CacheMissCount() = sqlZ3CacheMissCount + /** Records one successful Neo4j heuristic evaluation. */ fun reportNeo4jHeuristicEvaluationSuccess() { neo4jHeuristicEvaluationSuccessCount++ } + /** Records one failed Neo4j heuristic evaluation. */ fun reportNeo4jHeuristicEvaluationFailure() { neo4jHeuristicEvaluationFailureCount++ } @@ -423,8 +427,11 @@ class Statistics : SearchListener { /** Returns the average number of items inspected by DynamoDB heuristics. */ fun averageNumberOfEvaluatedItemsForDynamoDbHeuristics(): Double = dynamoDbItemsAverageCalculator.mean - fun getNeo4jHeuristicsEvaluationCount(): Int = neo4jHeuristicEvaluationSuccessCount + neo4jHeuristicEvaluationFailureCount + /** Returns the total number of Neo4j heuristic evaluations. */ + fun getNeo4jHeuristicsEvaluationCount(): Int = + neo4jHeuristicEvaluationSuccessCount + neo4jHeuristicEvaluationFailureCount + /** Returns the average number of nodes inspected by Neo4j heuristics. */ fun averageNumberOfEvaluatedNodesForNeo4jHeuristics(): Double = neo4jNodesAverageCalculator.mean override fun newActionsEvaluated(n: Int) { diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt deleted file mode 100644 index 12abc78188..0000000000 --- a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsNeo4jTest.kt +++ /dev/null @@ -1,22 +0,0 @@ -package org.evomaster.core.search.service - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test - -class StatisticsNeo4jTest { - - @Test - fun testNeo4jHeuristicsAverage() { - val statistics = Statistics() - statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(10) - statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(20) - statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(30) - - repeat(3) { - statistics.reportNeo4jHeuristicEvaluationSuccess() - } - - assertEquals(3, statistics.getNeo4jHeuristicsEvaluationCount()) - assertEquals((10 + 20 + 30).toDouble() / 3, statistics.averageNumberOfEvaluatedNodesForNeo4jHeuristics()) - } -} diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt index 26c52338f1..1d2f4c4209 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt @@ -52,6 +52,22 @@ class StatisticsTest { assertEquals(20.0, statistics.averageNumberOfEvaluatedItemsForDynamoDbHeuristics()) } + @Test + fun testNeo4jHeuristicsAverage() { + val statistics = Statistics() + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(10) + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(20) + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(30) + + repeat(2) { + statistics.reportNeo4jHeuristicEvaluationSuccess() + } + statistics.reportNeo4jHeuristicEvaluationFailure() + + assertEquals(3, statistics.getNeo4jHeuristicsEvaluationCount()) + assertEquals(20.0, statistics.averageNumberOfEvaluatedNodesForNeo4jHeuristics()) + } + @Test fun testSqlZ3CacheAccountingInvariant() { val statistics = Statistics() From 15974b179e4214834b91972fe292daed037333b1 Mon Sep 17 00:00:00 2001 From: Andres Felder <81707831+andyfelder16@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:11:09 -0300 Subject: [PATCH 3/4] report the calculator's maximum distance when a neo4j heuristic fails to evaluate --- .../internal/db/neo4j/Neo4jHandler.java | 2 +- .../heuristics/Neo4jHeuristicsCalculator.java | 9 +++++++- .../internal/db/neo4j/Neo4jHandlerTest.java | 22 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java index e78747b574..3f45e034c8 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java @@ -133,7 +133,7 @@ public List getEvaluatedNeo4jCommands() { metrics = new Neo4jDistanceWithMetrics(distance, graph.nodeCount(), false); } catch (Exception e) { SimpleLogger.uniqueWarn("Failed to compute Neo4j heuristic for query: " + query); - metrics = new Neo4jDistanceWithMetrics(1.0, graph.nodeCount(), true); + metrics = new Neo4jDistanceWithMetrics(Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE, graph.nodeCount(), true); } commandsWithDistances.add(new Neo4jCommandWithDistance(query, metrics)); } diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java index 2b65e1bc93..5d42fbc359 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java @@ -37,6 +37,13 @@ public class Neo4jHeuristicsCalculator { */ public static final double C = DistanceHelper.H_NOT_NULL; + /** + * Largest distance {@link #computeDistance} can report: the query matched nothing at all. A caller that + * cannot compute a distance (for example when the evaluation throws) reports this value, so a failure + * never looks closer to satisfied than a genuine miss. + */ + public static final double MAX_NEO4J_DISTANCE = 1.0d; + private final Neo4jStructuralMatcher matcher = new Neo4jStructuralMatcher(); private final Neo4jConditionEvaluator evaluator; @@ -66,7 +73,7 @@ public Truthness computeHeuristic(MatchOperation query, Neo4jGraph graph) { /** * Converts a heuristic to the distance form: {@code 1 - ofTrue}, in - * {@code [0,1]}, where 0 means the query is satisfied. + * {@code [0, MAX_NEO4J_DISTANCE]}, where 0 means the query is satisfied. */ public double computeDistance(MatchOperation query, Neo4jGraph graph) { Truthness heuristic = computeHeuristic(query, graph); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java index ea53f4f1eb..b101431523 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java @@ -1,5 +1,6 @@ package org.evomaster.client.java.controller.internal.db.neo4j; +import org.evomaster.client.java.controller.neo4j.heuristics.Neo4jHeuristicsCalculator; import org.evomaster.client.java.instrumentation.Neo4JRunCommand; import org.junit.jupiter.api.Test; @@ -86,6 +87,27 @@ void testHeuristicsAreNotComputedWhenDisabled() { assertTrue(handler.getEvaluatedNeo4jCommands().isEmpty()); } + @Test + void testAFailedEvaluationReportsTheMaximumDistance() { + // A relationship whose source node is not in the graph: the graph reads fine, but scoring the query + // against it fails, and that failure must never look closer to satisfied than a genuine miss. + List nodes = Arrays.asList( + nodeRecord("n1", labels("Person"), props("age", 25L))); + List rels = Arrays.asList( + relRecord("e1", "KNOWS", "ghost", "n1")); + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(new FakeDriver(nodes, rels)); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + List evaluated = handler.getEvaluatedNeo4jCommands(); + + assertEquals(1, evaluated.size()); + Neo4jDistanceWithMetrics metrics = evaluated.get(0).getDistanceWithMetrics(); + assertTrue(metrics.isEvaluationFailure()); + assertEquals(Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE, metrics.getDistance(), 0.0d); + assertEquals(1, metrics.getNumberOfEvaluatedNodes()); + } + @Test void testAnUnreadableGraphYieldsNoHeuristicsInsteadOfFailing() { Neo4jHandler handler = new Neo4jHandler(); From 1bfc0e3fcf43b4e2eb0899a61b10268b67d570a9 Mon Sep 17 00:00:00 2001 From: Andres Felder <81707831+andyfelder16@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:27:51 -0300 Subject: [PATCH 4/4] a failed neo4j heuristic now reports Double.MAX_VALUE, dropping the >1 check on the distance --- .../controller/internal/SutController.java | 2 +- .../db/neo4j/Neo4jDistanceWithMetrics.java | 25 +++++++++++---- .../internal/db/neo4j/Neo4jHandler.java | 3 +- .../heuristics/Neo4jHeuristicsCalculator.java | 11 +++---- .../neo4j/Neo4jDistanceWithMetricsTest.java | 31 +++++++++++++++++++ .../internal/db/neo4j/Neo4jHandlerTest.java | 3 +- 6 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetricsTest.java diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index ac7563a2bf..053b1a8e8b 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -569,7 +569,7 @@ public final void computeNeo4jHeuristics(ExtraHeuristicsDto dto, List + * The distance is {@code 1 - ofTrue} of the {@code Truthness} computed by the heuristics calculator, so a + * computed value lies in {@code [0,1]} by construction, with 0 meaning the query is satisfied. A failed + * evaluation carries {@link Neo4jHeuristicsCalculator#MAX_NEO4J_DISTANCE}. */ public final class Neo4jDistanceWithMetrics { @@ -12,24 +19,30 @@ public final class Neo4jDistanceWithMetrics { /** * Creates a Neo4j heuristic result. * - * @param distance normalized distance to satisfying the query, 0 meaning satisfied + * @param distance distance to satisfying the query, 0 meaning satisfied * @param numberOfEvaluatedNodes number of graph nodes considered - * @param evaluationFailure whether the evaluation failed + * @param evaluationFailure whether the evaluation failed, in which case the distance must be + * {@link Neo4jHeuristicsCalculator#MAX_NEO4J_DISTANCE} */ public Neo4jDistanceWithMetrics(double distance, int numberOfEvaluatedNodes, boolean evaluationFailure) { - if (distance < 0.0d || distance > 1.0d || Double.isNaN(distance)) { - throw new IllegalArgumentException("distance must be between 0 and 1, but was " + distance); + if (distance < 0.0d || Double.isNaN(distance)) { + throw new IllegalArgumentException("distance must be non-negative, but was " + distance); } if (numberOfEvaluatedNodes < 0) { throw new IllegalArgumentException("numberOfEvaluatedNodes must be non-negative"); } + if (evaluationFailure && distance != Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE) { + throw new IllegalArgumentException( + "a failed Neo4j distance computation cannot have a value different than MAX_NEO4J_DISTANCE"); + } this.distance = distance; this.numberOfEvaluatedNodes = numberOfEvaluatedNodes; this.evaluationFailure = evaluationFailure; } /** - * @return normalized distance to satisfying the query, 0 meaning satisfied + * @return distance to satisfying the query, 0 meaning satisfied, + * {@link Neo4jHeuristicsCalculator#MAX_NEO4J_DISTANCE} on failure */ public double getDistance() { return distance; diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java index 3f45e034c8..e66ee28d8a 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java @@ -132,7 +132,8 @@ public List getEvaluatedNeo4jCommands() { double distance = calculator.computeDistance(parsedQuery, graph); metrics = new Neo4jDistanceWithMetrics(distance, graph.nodeCount(), false); } catch (Exception e) { - SimpleLogger.uniqueWarn("Failed to compute Neo4j heuristic for query: " + query); + SimpleLogger.uniqueWarn("Failed to compute Neo4j heuristic for query: " + query + + " | cause: " + e.getClass().getName() + ": " + e.getMessage()); metrics = new Neo4jDistanceWithMetrics(Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE, graph.nodeCount(), true); } commandsWithDistances.add(new Neo4jCommandWithDistance(query, metrics)); diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java index 5d42fbc359..9868f0809f 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java @@ -38,11 +38,10 @@ public class Neo4jHeuristicsCalculator { public static final double C = DistanceHelper.H_NOT_NULL; /** - * Largest distance {@link #computeDistance} can report: the query matched nothing at all. A caller that - * cannot compute a distance (for example when the evaluation throws) reports this value, so a failure - * never looks closer to satisfied than a genuine miss. + * Distance reported when a query could not be evaluated at all, so that a failure never looks closer + * to satisfied than a genuine miss. */ - public static final double MAX_NEO4J_DISTANCE = 1.0d; + public static final double MAX_NEO4J_DISTANCE = Double.MAX_VALUE; private final Neo4jStructuralMatcher matcher = new Neo4jStructuralMatcher(); private final Neo4jConditionEvaluator evaluator; @@ -72,8 +71,8 @@ public Truthness computeHeuristic(MatchOperation query, Neo4jGraph graph) { } /** - * Converts a heuristic to the distance form: {@code 1 - ofTrue}, in - * {@code [0, MAX_NEO4J_DISTANCE]}, where 0 means the query is satisfied. + * Converts a heuristic to the distance form: {@code 1 - ofTrue}, which lies in + * {@code [0,1]} by construction of {@code Truthness}, where 0 means the query is satisfied. */ public double computeDistance(MatchOperation query, Neo4jGraph graph) { Truthness heuristic = computeHeuristic(query, graph); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetricsTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetricsTest.java new file mode 100644 index 0000000000..f9be52e600 --- /dev/null +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetricsTest.java @@ -0,0 +1,31 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +import org.evomaster.client.java.controller.neo4j.heuristics.Neo4jHeuristicsCalculator; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class Neo4jDistanceWithMetricsTest { + + @Test + void testComputedDistanceIsKeptAsIs() { + Neo4jDistanceWithMetrics metrics = new Neo4jDistanceWithMetrics(0.25d, 3, false); + assertEquals(0.25d, metrics.getDistance(), 0.0d); + assertEquals(3, metrics.getNumberOfEvaluatedNodes()); + assertFalse(metrics.isEvaluationFailure()); + } + + @Test + void testNegativeDistanceIsRejected() { + assertThrows(IllegalArgumentException.class, () -> new Neo4jDistanceWithMetrics(-0.1d, 0, false)); + } + + @Test + void testFailureMustCarryTheMaximumDistance() { + assertThrows(IllegalArgumentException.class, () -> new Neo4jDistanceWithMetrics(1.0d, 0, true)); + Neo4jDistanceWithMetrics failed = + new Neo4jDistanceWithMetrics(Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE, 0, true); + assertTrue(failed.isEvaluationFailure()); + assertEquals(Double.MAX_VALUE, failed.getDistance(), 0.0d); + } +} diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java index b101431523..c9b770909c 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java @@ -90,7 +90,8 @@ void testHeuristicsAreNotComputedWhenDisabled() { @Test void testAFailedEvaluationReportsTheMaximumDistance() { // A relationship whose source node is not in the graph: the graph reads fine, but scoring the query - // against it fails, and that failure must never look closer to satisfied than a genuine miss. + // against it fails, which must be reported as MAX_NEO4J_DISTANCE so it never looks closer to + // satisfied than a genuine miss. List nodes = Arrays.asList( nodeRecord("n1", labels("Person"), props("age", 25L))); List rels = Arrays.asList(