diff --git a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java index 518494e8d4..9c7122171d 100644 --- a/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java +++ b/client-java/controller-api/src/main/java/org/evomaster/client/java/controller/api/dto/ExtraHeuristicEntryDto.java @@ -9,9 +9,10 @@ public class ExtraHeuristicEntryDto implements Serializable { /** * The type of extra heuristic. - * Note: for the moment, we only have heuristics on SQL, MONGO, OPENSEARCH, REDIS, DYNAMODB and CASSANDRA commands + * Note: for the moment, we only have heuristics on SQL, MONGO, OPENSEARCH, REDIS, DYNAMODB, + * CASSANDRA and NEO4J commands */ - public enum Type {SQL, MONGO, OPENSEARCH, REDIS, DYNAMODB, CASSANDRA} + public enum Type {SQL, MONGO, OPENSEARCH, REDIS, DYNAMODB, CASSANDRA, NEO4J} /** * Should we try to minimize or maximize the heuristic? diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java index af6477642b..3a6c0a5749 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/SutHandler.java @@ -179,6 +179,14 @@ default void extractRPCSchema(){} default Object getMongoConnection() {return null;} + /** + * @return the Neo4j {@code org.neo4j.driver.Driver} of the SUT, or {@code null} if the SUT does + * not use Neo4j. Returned as {@code Object} and accessed by reflection, so the driver does not + * hard-depend on a specific {@code neo4j-java-driver} version. Used both to read the live graph + * when computing Cypher heuristics and (later) to insert test data. + */ + default Object getNeo4jConnection() {return null;} + default Object getOpenSearchConnection() {return null;} default ReflectionBasedRedisClient getRedisConnection() {return null;} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java index a0ec4f4d31..a28a78d76f 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/EMController.java @@ -386,6 +386,7 @@ public Response runSut(SutRunDto dto, @Context HttpServletRequest httpServletReq noKillSwitch(() -> sutController.initSqlHandler()); noKillSwitch(() -> sutController.registerOrExecuteInitSqlCommandsIfNeeded(true)); noKillSwitch(() -> sutController.initMongoHandler()); + noKillSwitch(() -> sutController.initNeo4jHandler()); noKillSwitch(() -> sutController.initOpenSearchHandler()); noKillSwitch(() -> sutController.initRedisHandler()); noKillSwitch(() -> sutController.initDynamoDbHandler()); diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index a20fef75cf..053b1a8e8b 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -35,6 +35,7 @@ import org.evomaster.client.java.sql.SqlScriptRunnerCached; import org.evomaster.client.java.sql.DbSpecification; import org.evomaster.client.java.controller.internal.db.mongo.MongoHandler; +import org.evomaster.client.java.controller.internal.db.neo4j.Neo4jHandler; import org.evomaster.client.java.sql.DbInfoExtractor; import org.evomaster.client.java.sql.internal.SqlHandler; import org.evomaster.client.java.controller.mongo.MongoScriptRunner; @@ -89,6 +90,8 @@ public abstract class SutController implements SutHandler, CustomizationHandler private final MongoHandler mongoHandler = new MongoHandler(); + private final Neo4jHandler neo4jHandler = new Neo4jHandler(); + private final OpenSearchHandler openSearchHandler = new OpenSearchHandler(); private final RedisHandler redisHandler = new RedisHandler(); @@ -351,6 +354,10 @@ public final void initMongoHandler() { } } + public final void initNeo4jHandler() { + neo4jHandler.setNeo4jConnection(getNeo4jConnection()); + } + // TODO: Refactor this initialization methods once Redis and OpenSearch implementations are done public final void initOpenSearchHandler() { // This is needed because the replacement use to get this info occurs during the start of the SUT. @@ -397,6 +404,7 @@ public final boolean doEmploySmartDbClean(){ public final void resetExtraHeuristics() { sqlHandler.reset(); mongoHandler.reset(); + neo4jHandler.reset(); redisHandler.reset(); dynamoDbHandler.reset(); } @@ -422,7 +430,7 @@ public final ExtraHeuristicsDto computeExtraHeuristics(boolean queryFromDatabase if (isSQLHeuristicsComputationAllowed() || isMongoHeuristicsComputationAllowed() || isOpenSearchHeuristicsComputationAllowed() || isRedisHeuristicsComputationAllowed() - || isDynamoDbHeuristicsComputationAllowed()) { + || isDynamoDbHeuristicsComputationAllowed() || isNeo4jHeuristicsComputationAllowed()) { List additionalInfoList = getAdditionalInfoList(); if (isSQLHeuristicsComputationAllowed()) { @@ -431,6 +439,9 @@ public final ExtraHeuristicsDto computeExtraHeuristics(boolean queryFromDatabase if (isMongoHeuristicsComputationAllowed()) { computeMongoHeuristics(dto, additionalInfoList); } + if (isNeo4jHeuristicsComputationAllowed()) { + computeNeo4jHeuristics(dto, additionalInfoList); + } if (isOpenSearchHeuristicsComputationAllowed()) { computeOpenSearchHeuristics(dto, additionalInfoList); } @@ -452,6 +463,10 @@ private boolean isMongoHeuristicsComputationAllowed() { return mongoHandler.isCalculateHeuristics() || mongoHandler.isExtractMongoExecution(); } + private boolean isNeo4jHeuristicsComputationAllowed() { + return neo4jHandler.isCalculateHeuristics(); + } + private boolean isOpenSearchHeuristicsComputationAllowed() { return openSearchHandler.isCalculateHeuristics(); } @@ -546,6 +561,34 @@ public final void computeMongoHeuristics(ExtraHeuristicsDto dto, List additionalInfoList){ + if(neo4jHandler.isCalculateHeuristics()){ + if(!additionalInfoList.isEmpty()) { + AdditionalInfo last = additionalInfoList.get(additionalInfoList.size() - 1); + last.getNeo4JInfoData().forEach(it -> { + try { + neo4jHandler.handle(it); + } catch (Exception e){ + SimpleLogger.error("FAILED TO HANDLE NEO4J COMMAND: " + e.getMessage()); + assert false; + } + }); + } + + neo4jHandler.getEvaluatedNeo4jCommands().stream() + .map(p -> + new ExtraHeuristicEntryDto( + ExtraHeuristicEntryDto.Type.NEO4J, + ExtraHeuristicEntryDto.Objective.MINIMIZE_TO_ZERO, + p.getCommand(), + p.getDistanceWithMetrics().getDistance(), + p.getDistanceWithMetrics().getNumberOfEvaluatedNodes(), + p.getDistanceWithMetrics().isEvaluationFailure() + )) + .forEach(h -> dto.heuristics.add(h)); + } + } + public final void computeOpenSearchHeuristics(ExtraHeuristicsDto dto, List additionalInfoList) { if (openSearchHandler.isCalculateHeuristics()) { if (!additionalInfoList.isEmpty()) { diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jCommandWithDistance.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jCommandWithDistance.java new file mode 100644 index 0000000000..c96511d73e --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jCommandWithDistance.java @@ -0,0 +1,35 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +/** + * Pairs a captured Cypher query with its computed distance to being satisfied by the live graph. + */ +public final class Neo4jCommandWithDistance { + + private final String command; + private final Neo4jDistanceWithMetrics distanceWithMetrics; + + /** + * Creates the evaluation of one captured query. + * + * @param command the Cypher query, as executed by the SUT + * @param distanceWithMetrics its heuristic result + */ + public Neo4jCommandWithDistance(String command, Neo4jDistanceWithMetrics distanceWithMetrics) { + this.command = command; + this.distanceWithMetrics = distanceWithMetrics; + } + + /** + * @return the Cypher query, as executed by the SUT + */ + public String getCommand() { + return command; + } + + /** + * @return the heuristic result of the query + */ + public Neo4jDistanceWithMetrics getDistanceWithMetrics() { + return distanceWithMetrics; + } +} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetrics.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetrics.java new file mode 100644 index 0000000000..bb5f39db2b --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetrics.java @@ -0,0 +1,64 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +import org.evomaster.client.java.controller.neo4j.heuristics.Neo4jHeuristicsCalculator; + +/** + * Result of scoring one captured Cypher query against the live graph: the distance to satisfying the + * query, how many graph nodes were considered, and whether the evaluation failed. + *

+ * The distance is {@code 1 - ofTrue} of the {@code Truthness} computed by the heuristics calculator, so a + * computed value lies in {@code [0,1]} by construction, with 0 meaning the query is satisfied. A failed + * evaluation carries {@link Neo4jHeuristicsCalculator#MAX_NEO4J_DISTANCE}. + */ +public final class Neo4jDistanceWithMetrics { + + private final double distance; + private final int numberOfEvaluatedNodes; + private final boolean evaluationFailure; + + /** + * Creates a Neo4j heuristic result. + * + * @param distance distance to satisfying the query, 0 meaning satisfied + * @param numberOfEvaluatedNodes number of graph nodes considered + * @param evaluationFailure whether the evaluation failed, in which case the distance must be + * {@link Neo4jHeuristicsCalculator#MAX_NEO4J_DISTANCE} + */ + public Neo4jDistanceWithMetrics(double distance, int numberOfEvaluatedNodes, boolean evaluationFailure) { + if (distance < 0.0d || Double.isNaN(distance)) { + throw new IllegalArgumentException("distance must be non-negative, but was " + distance); + } + if (numberOfEvaluatedNodes < 0) { + throw new IllegalArgumentException("numberOfEvaluatedNodes must be non-negative"); + } + if (evaluationFailure && distance != Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE) { + throw new IllegalArgumentException( + "a failed Neo4j distance computation cannot have a value different than MAX_NEO4J_DISTANCE"); + } + this.distance = distance; + this.numberOfEvaluatedNodes = numberOfEvaluatedNodes; + this.evaluationFailure = evaluationFailure; + } + + /** + * @return distance to satisfying the query, 0 meaning satisfied, + * {@link Neo4jHeuristicsCalculator#MAX_NEO4J_DISTANCE} on failure + */ + public double getDistance() { + return distance; + } + + /** + * @return number of graph nodes considered + */ + public int getNumberOfEvaluatedNodes() { + return numberOfEvaluatedNodes; + } + + /** + * @return whether the evaluation failed + */ + public boolean isEvaluationFailure() { + return evaluationFailure; + } +} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java new file mode 100644 index 0000000000..e66ee28d8a --- /dev/null +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandler.java @@ -0,0 +1,145 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +import org.evomaster.client.java.controller.neo4j.data.Neo4jGraph; +import org.evomaster.client.java.controller.neo4j.heuristics.Neo4jHeuristicsCalculator; +import org.evomaster.client.java.controller.neo4j.operations.MatchOperation; +import org.evomaster.client.java.controller.neo4j.parser.CypherParser; +import org.evomaster.client.java.controller.neo4j.parser.CypherParserException; +import org.evomaster.client.java.controller.neo4j.parser.CypherParserFactory; +import org.evomaster.client.java.controller.internal.TaintHandlerExecutionTracer; +import org.evomaster.client.java.instrumentation.Neo4JRunCommand; +import org.evomaster.client.java.utils.SimpleLogger; + +import java.util.ArrayList; +import java.util.List; + +/** + * Acts upon Cypher queries executed by the SUT (captured as {@link Neo4JRunCommand}s): for each + * captured query it computes how close the live graph is to satisfying it, as a distance to minimize. + * Only MATCH queries are scored; a query that does not parse as a MATCH (e.g. a write) is skipped. + */ +public class Neo4jHandler { + + /** Cypher queries captured from {@code Session.run}, pending evaluation. */ + private final List operations; + + /** The computed heuristics, one per scored query. */ + private final List commandsWithDistances; + + /** Whether to compute heuristics based on execution or not. */ + private volatile boolean calculateHeuristics; + + /** + * The SUT's {@code org.neo4j.driver.Driver}, kept as an {@code Object} and used by reflection so + * we do not hard-depend on a specific driver version. {@code null} when the SUT does not use Neo4j. + */ + private Object neo4jConnection = null; + + private final CypherParser parser = CypherParserFactory.buildParser(); + + private final Neo4jHeuristicsCalculator calculator = + new Neo4jHeuristicsCalculator(new TaintHandlerExecutionTracer()); + private final Neo4jGraphReader graphReader = new Neo4jGraphReader(); + + /** + * Creates a handler with heuristic calculation enabled. + */ + public Neo4jHandler() { + operations = new ArrayList<>(); + commandsWithDistances = new ArrayList<>(); + calculateHeuristics = true; + } + + /** + * Clears data collected for the current action. + */ + public void reset() { + operations.clear(); + commandsWithDistances.clear(); + } + + /** + * @return whether Neo4j heuristic calculation is enabled + */ + public boolean isCalculateHeuristics() { + return calculateHeuristics; + } + + /** + * Enables or disables Neo4j heuristic calculation. + * + * @param calculateHeuristics new calculation state + */ + public void setCalculateHeuristics(boolean calculateHeuristics) { + this.calculateHeuristics = calculateHeuristics; + } + + /** + * Sets the driver used to read the live graph. + * + * @param neo4jConnection the SUT's {@code org.neo4j.driver.Driver}, or {@code null} if it has none + */ + public void setNeo4jConnection(Object neo4jConnection) { + this.neo4jConnection = neo4jConnection; + } + + /** + * Registers one intercepted Cypher query. + * + * @param info intercepted query + */ + public void handle(Neo4JRunCommand info) { + if (calculateHeuristics && info.getQuery() != null) { + operations.add(info); + } + } + + /** + * Evaluates all registered queries against a single snapshot of the graph, and consumes them. + * The snapshot is read once per action rather than per query, since the SUT is not running while + * the heuristics are computed. + * + * @return evaluated queries for the current action + */ + public List getEvaluatedNeo4jCommands() { + + if (!calculateHeuristics || neo4jConnection == null || operations.isEmpty()) { + operations.clear(); + return commandsWithDistances; + } + + Neo4jGraph graph; + try { + graph = graphReader.read(neo4jConnection); + } catch (Exception e) { + SimpleLogger.uniqueWarn("Failed to read the Neo4j graph to compute heuristics: " + e.getMessage()); + operations.clear(); + return commandsWithDistances; + } + + for (Neo4JRunCommand op : operations) { + String query = op.getQuery(); + final MatchOperation parsedQuery; + try { + parsedQuery = parser.parse(query); + } catch (CypherParserException e) { + SimpleLogger.uniqueWarn("Failed to parse Cypher query for Neo4j heuristics: " + e.getMessage()); + continue; + } + + Neo4jDistanceWithMetrics metrics; + try { + double distance = calculator.computeDistance(parsedQuery, graph); + metrics = new Neo4jDistanceWithMetrics(distance, graph.nodeCount(), false); + } catch (Exception e) { + SimpleLogger.uniqueWarn("Failed to compute Neo4j heuristic for query: " + query + + " | cause: " + e.getClass().getName() + ": " + e.getMessage()); + metrics = new Neo4jDistanceWithMetrics(Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE, graph.nodeCount(), true); + } + commandsWithDistances.add(new Neo4jCommandWithDistance(query, metrics)); + } + + operations.clear(); + return commandsWithDistances; + } +} diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java index 2b65e1bc93..9868f0809f 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/neo4j/heuristics/Neo4jHeuristicsCalculator.java @@ -37,6 +37,12 @@ public class Neo4jHeuristicsCalculator { */ public static final double C = DistanceHelper.H_NOT_NULL; + /** + * Distance reported when a query could not be evaluated at all, so that a failure never looks closer + * to satisfied than a genuine miss. + */ + public static final double MAX_NEO4J_DISTANCE = Double.MAX_VALUE; + private final Neo4jStructuralMatcher matcher = new Neo4jStructuralMatcher(); private final Neo4jConditionEvaluator evaluator; @@ -65,8 +71,8 @@ public Truthness computeHeuristic(MatchOperation query, Neo4jGraph graph) { } /** - * Converts a heuristic to the distance form: {@code 1 - ofTrue}, in - * {@code [0,1]}, where 0 means the query is satisfied. + * Converts a heuristic to the distance form: {@code 1 - ofTrue}, which lies in + * {@code [0,1]} by construction of {@code Truthness}, where 0 means the query is satisfied. */ public double computeDistance(MatchOperation query, Neo4jGraph graph) { Truthness heuristic = computeHeuristic(query, graph); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetricsTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetricsTest.java new file mode 100644 index 0000000000..f9be52e600 --- /dev/null +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jDistanceWithMetricsTest.java @@ -0,0 +1,31 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +import org.evomaster.client.java.controller.neo4j.heuristics.Neo4jHeuristicsCalculator; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class Neo4jDistanceWithMetricsTest { + + @Test + void testComputedDistanceIsKeptAsIs() { + Neo4jDistanceWithMetrics metrics = new Neo4jDistanceWithMetrics(0.25d, 3, false); + assertEquals(0.25d, metrics.getDistance(), 0.0d); + assertEquals(3, metrics.getNumberOfEvaluatedNodes()); + assertFalse(metrics.isEvaluationFailure()); + } + + @Test + void testNegativeDistanceIsRejected() { + assertThrows(IllegalArgumentException.class, () -> new Neo4jDistanceWithMetrics(-0.1d, 0, false)); + } + + @Test + void testFailureMustCarryTheMaximumDistance() { + assertThrows(IllegalArgumentException.class, () -> new Neo4jDistanceWithMetrics(1.0d, 0, true)); + Neo4jDistanceWithMetrics failed = + new Neo4jDistanceWithMetrics(Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE, 0, true); + assertTrue(failed.isEvaluationFailure()); + assertEquals(Double.MAX_VALUE, failed.getDistance(), 0.0d); + } +} diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java new file mode 100644 index 0000000000..c9b770909c --- /dev/null +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/neo4j/Neo4jHandlerTest.java @@ -0,0 +1,238 @@ +package org.evomaster.client.java.controller.internal.db.neo4j; + +import org.evomaster.client.java.controller.neo4j.heuristics.Neo4jHeuristicsCalculator; +import org.evomaster.client.java.instrumentation.Neo4JRunCommand; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Exercises {@link Neo4jHandler} and {@link Neo4jGraphReader} end-to-end against a hand-rolled fake + * driver that exposes the same method names the reader reflects over ({@code session}/{@code run}/ + * {@code list}/{@code get}/{@code asString}/{@code asList}/{@code asMap}/{@code close}). This validates + * the reflection plumbing and the whole parse, score and report pipeline without needing a live Neo4j + * instance. + *

+ * Integers are returned as {@code Long} to mimic the real driver's value mapping. + */ +class Neo4jHandlerTest { + + private static final String MATCH_QUERY = + "MATCH (a:Person {age: 25})-[r:KNOWS]->(b:Person) WHERE b.age > 30 RETURN b"; + + private FakeDriver example1Driver() { + List nodes = Arrays.asList( + nodeRecord("n1", labels("Person"), props("age", 25L, "name", "Ana")), + nodeRecord("n2", labels("Person"), props("age", 28L, "name", "Luis")), + nodeRecord("n3", labels("Animal"), props("age", 5L, "name", "Rex")), + nodeRecord("n4", labels("Person"), props("age", 40L, "name", "Carlos"))); + List rels = Arrays.asList( + relRecord("e1", "KNOWS", "n1", "n2"), + relRecord("e2", "LIKES", "n1", "n3"), + relRecord("e3", "KNOWS", "n3", "n4")); + return new FakeDriver(nodes, rels); + } + + @Test + void testScoresMatchQueryAgainstLiveGraph() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(example1Driver()); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + List evaluated = handler.getEvaluatedNeo4jCommands(); + + assertEquals(1, evaluated.size()); + Neo4jCommandWithDistance result = evaluated.get(0); + assertEquals(MATCH_QUERY, result.getCommand()); + assertFalse(result.getDistanceWithMetrics().isEvaluationFailure()); + assertEquals(4, result.getDistanceWithMetrics().getNumberOfEvaluatedNodes()); + // distance is 1 - ofTrue, and here ofTrue is about 0.939. + assertEquals(0.061, result.getDistanceWithMetrics().getDistance(), 0.005); + } + + @Test + void testNonMatchQueryIsSkipped() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(example1Driver()); + handler.handle(new Neo4JRunCommand("CREATE (n:Person {name: 'Zoe'})", null, true, 1)); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + List evaluated = handler.getEvaluatedNeo4jCommands(); + // The write query does not parse as a MATCH and is skipped; only the read query is scored. + assertEquals(1, evaluated.size()); + assertEquals(MATCH_QUERY, evaluated.get(0).getCommand()); + } + + @Test + void testNoConnectionYieldsNoHeuristics() { + Neo4jHandler handler = new Neo4jHandler(); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + assertTrue(handler.getEvaluatedNeo4jCommands().isEmpty()); + } + + @Test + void testHeuristicsAreNotComputedWhenDisabled() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(example1Driver()); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + handler.setCalculateHeuristics(false); + + // Registered while enabled, so it is the check before evaluating that has to drop the query. + assertTrue(handler.getEvaluatedNeo4jCommands().isEmpty()); + } + + @Test + void testAFailedEvaluationReportsTheMaximumDistance() { + // A relationship whose source node is not in the graph: the graph reads fine, but scoring the query + // against it fails, which must be reported as MAX_NEO4J_DISTANCE so it never looks closer to + // satisfied than a genuine miss. + List nodes = Arrays.asList( + nodeRecord("n1", labels("Person"), props("age", 25L))); + List rels = Arrays.asList( + relRecord("e1", "KNOWS", "ghost", "n1")); + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(new FakeDriver(nodes, rels)); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + List evaluated = handler.getEvaluatedNeo4jCommands(); + + assertEquals(1, evaluated.size()); + Neo4jDistanceWithMetrics metrics = evaluated.get(0).getDistanceWithMetrics(); + assertTrue(metrics.isEvaluationFailure()); + assertEquals(Neo4jHeuristicsCalculator.MAX_NEO4J_DISTANCE, metrics.getDistance(), 0.0d); + assertEquals(1, metrics.getNumberOfEvaluatedNodes()); + } + + @Test + void testAnUnreadableGraphYieldsNoHeuristicsInsteadOfFailing() { + Neo4jHandler handler = new Neo4jHandler(); + handler.setNeo4jConnection(new BrokenDriver()); + handler.handle(new Neo4JRunCommand(MATCH_QUERY, null, true, 1)); + + // The SUT must keep running even if its driver cannot be queried. + assertTrue(handler.getEvaluatedNeo4jCommands().isEmpty()); + } + + // Fake Neo4j driver, exposing only the methods the reader reflects over. + + public static final class FakeDriver { + private final List nodes; + private final List rels; + + FakeDriver(List nodes, List rels) { + this.nodes = nodes; + this.rels = rels; + } + + public FakeSession session() { + return new FakeSession(nodes, rels); + } + } + + /** A driver whose session cannot be opened, standing in for a Neo4j that is down. */ + public static final class BrokenDriver { + + public Object session() { + throw new IllegalStateException("no connection to the database"); + } + } + + public static final class FakeSession { + private final List nodes; + private final List rels; + + FakeSession(List nodes, List rels) { + this.nodes = nodes; + this.rels = rels; + } + + public FakeResult run(String query) { + return new FakeResult(query.contains("labels(n)") ? nodes : rels); + } + + public void close() { + } + } + + public static final class FakeResult { + private final List records; + + FakeResult(List records) { + this.records = records; + } + + public List list() { + return records; + } + } + + public static final class FakeRecord { + private final Map fields; + + FakeRecord(Map fields) { + this.fields = fields; + } + + public FakeValue get(String key) { + return new FakeValue(fields.get(key)); + } + } + + public static final class FakeValue { + private final Object value; + + FakeValue(Object value) { + this.value = value; + } + + public String asString() { + return (String) value; + } + + @SuppressWarnings("unchecked") + public List asList() { + return (List) value; + } + + @SuppressWarnings("unchecked") + public Map asMap() { + return (Map) value; + } + } + + private static FakeRecord nodeRecord(String id, List labels, Map props) { + Map f = new LinkedHashMap<>(); + f.put("id", id); + f.put("labels", labels); + f.put("props", props); + return new FakeRecord(f); + } + + private static FakeRecord relRecord(String id, String type, String src, String tgt) { + Map f = new LinkedHashMap<>(); + f.put("id", id); + f.put("type", type); + f.put("src", src); + f.put("tgt", tgt); + f.put("props", new LinkedHashMap()); + return new FakeRecord(f); + } + + private static List labels(String... ls) { + return new ArrayList<>(Arrays.asList(ls)); + } + + private static Map props(Object... kv) { + Map m = new LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put((String) kv[i], kv[i + 1]); + } + return m; + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 262a608922..d1ab4acd2f 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1992,6 +1992,11 @@ class EMConfig { @DependsOnFalseFor("blackBox") var heuristicsForDynamoDb = false + @Experimental + @Cfg("Tracking of Neo4j commands to improve test generation") + @DependsOnFalseFor("blackBox") + var heuristicsForNeo4j = false + @Cfg("Enable extracting SQL execution info") @DependsOnFalseFor("blackBox") var extractSqlExecutionInfo = true diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt index 73d8ed0e91..571a084c47 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt @@ -403,6 +403,10 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual handleDynamoDbHeuristics(dto, fv) } + if (configuration.heuristicsForNeo4j) { + handleNeo4jHeuristics(dto, fv) + } + if (configuration.extractRedisExecutionInfo) { for (i in 0 until dto.extraHeuristics.size) { val extra = dto.extraHeuristics[i] @@ -551,4 +555,37 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual } } } + + /** Applies Cypher pattern distances and records their evaluation metrics. */ + private fun handleNeo4jHeuristics(dto: TestResultsDto, fv: FitnessValue) { + for (i in 0 until dto.extraHeuristics.size) { + val extra = dto.extraHeuristics[i] + + extraHeuristicsLogger.writeHeuristics(extra.heuristics, i) + + val toMinimize = extra.heuristics + .filter { + it != null + && it.objective == ExtraHeuristicEntryDto.Objective.MINIMIZE_TO_ZERO + && it.type == ExtraHeuristicEntryDto.Type.NEO4J + }.map { it.value } + .toList() + + if (toMinimize.isNotEmpty()) { + fv.addExtraObjectivesToMinimize(i, toMinimize) + } + + extra.heuristics + .filterNotNull().forEach { + if (it.type == ExtraHeuristicEntryDto.Type.NEO4J) { + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(it.numberOfEvaluatedRecords) + if (it.extraHeuristicEvaluationFailure) { + statistics.reportNeo4jHeuristicEvaluationFailure() + } else { + statistics.reportNeo4jHeuristicEvaluationSuccess() + } + } + } + } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index e682d4372e..fc034570de 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -146,6 +146,11 @@ class Statistics : SearchListener { private var dynamoDbHeuristicEvaluationFailureCount = 0 private val dynamoDbItemsAverageCalculator = IncrementalAverage() + // neo4j heuristic evaluation statistic + private var neo4jHeuristicEvaluationSuccessCount = 0 + private var neo4jHeuristicEvaluationFailureCount = 0 + private val neo4jNodesAverageCalculator = IncrementalAverage() + class Pair(val header: String, val element: String) @@ -247,6 +252,11 @@ class Statistics : SearchListener { dynamoDbItemsAverageCalculator.addValue(numberOfEvaluatedItems) } + /** Records the number of nodes inspected by one Neo4j heuristic evaluation. */ + fun reportNumberOfEvaluatedNodesForNeo4jHeuristic(numberOfEvaluatedNodes: Int) { + neo4jNodesAverageCalculator.addValue(numberOfEvaluatedNodes) + } + fun reportSqlParsingFailures(numberOfParsingFailures: Int) { if (numberOfParsingFailures<0) { throw IllegalArgumentException("Invalid number of parsing failures: $numberOfParsingFailures") @@ -379,6 +389,16 @@ class Statistics : SearchListener { internal fun getSqlZ3CacheHitCount() = sqlZ3CacheHitCount internal fun getSqlZ3CacheMissCount() = sqlZ3CacheMissCount + /** Records one successful Neo4j heuristic evaluation. */ + fun reportNeo4jHeuristicEvaluationSuccess() { + neo4jHeuristicEvaluationSuccessCount++ + } + + /** Records one failed Neo4j heuristic evaluation. */ + fun reportNeo4jHeuristicEvaluationFailure() { + neo4jHeuristicEvaluationFailureCount++ + } + // Exposed for tests: verify the failure breakdown adds up to the aggregate, and that the two // duration accumulators only ever move forward. internal fun getSqlZ3ParseFailureCount() = sqlZ3ParseFailureCount @@ -407,6 +427,13 @@ class Statistics : SearchListener { /** Returns the average number of items inspected by DynamoDB heuristics. */ fun averageNumberOfEvaluatedItemsForDynamoDbHeuristics(): Double = dynamoDbItemsAverageCalculator.mean + /** Returns the total number of Neo4j heuristic evaluations. */ + fun getNeo4jHeuristicsEvaluationCount(): Int = + neo4jHeuristicEvaluationSuccessCount + neo4jHeuristicEvaluationFailureCount + + /** Returns the average number of nodes inspected by Neo4j heuristics. */ + fun averageNumberOfEvaluatedNodesForNeo4jHeuristics(): Double = neo4jNodesAverageCalculator.mean + override fun newActionsEvaluated(n: Int) { if(!epc.isInSearch()){ @@ -581,6 +608,10 @@ class Statistics : SearchListener { add(Pair("sqlInsertionExecutions", "$sqlInsertionExecutionCount")) } + // statistics info for Neo4j Heuristics + add(Pair("averageNumberOfEvaluatedNodesForNeo4jHeuristics","${averageNumberOfEvaluatedNodesForNeo4jHeuristics()}")) + add(Pair("neo4jHeuristicsEvaluationCount","${getNeo4jHeuristicsEvaluationCount()}")) + for(phase in ExecutionPhaseController.Phase.entries){ add(Pair("phase_${phase.name}", "${epc.getPhaseDurationInSeconds(phase)}")) } diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt index 26c52338f1..1d2f4c4209 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/StatisticsTest.kt @@ -52,6 +52,22 @@ class StatisticsTest { assertEquals(20.0, statistics.averageNumberOfEvaluatedItemsForDynamoDbHeuristics()) } + @Test + fun testNeo4jHeuristicsAverage() { + val statistics = Statistics() + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(10) + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(20) + statistics.reportNumberOfEvaluatedNodesForNeo4jHeuristic(30) + + repeat(2) { + statistics.reportNeo4jHeuristicEvaluationSuccess() + } + statistics.reportNeo4jHeuristicEvaluationFailure() + + assertEquals(3, statistics.getNeo4jHeuristicsEvaluationCount()) + assertEquals(20.0, statistics.averageNumberOfEvaluatedNodesForNeo4jHeuristics()) + } + @Test fun testSqlZ3CacheAccountingInvariant() { val statistics = Statistics() diff --git a/docs/options.md b/docs/options.md index b53fa46b41..c3f863d321 100644 --- a/docs/options.md +++ b/docs/options.md @@ -308,6 +308,7 @@ There are 3 types of options: |`generateSqlDataWithZ3`| __Boolean__. Enable EvoMaster to generate SQL data with direct accesses to the database. Use the Z3 SMT solver. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`handleFlakiness`| __Boolean__. Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing. Note that flakiness is now supported only for fuzzing REST APIs. *Default value*: `false`.| |`heuristicsForDynamoDb`| __Boolean__. Tracking of DynamoDB commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.| +|`heuristicsForNeo4j`| __Boolean__. Tracking of Neo4j commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`heuristicsForRedis`| __Boolean__. Tracking of Redis commands to improve test generation. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`heuristicsForSQLAdvanced`| __Boolean__. If using SQL heuristics, enable more advanced version. *Depends on*: `blackBox=false`. *Default value*: `false`.| |`httpOracles`| __Boolean__. Extra checks on HTTP properties in returned responses, used as automated oracles to detect faults. *Default value*: `false`.|