From 5ee3185b4612c00bf9363aca255edda9c4e0507f Mon Sep 17 00:00:00 2001 From: fengting124 Date: Wed, 8 Jul 2026 18:21:35 +0800 Subject: [PATCH] feat: add evaluation batch execution framework --- .../DeterministicEvaluationModelClient.java | 27 +++ .../client/EvaluationModelClient.java | 7 + .../client/EvaluationModelRequest.java | 9 + .../client/EvaluationModelResult.java | 10 + .../controller/EvaluationController.java | 17 +- .../evaluation/domain/EvaluationRun.java | 54 +++++ .../evaluation/domain/EvaluationSample.java | 12 + .../dto/EvaluationDetailResponse.java | 2 + .../evaluation/dto/EvaluationRunResponse.java | 2 + .../service/EvaluationExecutionService.java | 158 +++++++++++++ .../evaluation/service/EvaluationService.java | 6 + .../V4__add_evaluation_execution_state.sql | 6 + ...eterministicEvaluationModelClientTest.java | 29 +++ .../controller/EvaluationControllerTest.java | 40 ++++ .../repository/EvaluationRepositoryTest.java | 9 + .../EvaluationExecutionServiceTest.java | 184 +++++++++++++++ docs/project-improvement-roadmap.md | 10 + docs/project-worklog.md | 75 +++++- .../2026-07-08-evaluation-batch-execution.md | 215 ++++++++++++++++++ 19 files changed, 862 insertions(+), 10 deletions(-) create mode 100644 backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java create mode 100644 backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java create mode 100644 backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java create mode 100644 backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java create mode 100644 backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java create mode 100644 backend-java/src/main/resources/db/migration/V4__add_evaluation_execution_state.sql create mode 100644 backend-java/src/test/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClientTest.java create mode 100644 backend-java/src/test/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionServiceTest.java create mode 100644 docs/superpowers/plans/2026-07-08-evaluation-batch-execution.md diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java new file mode 100644 index 0000000..3070e57 --- /dev/null +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java @@ -0,0 +1,27 @@ +package com.fengting.aigcforensics.evaluation.client; + +import java.nio.charset.StandardCharsets; +import java.util.zip.CRC32; + +import org.springframework.stereotype.Component; + +import com.fengting.aigcforensics.domain.ModelLabel; + +@Component +public class DeterministicEvaluationModelClient implements EvaluationModelClient { + + @Override + public EvaluationModelResult predict(EvaluationModelRequest request) { + double score = stableScore(request.modelId() + ":" + request.filename()); + ModelLabel label = score >= 0.5 ? ModelLabel.SYNTHETIC : ModelLabel.AUTHENTIC; + int latencyMs = 10 + (int) Math.round(score * 20); + return new EvaluationModelResult(label, score, latencyMs); + } + + private double stableScore(String value) { + CRC32 crc32 = new CRC32(); + crc32.update(value.getBytes(StandardCharsets.UTF_8)); + return crc32.getValue() / (double) 0xffffffffL; + } +} + diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java new file mode 100644 index 0000000..3dc8705 --- /dev/null +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java @@ -0,0 +1,7 @@ +package com.fengting.aigcforensics.evaluation.client; + +public interface EvaluationModelClient { + + EvaluationModelResult predict(EvaluationModelRequest request); +} + diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java new file mode 100644 index 0000000..543cac9 --- /dev/null +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java @@ -0,0 +1,9 @@ +package com.fengting.aigcforensics.evaluation.client; + +public record EvaluationModelRequest( + String evaluationId, + String sampleId, + String modelId, + String filename) { +} + diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java new file mode 100644 index 0000000..0e94046 --- /dev/null +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java @@ -0,0 +1,10 @@ +package com.fengting.aigcforensics.evaluation.client; + +import com.fengting.aigcforensics.domain.ModelLabel; + +public record EvaluationModelResult( + ModelLabel predictedLabel, + double score, + int latencyMs) { +} + diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/controller/EvaluationController.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/controller/EvaluationController.java index 76238dd..b587c1c 100644 --- a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/controller/EvaluationController.java +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/controller/EvaluationController.java @@ -16,6 +16,7 @@ import com.fengting.aigcforensics.evaluation.dto.EvaluationDetailResponse; import com.fengting.aigcforensics.evaluation.dto.EvaluationRunResponse; import com.fengting.aigcforensics.evaluation.dto.EvaluationSampleResponse; +import com.fengting.aigcforensics.evaluation.service.EvaluationExecutionService; import com.fengting.aigcforensics.evaluation.service.EvaluationService; import jakarta.validation.Valid; @@ -25,9 +26,13 @@ public class EvaluationController { private final EvaluationService evaluationService; + private final EvaluationExecutionService evaluationExecutionService; - public EvaluationController(EvaluationService evaluationService) { + public EvaluationController( + EvaluationService evaluationService, + EvaluationExecutionService evaluationExecutionService) { this.evaluationService = evaluationService; + this.evaluationExecutionService = evaluationExecutionService; } @PostMapping @@ -46,6 +51,16 @@ public EvaluationDetailResponse getEvaluation(@PathVariable String evaluationId) return evaluationService.getEvaluation(evaluationId); } + @PostMapping("/{evaluationId}/run") + public EvaluationDetailResponse runEvaluation(@PathVariable String evaluationId) { + return evaluationExecutionService.runEvaluation(evaluationId); + } + + @PostMapping("/{evaluationId}/retry") + public EvaluationDetailResponse retryEvaluation(@PathVariable String evaluationId) { + return evaluationExecutionService.runEvaluation(evaluationId); + } + @GetMapping("/{evaluationId}/samples") public List listSamples( @PathVariable String evaluationId, diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationRun.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationRun.java index 5219ed0..18cb1ed 100644 --- a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationRun.java +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationRun.java @@ -53,6 +53,12 @@ public class EvaluationRun { @Column(name = "f1_score") private Double f1; + @Column(name = "attempt_count", nullable = false) + private int attemptCount; + + @Column(name = "max_attempts", nullable = false) + private int maxAttempts; + @Column(name = "created_at", nullable = false) private Instant createdAt; @@ -80,6 +86,8 @@ public EvaluationRun( Double precision, Double recall, Double f1, + int attemptCount, + int maxAttempts, Instant createdAt, Instant startedAt, Instant completedAt, @@ -95,6 +103,8 @@ public EvaluationRun( this.precision = precision; this.recall = recall; this.f1 = f1; + this.attemptCount = attemptCount; + this.maxAttempts = maxAttempts; this.createdAt = createdAt; this.startedAt = startedAt; this.completedAt = completedAt; @@ -149,6 +159,14 @@ public Double getF1() { return f1; } + public int getAttemptCount() { + return attemptCount; + } + + public int getMaxAttempts() { + return maxAttempts; + } + public Instant getCreatedAt() { return createdAt; } @@ -164,4 +182,40 @@ public Instant getCompletedAt() { public String getFailureReason() { return failureReason; } + + public boolean canRetry() { + return attemptCount < maxAttempts; + } + + public void markStarted(Instant startedAt) { + this.status = EvaluationStatus.RUNNING; + this.attemptCount++; + this.startedAt = startedAt; + this.completedAt = null; + this.failureReason = null; + } + + public void markCompleted( + int completedSamples, + Double accuracy, + Double precision, + Double recall, + Double f1, + Instant completedAt) { + this.status = EvaluationStatus.COMPLETED; + this.completedSamples = completedSamples; + this.accuracy = accuracy; + this.precision = precision; + this.recall = recall; + this.f1 = f1; + this.completedAt = completedAt; + this.failureReason = null; + } + + public void markFailed(String failureReason, int completedSamples, Instant completedAt) { + this.status = EvaluationStatus.FAILED; + this.completedSamples = completedSamples; + this.completedAt = completedAt; + this.failureReason = failureReason; + } } diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationSample.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationSample.java index 61ba0a7..4b662a1 100644 --- a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationSample.java +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationSample.java @@ -122,4 +122,16 @@ public String getFailureReason() { public Instant getCreatedAt() { return createdAt; } + + public void markPredicted(ModelLabel predictedLabel, Double score, Integer latencyMs) { + this.predictedLabel = predictedLabel; + this.score = score; + this.latencyMs = latencyMs; + this.correct = predictedLabel == groundTruthLabel; + this.failureReason = null; + } + + public void markFailed(String failureReason) { + this.failureReason = failureReason; + } } diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationDetailResponse.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationDetailResponse.java index cd05607..23473bf 100644 --- a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationDetailResponse.java +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationDetailResponse.java @@ -17,6 +17,8 @@ public record EvaluationDetailResponse( Double precision, Double recall, Double f1, + int attemptCount, + int maxAttempts, Instant createdAt, Instant startedAt, Instant completedAt, diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationRunResponse.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationRunResponse.java index 8c863c8..cb62a21 100644 --- a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationRunResponse.java +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationRunResponse.java @@ -16,6 +16,8 @@ public record EvaluationRunResponse( Double precision, Double recall, Double f1, + int attemptCount, + int maxAttempts, Instant createdAt, Instant startedAt, Instant completedAt, diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java new file mode 100644 index 0000000..a83cd1a --- /dev/null +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java @@ -0,0 +1,158 @@ +package com.fengting.aigcforensics.evaluation.service; + +import java.time.Clock; +import java.time.Instant; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.fengting.aigcforensics.evaluation.client.EvaluationModelClient; +import com.fengting.aigcforensics.evaluation.client.EvaluationModelRequest; +import com.fengting.aigcforensics.evaluation.client.EvaluationModelResult; +import com.fengting.aigcforensics.evaluation.domain.EvaluationRun; +import com.fengting.aigcforensics.evaluation.domain.EvaluationSample; +import com.fengting.aigcforensics.evaluation.domain.EvaluationStatus; +import com.fengting.aigcforensics.evaluation.dto.EvaluationDetailResponse; +import com.fengting.aigcforensics.evaluation.dto.EvaluationSampleResponse; +import com.fengting.aigcforensics.evaluation.repository.EvaluationRunRepository; +import com.fengting.aigcforensics.evaluation.repository.EvaluationSampleRepository; +import com.fengting.aigcforensics.service.ResourceNotFoundException; + +@Service +public class EvaluationExecutionService { + + private final EvaluationRunRepository evaluationRunRepository; + private final EvaluationSampleRepository evaluationSampleRepository; + private final EvaluationMetricsCalculator metricsCalculator; + private final EvaluationModelClient modelClient; + private final Clock clock; + + @Autowired + public EvaluationExecutionService( + EvaluationRunRepository evaluationRunRepository, + EvaluationSampleRepository evaluationSampleRepository, + EvaluationMetricsCalculator metricsCalculator, + EvaluationModelClient modelClient) { + this( + evaluationRunRepository, + evaluationSampleRepository, + metricsCalculator, + modelClient, + Clock.systemUTC()); + } + + EvaluationExecutionService( + EvaluationRunRepository evaluationRunRepository, + EvaluationSampleRepository evaluationSampleRepository, + EvaluationMetricsCalculator metricsCalculator, + EvaluationModelClient modelClient, + Clock clock) { + this.evaluationRunRepository = evaluationRunRepository; + this.evaluationSampleRepository = evaluationSampleRepository; + this.metricsCalculator = metricsCalculator; + this.modelClient = modelClient; + this.clock = clock; + } + + @Transactional + public EvaluationDetailResponse runEvaluation(String evaluationId) { + EvaluationRun run = findRun(evaluationId); + List samples = evaluationSampleRepository.findByEvaluationIdOrderByCreatedAtAsc(evaluationId); + if (run.getStatus() == EvaluationStatus.COMPLETED) { + return toDetailResponse(run, samples); + } + if (run.getStatus() == EvaluationStatus.FAILED && !run.canRetry()) { + throw new IllegalStateException("Evaluation retry attempts exhausted: " + evaluationId); + } + + run.markStarted(Instant.now(clock)); + for (EvaluationSample sample : samples) { + if (sample.getPredictedLabel() != null) { + continue; + } + try { + EvaluationModelResult result = modelClient.predict(new EvaluationModelRequest( + run.getEvaluationId(), + sample.getSampleId(), + run.getModelId(), + sample.getFilename())); + sample.markPredicted(result.predictedLabel(), result.score(), result.latencyMs()); + } catch (RuntimeException exception) { + String failureReason = failureMessage(exception); + sample.markFailed(failureReason); + run.markFailed(failureReason, countCompleted(samples), Instant.now(clock)); + evaluationSampleRepository.saveAll(samples); + evaluationRunRepository.save(run); + return toDetailResponse(run, samples); + } + } + + EvaluationMetrics metrics = metricsCalculator.calculate(samples.stream() + .map(sample -> new EvaluationPredictionCase(sample.getGroundTruthLabel(), sample.getPredictedLabel())) + .toList()); + run.markCompleted( + samples.size(), + metrics.accuracy(), + metrics.precision(), + metrics.recall(), + metrics.f1(), + Instant.now(clock)); + evaluationSampleRepository.saveAll(samples); + evaluationRunRepository.save(run); + return toDetailResponse(run, samples); + } + + private int countCompleted(List samples) { + return (int) samples.stream().filter(sample -> sample.getPredictedLabel() != null).count(); + } + + private String failureMessage(RuntimeException exception) { + if (exception.getMessage() == null || exception.getMessage().isBlank()) { + return exception.getClass().getSimpleName(); + } + return exception.getMessage(); + } + + private EvaluationRun findRun(String evaluationId) { + return evaluationRunRepository.findByEvaluationId(evaluationId) + .orElseThrow(() -> new ResourceNotFoundException("Evaluation not found: " + evaluationId)); + } + + private EvaluationDetailResponse toDetailResponse(EvaluationRun run, List samples) { + return new EvaluationDetailResponse( + run.getEvaluationId(), + run.getName(), + run.getDatasetName(), + run.getModelId(), + run.getStatus(), + run.getTotalSamples(), + run.getCompletedSamples(), + run.getAccuracy(), + run.getPrecision(), + run.getRecall(), + run.getF1(), + run.getAttemptCount(), + run.getMaxAttempts(), + run.getCreatedAt(), + run.getStartedAt(), + run.getCompletedAt(), + run.getFailureReason(), + samples.stream().map(this::toSampleResponse).toList()); + } + + private EvaluationSampleResponse toSampleResponse(EvaluationSample sample) { + return new EvaluationSampleResponse( + sample.getSampleId(), + sample.getEvaluationId(), + sample.getFilename(), + sample.getGroundTruthLabel(), + sample.getPredictedLabel(), + sample.getScore(), + sample.getLatencyMs(), + sample.getCorrect(), + sample.getFailureReason(), + sample.getCreatedAt()); + } +} diff --git a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationService.java b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationService.java index 6c9f2b4..da2ec03 100644 --- a/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationService.java +++ b/backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationService.java @@ -90,6 +90,8 @@ public EvaluationRunResponse createEvaluation(CreateEvaluationRequest request) { metrics == null ? null : metrics.precision(), metrics == null ? null : metrics.recall(), metrics == null ? null : metrics.f1(), + 0, + 3, now, completed ? now : null, completed ? now : null, @@ -137,6 +139,8 @@ public EvaluationDetailResponse getEvaluation(String evaluationId) { run.getPrecision(), run.getRecall(), run.getF1(), + run.getAttemptCount(), + run.getMaxAttempts(), run.getCreatedAt(), run.getStartedAt(), run.getCompletedAt(), @@ -254,6 +258,8 @@ private EvaluationRunResponse toRunResponse(EvaluationRun run) { run.getPrecision(), run.getRecall(), run.getF1(), + run.getAttemptCount(), + run.getMaxAttempts(), run.getCreatedAt(), run.getStartedAt(), run.getCompletedAt(), diff --git a/backend-java/src/main/resources/db/migration/V4__add_evaluation_execution_state.sql b/backend-java/src/main/resources/db/migration/V4__add_evaluation_execution_state.sql new file mode 100644 index 0000000..f274876 --- /dev/null +++ b/backend-java/src/main/resources/db/migration/V4__add_evaluation_execution_state.sql @@ -0,0 +1,6 @@ +alter table evaluation_run + add column attempt_count integer not null default 0; + +alter table evaluation_run + add column max_attempts integer not null default 3; + diff --git a/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClientTest.java b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClientTest.java new file mode 100644 index 0000000..4ede4e5 --- /dev/null +++ b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClientTest.java @@ -0,0 +1,29 @@ +package com.fengting.aigcforensics.evaluation.client; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +import com.fengting.aigcforensics.domain.ModelLabel; + +class DeterministicEvaluationModelClientTest { + + @Test + void returnsStablePredictionForSameModelAndFilename() { + DeterministicEvaluationModelClient client = new DeterministicEvaluationModelClient(); + EvaluationModelRequest request = new EvaluationModelRequest( + "eval_001", + "sample_001", + "nonescape-mini", + "dataset/fake_001.jpg"); + + EvaluationModelResult first = client.predict(request); + EvaluationModelResult second = client.predict(request); + + assertThat(second).isEqualTo(first); + assertThat(first.predictedLabel()).isIn(ModelLabel.AUTHENTIC, ModelLabel.SYNTHETIC); + assertThat(first.score()).isBetween(0.0, 1.0); + assertThat(first.latencyMs()).isGreaterThanOrEqualTo(0); + } +} + diff --git a/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/controller/EvaluationControllerTest.java b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/controller/EvaluationControllerTest.java index cbd481c..1257ee0 100644 --- a/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/controller/EvaluationControllerTest.java +++ b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/controller/EvaluationControllerTest.java @@ -114,4 +114,44 @@ void createEvaluationRejectsInvalidManifest() throws Exception { .andExpect(status().isBadRequest()) .andExpect(jsonPath("$.message").value("Unsupported label at manifest line 2: UNKNOWN")); } + + @Test + void runsQueuedEvaluationThroughHttpAndPersistsGeneratedPredictions() throws Exception { + String response = mockMvc.perform(post("/api/evaluations") + .contentType(MediaType.APPLICATION_JSON) + .content(""" + { + "name": "Queued Dataset", + "datasetName": "sample-v2", + "modelId": "nonescape-mini", + "manifest": "filename,groundTruthLabel\\nreal_010.jpg,AUTHENTIC\\nfake_010.jpg,SYNTHETIC" + } + """)) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.status").value("QUEUED")) + .andExpect(jsonPath("$.attemptCount").value(0)) + .andReturn() + .getResponse() + .getContentAsString(); + JsonNode created = objectMapper.readTree(response); + String evaluationId = created.get("evaluationId").asText(); + + mockMvc.perform(post("/api/evaluations/{evaluationId}/run", evaluationId)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.evaluationId").value(evaluationId)) + .andExpect(jsonPath("$.status").value("COMPLETED")) + .andExpect(jsonPath("$.attemptCount").value(1)) + .andExpect(jsonPath("$.maxAttempts").value(3)) + .andExpect(jsonPath("$.completedSamples").value(2)) + .andExpect(jsonPath("$.accuracy").isNumber()) + .andExpect(jsonPath("$.samples.length()").value(2)) + .andExpect(jsonPath("$.samples[0].predictedLabel").isString()) + .andExpect(jsonPath("$.samples[1].score").isNumber()); + + mockMvc.perform(post("/api/evaluations/{evaluationId}/retry", evaluationId)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.evaluationId").value(evaluationId)) + .andExpect(jsonPath("$.status").value("COMPLETED")) + .andExpect(jsonPath("$.attemptCount").value(1)); + } } diff --git a/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/repository/EvaluationRepositoryTest.java b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/repository/EvaluationRepositoryTest.java index b6c6dde..a436754 100644 --- a/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/repository/EvaluationRepositoryTest.java +++ b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/repository/EvaluationRepositoryTest.java @@ -44,6 +44,8 @@ void persistsEvaluationRunAndSamples() { 0.0, 0.0, 0.0, + 1, + 3, now, now, now, @@ -76,6 +78,13 @@ void persistsEvaluationRunAndSamples() { .get() .extracting(EvaluationRun::getStatus) .isEqualTo(EvaluationStatus.COMPLETED); + assertThat(evaluationRunRepository.findByEvaluationId("eval_001")) + .isPresent() + .get() + .satisfies(run -> { + assertThat(run.getAttemptCount()).isEqualTo(1); + assertThat(run.getMaxAttempts()).isEqualTo(3); + }); assertThat(evaluationSampleRepository.findByEvaluationIdOrderByCreatedAtAsc("eval_001")) .extracting(EvaluationSample::getFilename) .containsExactly("real_001.jpg", "fake_001.jpg"); diff --git a/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionServiceTest.java b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionServiceTest.java new file mode 100644 index 0000000..a376cf3 --- /dev/null +++ b/backend-java/src/test/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionServiceTest.java @@ -0,0 +1,184 @@ +package com.fengting.aigcforensics.evaluation.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.fengting.aigcforensics.domain.ModelLabel; +import com.fengting.aigcforensics.evaluation.client.EvaluationModelClient; +import com.fengting.aigcforensics.evaluation.client.EvaluationModelResult; +import com.fengting.aigcforensics.evaluation.domain.EvaluationRun; +import com.fengting.aigcforensics.evaluation.domain.EvaluationSample; +import com.fengting.aigcforensics.evaluation.domain.EvaluationStatus; +import com.fengting.aigcforensics.evaluation.dto.EvaluationDetailResponse; +import com.fengting.aigcforensics.evaluation.repository.EvaluationRunRepository; +import com.fengting.aigcforensics.evaluation.repository.EvaluationSampleRepository; + +@ExtendWith(MockitoExtension.class) +class EvaluationExecutionServiceTest { + + private static final Instant NOW = Instant.parse("2026-07-08T00:00:00Z"); + + @Mock + private EvaluationRunRepository evaluationRunRepository; + + @Mock + private EvaluationSampleRepository evaluationSampleRepository; + + @Test + void runsQueuedEvaluationAndPersistsPredictionsAndMetrics() { + EvaluationRun run = queuedRun(); + List samples = List.of( + sample("sample_001", "real_001.jpg", ModelLabel.AUTHENTIC), + sample("sample_002", "fake_001.jpg", ModelLabel.SYNTHETIC)); + when(evaluationRunRepository.findByEvaluationId("eval_001")).thenReturn(Optional.of(run)); + when(evaluationSampleRepository.findByEvaluationIdOrderByCreatedAtAsc("eval_001")).thenReturn(samples); + EvaluationModelClient modelClient = request -> request.filename().startsWith("fake") + ? new EvaluationModelResult(ModelLabel.SYNTHETIC, 0.91, 42) + : new EvaluationModelResult(ModelLabel.AUTHENTIC, 0.12, 31); + EvaluationExecutionService service = service(modelClient); + + EvaluationDetailResponse response = service.runEvaluation("eval_001"); + + assertThat(response.status()).isEqualTo(EvaluationStatus.COMPLETED); + assertThat(response.completedSamples()).isEqualTo(2); + assertThat(response.attemptCount()).isEqualTo(1); + assertThat(response.accuracy()).isEqualTo(1.0); + assertThat(response.precision()).isEqualTo(1.0); + assertThat(response.recall()).isEqualTo(1.0); + assertThat(response.samples()) + .extracting(sample -> sample.predictedLabel()) + .containsExactly(ModelLabel.AUTHENTIC, ModelLabel.SYNTHETIC); + verify(evaluationRunRepository).save(run); + verify(evaluationSampleRepository).saveAll(samples); + } + + @Test + void marksEvaluationFailedWhenPredictionFails() { + EvaluationRun run = queuedRun(); + List samples = List.of( + sample("sample_001", "real_001.jpg", ModelLabel.AUTHENTIC), + sample("sample_002", "fake_001.jpg", ModelLabel.SYNTHETIC)); + when(evaluationRunRepository.findByEvaluationId("eval_001")).thenReturn(Optional.of(run)); + when(evaluationSampleRepository.findByEvaluationIdOrderByCreatedAtAsc("eval_001")).thenReturn(samples); + EvaluationModelClient modelClient = request -> { + if (request.filename().startsWith("fake")) { + throw new IllegalStateException("model service unavailable"); + } + return new EvaluationModelResult(ModelLabel.AUTHENTIC, 0.12, 31); + }; + + EvaluationDetailResponse response = service(modelClient).runEvaluation("eval_001"); + + assertThat(response.status()).isEqualTo(EvaluationStatus.FAILED); + assertThat(response.completedSamples()).isEqualTo(1); + assertThat(response.attemptCount()).isEqualTo(1); + assertThat(response.failureReason()).isEqualTo("model service unavailable"); + assertThat(response.samples().get(0).predictedLabel()).isEqualTo(ModelLabel.AUTHENTIC); + assertThat(response.samples().get(1).failureReason()).isEqualTo("model service unavailable"); + verify(evaluationRunRepository).save(run); + verify(evaluationSampleRepository).saveAll(samples); + } + + @Test + void retriesFailedEvaluationAndCompletesRemainingSamples() { + EvaluationRun run = queuedRun(); + List samples = List.of( + sample("sample_001", "real_001.jpg", ModelLabel.AUTHENTIC), + sample("sample_002", "fake_001.jpg", ModelLabel.SYNTHETIC)); + when(evaluationRunRepository.findByEvaluationId("eval_001")).thenReturn(Optional.of(run)); + when(evaluationSampleRepository.findByEvaluationIdOrderByCreatedAtAsc("eval_001")).thenReturn(samples); + RetryOnceModelClient modelClient = new RetryOnceModelClient(); + EvaluationExecutionService service = service(modelClient); + + EvaluationDetailResponse failed = service.runEvaluation("eval_001"); + EvaluationDetailResponse completed = service.runEvaluation("eval_001"); + + assertThat(failed.status()).isEqualTo(EvaluationStatus.FAILED); + assertThat(completed.status()).isEqualTo(EvaluationStatus.COMPLETED); + assertThat(completed.attemptCount()).isEqualTo(2); + assertThat(completed.completedSamples()).isEqualTo(2); + assertThat(completed.failureReason()).isNull(); + assertThat(completed.samples()) + .extracting(sample -> sample.failureReason()) + .containsExactly(null, null); + assertThat(modelClient.callCount()).isEqualTo(3); + } + + private EvaluationExecutionService service(EvaluationModelClient modelClient) { + return new EvaluationExecutionService( + evaluationRunRepository, + evaluationSampleRepository, + new EvaluationMetricsCalculator(), + modelClient, + Clock.fixed(NOW, ZoneOffset.UTC)); + } + + private EvaluationRun queuedRun() { + return new EvaluationRun( + "eval_001", + "Smoke Dataset", + "sample-v1", + "nonescape-mini", + EvaluationStatus.QUEUED, + 2, + 0, + null, + null, + null, + null, + 0, + 3, + NOW, + null, + null, + null); + } + + private EvaluationSample sample(String sampleId, String filename, ModelLabel groundTruth) { + return new EvaluationSample( + sampleId, + "eval_001", + filename, + groundTruth, + null, + null, + null, + null, + null, + NOW); + } + + private static final class RetryOnceModelClient implements EvaluationModelClient { + private int callCount; + private boolean failedFakeOnce; + + @Override + public EvaluationModelResult predict(com.fengting.aigcforensics.evaluation.client.EvaluationModelRequest request) { + callCount++; + if (request.filename().startsWith("fake") && !failedFakeOnce) { + failedFakeOnce = true; + throw new IllegalStateException("temporary model error"); + } + if (request.filename().startsWith("fake")) { + return new EvaluationModelResult(ModelLabel.SYNTHETIC, 0.91, 42); + } + return new EvaluationModelResult(ModelLabel.AUTHENTIC, 0.12, 31); + } + + int callCount() { + return callCount; + } + } +} diff --git a/docs/project-improvement-roadmap.md b/docs/project-improvement-roadmap.md index 64cc0ae..11c13cc 100644 --- a/docs/project-improvement-roadmap.md +++ b/docs/project-improvement-roadmap.md @@ -599,3 +599,13 @@ Nonescape Mini 权重更轻，适合 3090、本地 CPU fallback 和 Docker 演 - 第一片范围：评估运行表、评估样本表、CSV manifest 导入、指标计算、查询 API。 - 明确暂不做：评估图片集上传、Redis 批量执行、逐样本调用模型服务、前端评估页。 - 原因：先稳定数据模型和指标口径，再接异步执行和前端展示，避免一次性拉大范围。 + +2026-07-08 Phase B second backend slice: + +- Branch: `feature/evaluation-batch-execution`. +- Scope: batch execution orchestration, evaluation model-call boundary, failed + attempt persistence, manual retry, and generated sample predictions. +- Constraint: real model weights and GPU inference remain deferred to the later + server model integration branch. +- Next frontend branch: `feature/evaluation-frontend`, focused on listing + evaluation runs, showing metrics, and inspecting wrong samples. diff --git a/docs/project-worklog.md b/docs/project-worklog.md index 9502c7f..d85cd1a 100644 --- a/docs/project-worklog.md +++ b/docs/project-worklog.md @@ -46,6 +46,62 @@ Future acceptance: ## Timeline +### 2026-07-08: Evaluation Batch Execution Framework + +Branch: + +```text +feature/evaluation-batch-execution +``` + +What changed: + +- Added `attemptCount` and `maxAttempts` to evaluation runs. +- Added an evaluation-specific model boundary: + `EvaluationModelClient`, `EvaluationModelRequest`, and `EvaluationModelResult`. +- Added a deterministic placeholder model client for local execution without + model weights, GPU, or image files. +- Added `EvaluationExecutionService` to run queued evaluations sample by + sample. +- Added failure handling that marks the run `FAILED`, records the failing sample + reason, and persists partial progress. +- Added retry behavior that skips already completed samples and continues + remaining samples while attempts remain. +- Added `POST /api/evaluations/{evaluationId}/run`. +- Added `POST /api/evaluations/{evaluationId}/retry`. +- Added focused tests for persistence, model boundary, execution success, + execution failure, retry, and HTTP execution. + +Why: + +- The previous evaluation slice could store records and calculate metrics only + when predictions were already present in the manifest. +- This branch turns evaluation into a real backend workflow while still + respecting the current environment constraint: no model weights are downloaded + and no GPU runtime is required. +- The deterministic client is intentionally a boundary adapter, not a claimed + detector. It lets the Java orchestration, retry semantics, metrics, and + database writes become testable now, while leaving the real model adapter for + the later GPU-server branch. + +Verification: + +```powershell +cd backend-java +mvn -Dtest=EvaluationRepositoryTest test +mvn -Dtest=DeterministicEvaluationModelClientTest test +mvn -Dtest=EvaluationExecutionServiceTest test +mvn -Dtest=EvaluationControllerTest test +``` + +Deferred: + +- Downloading or loading `nonescape-mini-v0.onnx`. +- Resolving evaluation manifest filenames to uploaded dataset files. +- Asynchronous Redis-backed evaluation execution. +- Frontend evaluation dashboard and confusion matrix visualization. +- Replacing the deterministic local adapter with a Python model-service adapter. + ### 2026-07-08: Evaluation Backend Foundation Branch: @@ -340,21 +396,22 @@ Why these matter: ## Next Recommended Work -Start Phase B from `docs/project-improvement-roadmap.md`: +Continue Phase B from `docs/project-improvement-roadmap.md`: ```text -feature/evaluation-backend +feature/evaluation-frontend ``` Scope: -- Add evaluation database tables. -- Add manifest parsing. -- Add metrics calculation. -- Add evaluation task APIs. -- Keep frontend work for a separate branch. +- Add an evaluation list and detail page. +- Display status, attempts, aggregate metrics, and sample rows. +- Show wrong-sample filtering first; confusion matrix can follow in a later + polish branch. +- Keep the existing frontend visual style. Reason: -The project needs to prove model behavior with data. Evaluation is more valuable -for interviews than adding more UI pages before metrics exist. +The backend now has a measurable evaluation workflow. The next interview-visible +step is to make the evaluation result easy to inspect without changing the +project into a broad dashboard. diff --git a/docs/superpowers/plans/2026-07-08-evaluation-batch-execution.md b/docs/superpowers/plans/2026-07-08-evaluation-batch-execution.md new file mode 100644 index 0000000..48ae55b --- /dev/null +++ b/docs/superpowers/plans/2026-07-08-evaluation-batch-execution.md @@ -0,0 +1,215 @@ +# Evaluation Batch Execution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn queued evaluation records into an executable backend batch framework with model-call boundaries, retry state, and persisted sample results, without downloading or loading model weights. + +**Architecture:** Keep evaluation creation and metric calculation in `EvaluationService`, and add a separate `EvaluationExecutionService` for state transitions and batch orchestration. Add a small `EvaluationModelClient` boundary so the executor can later call a real model service; for this branch it uses a deterministic placeholder implementation that is stable, testable, and clearly replaceable. + +**Tech Stack:** Java 21, Spring Boot, Spring MVC, Spring Data JPA, Flyway, H2 tests, JUnit 5, AssertJ, Mockito. + +## Global Constraints + +- Do not download model weights or require GPU/runtime model files in this branch. +- Keep the implementation backend-first; do not change the existing frontend UI. +- Preserve existing API behavior for `POST /api/evaluations`, `GET /api/evaluations`, `GET /api/evaluations/{id}`, and sample listing. +- Use Java business code and follow current package conventions under `com.fengting.aigcforensics`. +- Add database changes via Flyway migrations, not Hibernate auto-DDL. +- Use explicit tests for execution success, failed execution, and retry behavior. + +--- + +## File Structure + +- Create `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java`: orchestrates a single batch evaluation attempt. +- Create `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java`: model-call interface for evaluation samples. +- Create `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java`: immutable request DTO for model calls. +- Create `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java`: immutable result DTO for model calls. +- Create `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java`: local placeholder model adapter. +- Modify `EvaluationRun`: add attempt counters and state transition methods. +- Modify `EvaluationSample`: add methods to mark predicted or failed. +- Modify `EvaluationService`: expose response mapping for execution service and include attempt fields in responses. +- Modify `EvaluationController`: add execution and retry endpoints. +- Modify DTO records: include `attemptCount` and `maxAttempts`. +- Add `backend-java/src/main/resources/db/migration/V4__add_evaluation_execution_state.sql`: persist attempt fields. +- Add `EvaluationExecutionServiceTest`: service-level orchestration tests. +- Extend `EvaluationControllerTest`: API-level execution endpoint coverage. +- Extend `EvaluationRepositoryTest`: migration/entity validation for attempt fields. +- Update `docs/project-worklog.md`: record the implementation rationale and progress. + +## Task 1: Persist Retry State + +**Files:** +- Create: `backend-java/src/main/resources/db/migration/V4__add_evaluation_execution_state.sql` +- Modify: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationRun.java` +- Modify: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationRunResponse.java` +- Modify: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationDetailResponse.java` +- Modify: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationService.java` +- Test: `backend-java/src/test/java/com/fengting/aigcforensics/evaluation/repository/EvaluationRepositoryTest.java` + +**Interfaces:** +- Produces `EvaluationRun.getAttemptCount()`, `EvaluationRun.getMaxAttempts()`, `EvaluationRun.markStarted(Instant)`, `EvaluationRun.markCompleted(...)`, `EvaluationRun.markFailed(String, Instant)`, `EvaluationRun.canRetry()`. +- Produces response fields `attemptCount` and `maxAttempts`. + +- [ ] **Step 1: Write failing repository/DTO test** + +Add assertions that a saved run exposes `attemptCount = 1`, `maxAttempts = 3`, and can be reloaded with those values. + +- [ ] **Step 2: Verify RED** + +Run: `mvn -B -Dtest=EvaluationRepositoryTest test` +Expected: compilation failure because attempt fields do not exist yet. + +- [ ] **Step 3: Implement minimal persistence and response mapping** + +Add Flyway columns, entity fields, constructor parameters, getters, and response record fields. + +- [ ] **Step 4: Verify GREEN** + +Run: `mvn -B -Dtest=EvaluationRepositoryTest test` +Expected: test passes. + +## Task 2: Add Model Invocation Boundary + +**Files:** +- Create: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java` +- Create: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java` +- Create: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java` +- Create: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java` +- Test: `backend-java/src/test/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClientTest.java` + +**Interfaces:** +- Consumes `ModelLabel`. +- Produces `EvaluationModelClient.predict(EvaluationModelRequest request): EvaluationModelResult`. + +- [ ] **Step 1: Write failing deterministic client test** + +Assert the client returns stable labels and scores for the same filename/model pair and does not require external files. + +- [ ] **Step 2: Verify RED** + +Run: `mvn -B -Dtest=DeterministicEvaluationModelClientTest test` +Expected: compilation failure because the client types do not exist. + +- [ ] **Step 3: Implement minimal client boundary** + +Create records, interface, and deterministic implementation. + +- [ ] **Step 4: Verify GREEN** + +Run: `mvn -B -Dtest=DeterministicEvaluationModelClientTest test` +Expected: test passes. + +## Task 3: Execute Queued Evaluations + +**Files:** +- Create: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java` +- Modify: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationSample.java` +- Modify: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationService.java` +- Test: `backend-java/src/test/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionServiceTest.java` + +**Interfaces:** +- Consumes `EvaluationModelClient.predict(EvaluationModelRequest)`. +- Produces `EvaluationExecutionService.runEvaluation(String evaluationId): EvaluationDetailResponse`. + +- [ ] **Step 1: Write failing success test** + +Create a queued evaluation with missing predictions, run it, and assert status `COMPLETED`, all samples predicted, and metrics persisted. + +- [ ] **Step 2: Verify RED** + +Run: `mvn -B -Dtest=EvaluationExecutionServiceTest test` +Expected: compilation failure because `EvaluationExecutionService` does not exist. + +- [ ] **Step 3: Implement success path** + +Load run and samples, mark started, call model client for missing predictions, save samples, calculate metrics, and mark completed. + +- [ ] **Step 4: Verify GREEN** + +Run: `mvn -B -Dtest=EvaluationExecutionServiceTest test` +Expected: success-path test passes. + +## Task 4: Failed Attempts and Retry + +**Files:** +- Modify: `EvaluationExecutionService.java` +- Modify: `EvaluationRun.java` +- Modify: `EvaluationSample.java` +- Test: `EvaluationExecutionServiceTest.java` + +**Interfaces:** +- Produces failure status with `failureReason`. +- Allows rerun of `FAILED` evaluations while `attemptCount < maxAttempts`. + +- [ ] **Step 1: Write failing failure/retry tests** + +Assert one model exception marks the run failed with a reason, then a second execution can complete if attempts remain. + +- [ ] **Step 2: Verify RED** + +Run: `mvn -B -Dtest=EvaluationExecutionServiceTest test` +Expected: failure because retry behavior is missing. + +- [ ] **Step 3: Implement failure and retry state transitions** + +Increment attempt on every execution, clear stale failure state at start, preserve completed sample predictions, and fail fast with a clear message when attempts are exhausted. + +- [ ] **Step 4: Verify GREEN** + +Run: `mvn -B -Dtest=EvaluationExecutionServiceTest test` +Expected: all execution tests pass. + +## Task 5: Expose HTTP Execution API + +**Files:** +- Modify: `backend-java/src/main/java/com/fengting/aigcforensics/evaluation/controller/EvaluationController.java` +- Modify: `backend-java/src/test/java/com/fengting/aigcforensics/evaluation/controller/EvaluationControllerTest.java` + +**Interfaces:** +- Produces `POST /api/evaluations/{evaluationId}/run`. +- Produces `POST /api/evaluations/{evaluationId}/retry`. + +- [ ] **Step 1: Write failing controller test** + +Create a queued evaluation through HTTP, run it through HTTP, and assert the response includes completed samples, attempts, and metrics. + +- [ ] **Step 2: Verify RED** + +Run: `mvn -B -Dtest=EvaluationControllerTest test` +Expected: 404 for missing endpoint. + +- [ ] **Step 3: Implement controller endpoints** + +Delegate both endpoints to `EvaluationExecutionService.runEvaluation`. + +- [ ] **Step 4: Verify GREEN** + +Run: `mvn -B -Dtest=EvaluationControllerTest test` +Expected: controller tests pass. + +## Task 6: Documentation and Full Verification + +**Files:** +- Modify: `docs/project-worklog.md` +- Modify: `docs/project-improvement-roadmap.md` + +**Interfaces:** +- Produces a readable worklog entry explaining what was built, why it was scoped this way, and what remains for real model execution. + +- [ ] **Step 1: Update docs** + +Record the execution framework, deterministic placeholder, retry scope, and deferred GPU/model-weight work. + +- [ ] **Step 2: Run backend verification** + +Run: `mvn -B test` +Expected: all backend tests pass. + +- [ ] **Step 3: Commit and push** + +Run: +`git add backend-java docs && git commit -m "feat: add evaluation batch execution framework" && git push -u origin feature/evaluation-batch-execution` + +Expected: branch is pushed to GitHub for PR/CI. +