Add evaluation batch execution framework

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/DeterministicEvaluationModelClient.java

@@ -0,0 +1,27 @@
+package com.fengting.aigcforensics.evaluation.client;
+
+import java.nio.charset.StandardCharsets;
+import java.util.zip.CRC32;
+
+import org.springframework.stereotype.Component;
+
+import com.fengting.aigcforensics.domain.ModelLabel;
+
+@Component
+public class DeterministicEvaluationModelClient implements EvaluationModelClient {
+
+    @Override
+    public EvaluationModelResult predict(EvaluationModelRequest request) {
+        double score = stableScore(request.modelId() + ":" + request.filename());
+        ModelLabel label = score >= 0.5 ? ModelLabel.SYNTHETIC : ModelLabel.AUTHENTIC;
+        int latencyMs = 10 + (int) Math.round(score * 20);
+        return new EvaluationModelResult(label, score, latencyMs);
+    }
+
+    private double stableScore(String value) {
+        CRC32 crc32 = new CRC32();
+        crc32.update(value.getBytes(StandardCharsets.UTF_8));
+        return crc32.getValue() / (double) 0xffffffffL;
+    }
+}
+

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelClient.java

@@ -0,0 +1,7 @@
+package com.fengting.aigcforensics.evaluation.client;
+
+public interface EvaluationModelClient {
+
+    EvaluationModelResult predict(EvaluationModelRequest request);
+}
+

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelRequest.java

@@ -0,0 +1,9 @@
+package com.fengting.aigcforensics.evaluation.client;
+
+public record EvaluationModelRequest(
+        String evaluationId,
+        String sampleId,
+        String modelId,
+        String filename) {
+}
+

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/client/EvaluationModelResult.java

@@ -0,0 +1,10 @@
+package com.fengting.aigcforensics.evaluation.client;
+
+import com.fengting.aigcforensics.domain.ModelLabel;
+
+public record EvaluationModelResult(
+        ModelLabel predictedLabel,
+        double score,
+        int latencyMs) {
+}
+

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/controller/EvaluationController.java

@@ -16,6 +16,7 @@
 import com.fengting.aigcforensics.evaluation.dto.EvaluationDetailResponse;
 import com.fengting.aigcforensics.evaluation.dto.EvaluationRunResponse;
 import com.fengting.aigcforensics.evaluation.dto.EvaluationSampleResponse;
+import com.fengting.aigcforensics.evaluation.service.EvaluationExecutionService;
 import com.fengting.aigcforensics.evaluation.service.EvaluationService;
 
 import jakarta.validation.Valid;
@@ -25,9 +26,13 @@
 public class EvaluationController {
 
     private final EvaluationService evaluationService;
+    private final EvaluationExecutionService evaluationExecutionService;
 
-    public EvaluationController(EvaluationService evaluationService) {
+    public EvaluationController(
+            EvaluationService evaluationService,
+            EvaluationExecutionService evaluationExecutionService) {
         this.evaluationService = evaluationService;
+        this.evaluationExecutionService = evaluationExecutionService;
     }
 
     @PostMapping
@@ -46,6 +51,16 @@ public EvaluationDetailResponse getEvaluation(@PathVariable String evaluationId)
         return evaluationService.getEvaluation(evaluationId);
     }
 
+    @PostMapping("/{evaluationId}/run")
+    public EvaluationDetailResponse runEvaluation(@PathVariable String evaluationId) {
+        return evaluationExecutionService.runEvaluation(evaluationId);
+    }
+
+    @PostMapping("/{evaluationId}/retry")
+    public EvaluationDetailResponse retryEvaluation(@PathVariable String evaluationId) {
+        return evaluationExecutionService.runEvaluation(evaluationId);
+    }
+
     @GetMapping("/{evaluationId}/samples")
     public List<EvaluationSampleResponse> listSamples(
             @PathVariable String evaluationId,

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationRun.java

@@ -53,6 +53,12 @@ public class EvaluationRun {
     @Column(name = "f1_score")
     private Double f1;
 
+    @Column(name = "attempt_count", nullable = false)
+    private int attemptCount;
+
+    @Column(name = "max_attempts", nullable = false)
+    private int maxAttempts;
+
     @Column(name = "created_at", nullable = false)
     private Instant createdAt;
 
@@ -80,6 +86,8 @@ public EvaluationRun(
             Double precision,
             Double recall,
             Double f1,
+            int attemptCount,
+            int maxAttempts,
             Instant createdAt,
             Instant startedAt,
             Instant completedAt,
@@ -95,6 +103,8 @@ public EvaluationRun(
         this.precision = precision;
         this.recall = recall;
         this.f1 = f1;
+        this.attemptCount = attemptCount;
+        this.maxAttempts = maxAttempts;
         this.createdAt = createdAt;
         this.startedAt = startedAt;
         this.completedAt = completedAt;
@@ -149,6 +159,14 @@ public Double getF1() {
         return f1;
     }
 
+    public int getAttemptCount() {
+        return attemptCount;
+    }
+
+    public int getMaxAttempts() {
+        return maxAttempts;
+    }
+
     public Instant getCreatedAt() {
         return createdAt;
     }
@@ -164,4 +182,40 @@ public Instant getCompletedAt() {
     public String getFailureReason() {
         return failureReason;
     }
+
+    public boolean canRetry() {
+        return attemptCount < maxAttempts;
+    }
+
+    public void markStarted(Instant startedAt) {
+        this.status = EvaluationStatus.RUNNING;
+        this.attemptCount++;
+        this.startedAt = startedAt;
+        this.completedAt = null;
+        this.failureReason = null;
+    }
+
+    public void markCompleted(
+            int completedSamples,
+            Double accuracy,
+            Double precision,
+            Double recall,
+            Double f1,
+            Instant completedAt) {
+        this.status = EvaluationStatus.COMPLETED;
+        this.completedSamples = completedSamples;
+        this.accuracy = accuracy;
+        this.precision = precision;
+        this.recall = recall;
+        this.f1 = f1;
+        this.completedAt = completedAt;
+        this.failureReason = null;
+    }
+
+    public void markFailed(String failureReason, int completedSamples, Instant completedAt) {
+        this.status = EvaluationStatus.FAILED;
+        this.completedSamples = completedSamples;
+        this.completedAt = completedAt;
+        this.failureReason = failureReason;
+    }
 }

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/domain/EvaluationSample.java

@@ -122,4 +122,16 @@ public String getFailureReason() {
     public Instant getCreatedAt() {
         return createdAt;
     }
+
+    public void markPredicted(ModelLabel predictedLabel, Double score, Integer latencyMs) {
+        this.predictedLabel = predictedLabel;
+        this.score = score;
+        this.latencyMs = latencyMs;
+        this.correct = predictedLabel == groundTruthLabel;
+        this.failureReason = null;
+    }
+
+    public void markFailed(String failureReason) {
+        this.failureReason = failureReason;
+    }
 }

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationDetailResponse.java

@@ -17,6 +17,8 @@ public record EvaluationDetailResponse(
         Double precision,
         Double recall,
         Double f1,
+        int attemptCount,
+        int maxAttempts,
         Instant createdAt,
         Instant startedAt,
         Instant completedAt,

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/dto/EvaluationRunResponse.java

@@ -16,6 +16,8 @@ public record EvaluationRunResponse(
         Double precision,
         Double recall,
         Double f1,
+        int attemptCount,
+        int maxAttempts,
         Instant createdAt,
         Instant startedAt,
         Instant completedAt,

backend-java/src/main/java/com/fengting/aigcforensics/evaluation/service/EvaluationExecutionService.java

@@ -0,0 +1,158 @@
+package com.fengting.aigcforensics.evaluation.service;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.fengting.aigcforensics.evaluation.client.EvaluationModelClient;
+import com.fengting.aigcforensics.evaluation.client.EvaluationModelRequest;
+import com.fengting.aigcforensics.evaluation.client.EvaluationModelResult;
+import com.fengting.aigcforensics.evaluation.domain.EvaluationRun;
+import com.fengting.aigcforensics.evaluation.domain.EvaluationSample;
+import com.fengting.aigcforensics.evaluation.domain.EvaluationStatus;
+import com.fengting.aigcforensics.evaluation.dto.EvaluationDetailResponse;
+import com.fengting.aigcforensics.evaluation.dto.EvaluationSampleResponse;
+import com.fengting.aigcforensics.evaluation.repository.EvaluationRunRepository;
+import com.fengting.aigcforensics.evaluation.repository.EvaluationSampleRepository;
+import com.fengting.aigcforensics.service.ResourceNotFoundException;
+
+@Service
+public class EvaluationExecutionService {
+
+    private final EvaluationRunRepository evaluationRunRepository;
+    private final EvaluationSampleRepository evaluationSampleRepository;
+    private final EvaluationMetricsCalculator metricsCalculator;
+    private final EvaluationModelClient modelClient;
+    private final Clock clock;
+
+    @Autowired
+    public EvaluationExecutionService(
+            EvaluationRunRepository evaluationRunRepository,
+            EvaluationSampleRepository evaluationSampleRepository,
+            EvaluationMetricsCalculator metricsCalculator,
+            EvaluationModelClient modelClient) {
+        this(
+                evaluationRunRepository,
+                evaluationSampleRepository,
+                metricsCalculator,
+                modelClient,
+                Clock.systemUTC());
+    }
+
+    EvaluationExecutionService(
+            EvaluationRunRepository evaluationRunRepository,
+            EvaluationSampleRepository evaluationSampleRepository,
+            EvaluationMetricsCalculator metricsCalculator,
+            EvaluationModelClient modelClient,
+            Clock clock) {
+        this.evaluationRunRepository = evaluationRunRepository;
+        this.evaluationSampleRepository = evaluationSampleRepository;
+        this.metricsCalculator = metricsCalculator;
+        this.modelClient = modelClient;
+        this.clock = clock;
+    }
+
+    @Transactional
+    public EvaluationDetailResponse runEvaluation(String evaluationId) {
+        EvaluationRun run = findRun(evaluationId);
+        List<EvaluationSample> samples = evaluationSampleRepository.findByEvaluationIdOrderByCreatedAtAsc(evaluationId);
+        if (run.getStatus() == EvaluationStatus.COMPLETED) {
+            return toDetailResponse(run, samples);
+        }
+        if (run.getStatus() == EvaluationStatus.FAILED && !run.canRetry()) {
+            throw new IllegalStateException("Evaluation retry attempts exhausted: " + evaluationId);
+        }
+
+        run.markStarted(Instant.now(clock));
+        for (EvaluationSample sample : samples) {
+            if (sample.getPredictedLabel() != null) {
+                continue;
+            }
+            try {
+                EvaluationModelResult result = modelClient.predict(new EvaluationModelRequest(
+                        run.getEvaluationId(),
+                        sample.getSampleId(),
+                        run.getModelId(),
+                        sample.getFilename()));
+                sample.markPredicted(result.predictedLabel(), result.score(), result.latencyMs());
+            } catch (RuntimeException exception) {
+                String failureReason = failureMessage(exception);
+                sample.markFailed(failureReason);
+                run.markFailed(failureReason, countCompleted(samples), Instant.now(clock));
+                evaluationSampleRepository.saveAll(samples);
+                evaluationRunRepository.save(run);
+                return toDetailResponse(run, samples);
+            }
+        }
+
+        EvaluationMetrics metrics = metricsCalculator.calculate(samples.stream()
+                .map(sample -> new EvaluationPredictionCase(sample.getGroundTruthLabel(), sample.getPredictedLabel()))
+                .toList());
+        run.markCompleted(
+                samples.size(),
+                metrics.accuracy(),
+                metrics.precision(),
+                metrics.recall(),
+                metrics.f1(),
+                Instant.now(clock));
+        evaluationSampleRepository.saveAll(samples);
+        evaluationRunRepository.save(run);
+        return toDetailResponse(run, samples);
+    }
+
+    private int countCompleted(List<EvaluationSample> samples) {
+        return (int) samples.stream().filter(sample -> sample.getPredictedLabel() != null).count();
+    }
+
+    private String failureMessage(RuntimeException exception) {
+        if (exception.getMessage() == null || exception.getMessage().isBlank()) {
+            return exception.getClass().getSimpleName();
+        }
+        return exception.getMessage();
+    }
+
+    private EvaluationRun findRun(String evaluationId) {
+        return evaluationRunRepository.findByEvaluationId(evaluationId)
+                .orElseThrow(() -> new ResourceNotFoundException("Evaluation not found: " + evaluationId));
+    }
+
+    private EvaluationDetailResponse toDetailResponse(EvaluationRun run, List<EvaluationSample> samples) {
+        return new EvaluationDetailResponse(
+                run.getEvaluationId(),
+                run.getName(),
+                run.getDatasetName(),
+                run.getModelId(),
+                run.getStatus(),
+                run.getTotalSamples(),
+                run.getCompletedSamples(),
+                run.getAccuracy(),
+                run.getPrecision(),
+                run.getRecall(),
+                run.getF1(),
+                run.getAttemptCount(),
+                run.getMaxAttempts(),
+                run.getCreatedAt(),
+                run.getStartedAt(),
+                run.getCompletedAt(),
+                run.getFailureReason(),
+                samples.stream().map(this::toSampleResponse).toList());
+    }
+
+    private EvaluationSampleResponse toSampleResponse(EvaluationSample sample) {
+        return new EvaluationSampleResponse(
+                sample.getSampleId(),
+                sample.getEvaluationId(),
+                sample.getFilename(),
+                sample.getGroundTruthLabel(),
+                sample.getPredictedLabel(),
+                sample.getScore(),
+                sample.getLatencyMs(),
+                sample.getCorrect(),
+                sample.getFailureReason(),
+                sample.getCreatedAt());
+    }
+}