diff --git a/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportService.java b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportService.java index 5818b3ab..e78b7cab 100644 --- a/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportService.java +++ b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportService.java @@ -5,11 +5,13 @@ import com.devkor.ifive.nadab.domain.dailyreport.api.dto.response.CreateAnswerImageUploadUrlResponse; import com.devkor.ifive.nadab.domain.dailyreport.api.dto.response.CreateDailyReportResponse; import com.devkor.ifive.nadab.domain.dailyreport.api.dto.response.ImageStatusResponse; +import com.devkor.ifive.nadab.domain.dailyreport.application.helper.DailyReportModelSelector; import com.devkor.ifive.nadab.domain.dailyreport.core.dto.ConfirmDailyAndRewardDto; import com.devkor.ifive.nadab.domain.dailyreport.core.dto.PrepareDailyResultDto; import com.devkor.ifive.nadab.domain.dailyreport.core.dto.AiDailyReportResultDto; import com.devkor.ifive.nadab.domain.dailyreport.core.entity.AnswerEntry; import com.devkor.ifive.nadab.domain.dailyreport.core.entity.ImageStatus; +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.ModelCandidate; import com.devkor.ifive.nadab.domain.dailyreport.infra.DailyReportLlmClient; import com.devkor.ifive.nadab.domain.question.core.entity.DailyQuestion; import com.devkor.ifive.nadab.domain.question.core.entity.UserDailyQuestion; @@ -48,6 +50,7 @@ public class DailyReportService { private final DailyReportTxService dailyReportTxService; private final ProfileImageService profileImageService; + private final DailyReportModelSelector dailyReportModelSelector; private final DailyReportLlmClient dailyReportLlmClient; private final ReportGenerationLogRecorder reportGenerationLogRecorder; @@ -88,19 +91,20 @@ public CreateDailyReportResponse generateDailyReport(Long userId, DailyReportReq PrepareDailyResultDto prep = dailyReportTxService.prepareDaily(user, question, request.answer(), isDayPassed, request.objectKey()); AnswerEntry answerEntry = prep.entry(); + ModelCandidate modelCandidate = dailyReportModelSelector.select(); Long generationLogId = reportGenerationLogRecorder.start( userId, ReportGenerationType.DAILY, prep.reportId(), ReportGenerationStep.DAILY_GENERATE, LlmProvider.OPENAI, - dailyReportLlmClient.model() + modelCandidate.getModel() ); AiDailyReportResultDto dto; try { LlmGenerationResult generationResult = - dailyReportLlmClient.generate(question.getQuestionText(), answerEntry); + dailyReportLlmClient.generate(question.getQuestionText(), answerEntry, modelCandidate); dto = generationResult.content(); LlmTokenUsage tokenUsage = generationResult.tokenUsage(); reportGenerationLogRecorder.recordTokenUsage( diff --git a/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/application/helper/DailyReportModelSelector.java b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/application/helper/DailyReportModelSelector.java new file mode 100644 index 00000000..32760e70 --- /dev/null +++ b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/application/helper/DailyReportModelSelector.java @@ -0,0 +1,45 @@ +package com.devkor.ifive.nadab.domain.dailyreport.application.helper; + +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties; +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.ModelCandidate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.IntUnaryOperator; + +@Component +public class DailyReportModelSelector { + + private static final int TOTAL_WEIGHT = 100; + + private final DailyReportLlmProperties properties; + private final IntUnaryOperator randomValueGenerator; + + @Autowired + public DailyReportModelSelector(DailyReportLlmProperties properties) { + this(properties, bound -> ThreadLocalRandom.current().nextInt(bound)); + } + + DailyReportModelSelector( + DailyReportLlmProperties properties, + IntUnaryOperator randomValueGenerator + ) { + this.properties = properties; + this.randomValueGenerator = randomValueGenerator; + } + + public ModelCandidate select() { + int randomValue = randomValueGenerator.applyAsInt(TOTAL_WEIGHT); + int cumulativeWeight = 0; + + for (ModelCandidate candidate : properties.getCandidates()) { + cumulativeWeight += candidate.getWeight(); + if (randomValue < cumulativeWeight) { + return candidate; + } + } + + throw new IllegalStateException("Failed to select a DailyReport LLM model candidate"); + } +} diff --git a/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/core/properties/DailyReportLlmProperties.java b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/core/properties/DailyReportLlmProperties.java index 2c0b4a0c..82fc14af 100644 --- a/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/core/properties/DailyReportLlmProperties.java +++ b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/core/properties/DailyReportLlmProperties.java @@ -1,9 +1,13 @@ package com.devkor.ifive.nadab.domain.dailyreport.core.properties; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.DecimalMax; import jakarta.validation.constraints.DecimalMin; +import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; @@ -11,6 +15,11 @@ import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; + @Component @Getter @Setter @@ -18,20 +27,60 @@ @ConfigurationProperties(prefix = "daily-report.llm") public class DailyReportLlmProperties { - @NotBlank - private String model = "gpt-4o-mini"; + @Valid + @NotEmpty + private List<@NotNull ModelCandidate> candidates = new ArrayList<>(); + + @AssertTrue(message = "daily-report.llm.candidates weights must total 100") + public boolean isCandidateWeightTotalValid() { + if (candidates == null || candidates.isEmpty()) { + return true; + } + + return candidates.stream() + .filter(Objects::nonNull) + .mapToInt(ModelCandidate::getWeight) + .sum() == 100; + } + + @AssertTrue(message = "daily-report.llm.candidates models must be unique") + public boolean isCandidateModelUnique() { + if (candidates == null || candidates.isEmpty()) { + return true; + } + + List models = candidates.stream() + .filter(Objects::nonNull) + .map(ModelCandidate::getModel) + .filter(Objects::nonNull) + .toList(); - @DecimalMin("0.0") - @DecimalMax("2.0") - private double temperature = 0.3; + return new HashSet<>(models).size() == models.size(); + } + + @Getter + @Setter + public static class ModelCandidate { + + @NotBlank + private String model; - @Min(1) - private int maxOutputTokens = 512; + @Min(1) + @Max(100) + private int weight; - @NotNull - private TokenLimitParameter tokenLimitParameter = TokenLimitParameter.MAX_TOKENS; + @DecimalMin("0.0") + @DecimalMax("2.0") + private double temperature; - private String reasoningEffort; + @Min(1) + private int maxOutputTokens; + + @NotNull + private TokenLimitParameter tokenLimitParameter; + + private String reasoningEffort; + } public enum TokenLimitParameter { MAX_TOKENS, diff --git a/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClient.java b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClient.java index 019e81d9..1937ffc5 100644 --- a/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClient.java +++ b/src/main/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClient.java @@ -3,7 +3,7 @@ import com.devkor.ifive.nadab.domain.dailyreport.core.dto.AiDailyReportResultDto; import com.devkor.ifive.nadab.domain.dailyreport.core.dto.LlmDailyResultDto; import com.devkor.ifive.nadab.domain.dailyreport.core.entity.AnswerEntry; -import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties; +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.ModelCandidate; import com.devkor.ifive.nadab.domain.user.infra.ProfileImageUrlBuilder; import com.devkor.ifive.nadab.global.core.prompt.daily.DailyReportPromptLoader; import com.devkor.ifive.nadab.global.core.response.ErrorCode; @@ -37,11 +37,14 @@ public class DailyReportLlmClient { private final ObjectMapper objectMapper; private final LlmRouter llmRouter; private final ProfileImageUrlBuilder profileImageUrlBuilder; - private final DailyReportLlmProperties properties; private final LlmProvider provider = LlmProvider.OPENAI; - public LlmGenerationResult generate(String question, AnswerEntry answerEntry) { + public LlmGenerationResult generate( + String question, + AnswerEntry answerEntry, + ModelCandidate modelCandidate + ) { String answer = answerEntry.getContent(); @@ -55,7 +58,7 @@ public LlmGenerationResult generate(String question, Ans ChatClient chatClient = llmRouter.route(provider); - OpenAiChatOptions options = buildOptions(); + OpenAiChatOptions options = buildOptions(modelCandidate); UserMessage userMessage = buildUserMessage(prompt, withImagePrompt,answerEntry); @@ -103,22 +106,18 @@ public LlmGenerationResult generate(String question, Ans } } - public String model() { - return properties.getModel(); - } - - private OpenAiChatOptions buildOptions() { + private OpenAiChatOptions buildOptions(ModelCandidate modelCandidate) { var builder = OpenAiChatOptions.builder() - .model(properties.getModel()) - .temperature(properties.getTemperature()); + .model(modelCandidate.getModel()) + .temperature(modelCandidate.getTemperature()); - switch (properties.getTokenLimitParameter()) { - case MAX_TOKENS -> builder.maxTokens(properties.getMaxOutputTokens()); - case MAX_COMPLETION_TOKENS -> builder.maxCompletionTokens(properties.getMaxOutputTokens()); + switch (modelCandidate.getTokenLimitParameter()) { + case MAX_TOKENS -> builder.maxTokens(modelCandidate.getMaxOutputTokens()); + case MAX_COMPLETION_TOKENS -> builder.maxCompletionTokens(modelCandidate.getMaxOutputTokens()); } - if (!isBlank(properties.getReasoningEffort())) { - builder.reasoningEffort(properties.getReasoningEffort()); + if (!isBlank(modelCandidate.getReasoningEffort())) { + builder.reasoningEffort(modelCandidate.getReasoningEffort()); } return builder.build(); diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index b6e4a0f6..e97fd634 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -15,14 +15,6 @@ spring: ddl-auto: validate show-sql: false -daily-report: - llm: - model: gpt-5.6-luna - temperature: 1.0 - max-output-tokens: 512 - token-limit-parameter: MAX_COMPLETION_TOKENS - reasoning-effort: none - springdoc: api-docs: enabled: true diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 950ec18d..1949036d 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -97,10 +97,18 @@ api_prefix: /api/v1 daily-report: llm: - model: gpt-4o-mini - temperature: 0.3 - max-output-tokens: 512 - token-limit-parameter: MAX_TOKENS + candidates: + - model: gpt-4o-mini + weight: 50 + temperature: 0.3 + max-output-tokens: 512 + token-limit-parameter: MAX_TOKENS + - model: gpt-5.6-luna + weight: 50 + temperature: 1.0 + max-output-tokens: 512 + token-limit-parameter: MAX_COMPLETION_TOKENS + reasoning-effort: none ask-chat: answer: diff --git a/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportServiceTest.java b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportServiceTest.java index 909e9ddd..ba29125c 100644 --- a/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportServiceTest.java +++ b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/application/DailyReportServiceTest.java @@ -2,12 +2,14 @@ import com.devkor.ifive.nadab.domain.dailyreport.api.dto.request.DailyReportRequest; import com.devkor.ifive.nadab.domain.dailyreport.api.dto.response.CreateDailyReportResponse; +import com.devkor.ifive.nadab.domain.dailyreport.application.helper.DailyReportModelSelector; import com.devkor.ifive.nadab.domain.dailyreport.core.dto.AiDailyReportResultDto; import com.devkor.ifive.nadab.domain.dailyreport.core.dto.ConfirmDailyAndRewardDto; import com.devkor.ifive.nadab.domain.dailyreport.core.dto.PrepareDailyResultDto; import com.devkor.ifive.nadab.domain.dailyreport.core.entity.AnswerEntry; import com.devkor.ifive.nadab.domain.dailyreport.core.entity.Emotion; import com.devkor.ifive.nadab.domain.dailyreport.core.entity.EmotionCode; +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.ModelCandidate; import com.devkor.ifive.nadab.domain.dailyreport.infra.DailyReportLlmClient; import com.devkor.ifive.nadab.domain.question.core.entity.DailyQuestion; import com.devkor.ifive.nadab.domain.question.core.entity.UserDailyQuestion; @@ -61,6 +63,9 @@ class DailyReportServiceTest { @Mock ProfileImageService profileImageService; + @Mock + DailyReportModelSelector dailyReportModelSelector; + @Mock DailyReportLlmClient dailyReportLlmClient; @@ -80,6 +85,7 @@ void setUp() { userDailyQuestionRepository, dailyReportTxService, profileImageService, + dailyReportModelSelector, dailyReportLlmClient, reportGenerationLogRecorder, profileImageUrlBuilder @@ -99,6 +105,7 @@ void generate_daily_report_records_token_usage_before_succeeding_generation_log( AnswerEntry answerEntry = AnswerEntry.create(user, question, "answer", today, null); AiDailyReportResultDto aiResult = new AiDailyReportResultDto("message", "ACHIEVEMENT"); Emotion emotion = emotion(EmotionCode.ACHIEVEMENT); + ModelCandidate modelCandidate = modelCandidate("gpt-5.6-luna"); when(userRepository.findById(userId)).thenReturn(Optional.of(user)); when(dailyQuestionRepository.findByIdWithInterest(20L)).thenReturn(Optional.of(question)); @@ -106,7 +113,7 @@ void generate_daily_report_records_token_usage_before_succeeding_generation_log( .thenReturn(Optional.of(UserDailyQuestion.create(user, today, question))); when(dailyReportTxService.prepareDaily(user, question, "answer", false, null)) .thenReturn(new PrepareDailyResultDto(answerEntry, reportId, userId)); - when(dailyReportLlmClient.model()).thenReturn("gpt-5.6-luna"); + when(dailyReportModelSelector.select()).thenReturn(modelCandidate); when(reportGenerationLogRecorder.start( userId, ReportGenerationType.DAILY, @@ -115,7 +122,7 @@ void generate_daily_report_records_token_usage_before_succeeding_generation_log( LlmProvider.OPENAI, "gpt-5.6-luna" )).thenReturn(generationLogId); - when(dailyReportLlmClient.generate("question", answerEntry)) + when(dailyReportLlmClient.generate("question", answerEntry, modelCandidate)) .thenReturn(new LlmGenerationResult<>(aiResult, new LlmTokenUsage(100L, 50L, 150L))); when(dailyReportTxService.confirmDailyAndReward( any(PrepareDailyResultDto.class), @@ -133,6 +140,8 @@ void generate_daily_report_records_token_usage_before_succeeding_generation_log( assertThat(response.reportId()).isEqualTo(reportId); assertThat(response.content()).isEqualTo("message"); assertThat(response.balanceAfter()).isEqualTo(110L); + verify(dailyReportModelSelector).select(); + verify(dailyReportLlmClient).generate("question", answerEntry, modelCandidate); InOrder inOrder = inOrder(reportGenerationLogRecorder); inOrder.verify(reportGenerationLogRecorder).recordTokenUsage(generationLogId, 100L, 50L, 150L, null); @@ -151,6 +160,7 @@ void generate_daily_report_records_null_token_usage_when_usage_is_empty() { AnswerEntry answerEntry = AnswerEntry.create(user, question, "answer", today, null); AiDailyReportResultDto aiResult = new AiDailyReportResultDto("message", "ACHIEVEMENT"); Emotion emotion = emotion(EmotionCode.ACHIEVEMENT); + ModelCandidate modelCandidate = modelCandidate("gpt-4o-mini"); when(userRepository.findById(userId)).thenReturn(Optional.of(user)); when(dailyQuestionRepository.findByIdWithInterest(20L)).thenReturn(Optional.of(question)); @@ -158,9 +168,10 @@ void generate_daily_report_records_null_token_usage_when_usage_is_empty() { .thenReturn(Optional.of(UserDailyQuestion.create(user, today, question))); when(dailyReportTxService.prepareDaily(user, question, "answer", false, null)) .thenReturn(new PrepareDailyResultDto(answerEntry, reportId, userId)); + when(dailyReportModelSelector.select()).thenReturn(modelCandidate); when(reportGenerationLogRecorder.start(any(), any(), any(), any(), any(), any())) .thenReturn(generationLogId); - when(dailyReportLlmClient.generate("question", answerEntry)) + when(dailyReportLlmClient.generate("question", answerEntry, modelCandidate)) .thenReturn(new LlmGenerationResult<>(aiResult, LlmTokenUsage.empty())); when(dailyReportTxService.confirmDailyAndReward(any(), eq(aiResult), eq(null))) .thenReturn(new ConfirmDailyAndRewardDto(emotion, 110L)); @@ -190,4 +201,10 @@ private Emotion emotion(EmotionCode code) { when(emotion.getCode()).thenReturn(code); return emotion; } + + private ModelCandidate modelCandidate(String model) { + ModelCandidate modelCandidate = new ModelCandidate(); + modelCandidate.setModel(model); + return modelCandidate; + } } diff --git a/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/application/helper/DailyReportModelSelectorTest.java b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/application/helper/DailyReportModelSelectorTest.java new file mode 100644 index 00000000..018b6cc1 --- /dev/null +++ b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/application/helper/DailyReportModelSelectorTest.java @@ -0,0 +1,64 @@ +package com.devkor.ifive.nadab.domain.dailyreport.application.helper; + +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties; +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.ModelCandidate; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +class DailyReportModelSelectorTest { + + @ParameterizedTest + @CsvSource({ + "0, gpt-4o-mini", + "49, gpt-4o-mini", + "50, gpt-5.6-luna", + "99, gpt-5.6-luna" + }) + void select_maps_random_value_to_weighted_candidate(int randomValue, String expectedModel) { + DailyReportLlmProperties properties = new DailyReportLlmProperties(); + properties.setCandidates(List.of( + candidate("gpt-4o-mini", 50), + candidate("gpt-5.6-luna", 50) + )); + DailyReportModelSelector selector = new DailyReportModelSelector(properties, ignored -> randomValue); + + ModelCandidate selected = selector.select(); + + assertThat(selected.getModel()).isEqualTo(expectedModel); + } + + @Test + void select_draws_again_on_each_invocation() { + DailyReportLlmProperties properties = new DailyReportLlmProperties(); + properties.setCandidates(List.of( + candidate("gpt-4o-mini", 50), + candidate("gpt-5.6-luna", 50) + )); + int[] randomValues = {0, 50}; + AtomicInteger invocationCount = new AtomicInteger(); + DailyReportModelSelector selector = new DailyReportModelSelector( + properties, + ignored -> randomValues[invocationCount.getAndIncrement()] + ); + + ModelCandidate first = selector.select(); + ModelCandidate second = selector.select(); + + assertThat(first.getModel()).isEqualTo("gpt-4o-mini"); + assertThat(second.getModel()).isEqualTo("gpt-5.6-luna"); + assertThat(invocationCount).hasValue(2); + } + + private ModelCandidate candidate(String model, int weight) { + ModelCandidate candidate = new ModelCandidate(); + candidate.setModel(model); + candidate.setWeight(weight); + return candidate; + } +} diff --git a/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/core/properties/DailyReportLlmPropertiesTest.java b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/core/properties/DailyReportLlmPropertiesTest.java new file mode 100644 index 00000000..edee9f9c --- /dev/null +++ b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/core/properties/DailyReportLlmPropertiesTest.java @@ -0,0 +1,132 @@ +package com.devkor.ifive.nadab.domain.dailyreport.core.properties; + +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.ModelCandidate; +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.TokenLimitParameter; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.ConfigDataApplicationContextInitializer; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +class DailyReportLlmPropertiesTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withInitializer(new ConfigDataApplicationContextInitializer()) + .withUserConfiguration(PropertiesConfiguration.class); + + private final Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); + + @Test + void application_configuration_binds_two_equal_weight_candidates_with_model_specific_options() { + contextRunner.run(context -> { + assertThat(context.getStartupFailure()).isNull(); + + DailyReportLlmProperties properties = context.getBean(DailyReportLlmProperties.class); + + assertThat(properties.getCandidates()) + .extracting(ModelCandidate::getModel, ModelCandidate::getWeight) + .containsExactly( + tuple("gpt-4o-mini", 50), + tuple("gpt-5.6-luna", 50) + ); + + ModelCandidate mini = properties.getCandidates().get(0); + assertThat(mini.getTemperature()).isEqualTo(0.3); + assertThat(mini.getMaxOutputTokens()).isEqualTo(512); + assertThat(mini.getTokenLimitParameter()).isEqualTo(TokenLimitParameter.MAX_TOKENS); + assertThat(mini.getReasoningEffort()).isNull(); + + ModelCandidate luna = properties.getCandidates().get(1); + assertThat(luna.getTemperature()).isEqualTo(1.0); + assertThat(luna.getMaxOutputTokens()).isEqualTo(512); + assertThat(luna.getTokenLimitParameter()).isEqualTo(TokenLimitParameter.MAX_COMPLETION_TOKENS); + assertThat(luna.getReasoningEffort()).isEqualTo("none"); + }); + } + + @ParameterizedTest + @ValueSource(strings = {"local", "dev", "prod", "test"}) + void runtime_profiles_inherit_shared_candidates(String profile) { + contextRunner + .withPropertyValues("spring.profiles.active=" + profile) + .run(context -> { + assertThat(context.getStartupFailure()).isNull(); + + DailyReportLlmProperties properties = context.getBean(DailyReportLlmProperties.class); + + assertThat(properties.getCandidates()) + .extracting(ModelCandidate::getModel, ModelCandidate::getWeight) + .containsExactly( + tuple("gpt-4o-mini", 50), + tuple("gpt-5.6-luna", 50) + ); + }); + } + + @Test + void validation_rejects_candidate_weights_that_do_not_total_100() { + DailyReportLlmProperties properties = propertiesWith( + candidate("gpt-4o-mini", 40), + candidate("gpt-5.6-luna", 50) + ); + + assertThat(validator.validate(properties)) + .extracting(ConstraintViolation::getMessage) + .contains("daily-report.llm.candidates weights must total 100"); + } + + @Test + void validation_rejects_duplicate_candidate_models() { + DailyReportLlmProperties properties = propertiesWith( + candidate("gpt-4o-mini", 50), + candidate("gpt-4o-mini", 50) + ); + + assertThat(validator.validate(properties)) + .extracting(ConstraintViolation::getMessage) + .contains("daily-report.llm.candidates models must be unique"); + } + + @Test + void validation_rejects_invalid_nested_candidate_options() { + ModelCandidate invalidCandidate = candidate("gpt-4o-mini", 100); + invalidCandidate.setMaxOutputTokens(0); + + DailyReportLlmProperties properties = propertiesWith(invalidCandidate); + + assertThat(validator.validate(properties)) + .extracting(violation -> violation.getPropertyPath().toString()) + .contains("candidates[0].maxOutputTokens"); + } + + private DailyReportLlmProperties propertiesWith(ModelCandidate... candidates) { + DailyReportLlmProperties properties = new DailyReportLlmProperties(); + properties.setCandidates(List.of(candidates)); + return properties; + } + + private ModelCandidate candidate(String model, int weight) { + ModelCandidate candidate = new ModelCandidate(); + candidate.setModel(model); + candidate.setWeight(weight); + candidate.setTemperature(0.3); + candidate.setMaxOutputTokens(512); + candidate.setTokenLimitParameter(TokenLimitParameter.MAX_TOKENS); + return candidate; + } + + @Configuration(proxyBeanMethods = false) + @EnableConfigurationProperties(DailyReportLlmProperties.class) + static class PropertiesConfiguration { + } +} diff --git a/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClientTest.java b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClientTest.java index f5ffd78d..a0730486 100644 --- a/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClientTest.java +++ b/src/test/java/com/devkor/ifive/nadab/domain/dailyreport/infra/DailyReportLlmClientTest.java @@ -1,7 +1,7 @@ package com.devkor.ifive.nadab.domain.dailyreport.infra; import com.devkor.ifive.nadab.domain.dailyreport.core.entity.AnswerEntry; -import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties; +import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.ModelCandidate; import com.devkor.ifive.nadab.domain.dailyreport.core.properties.DailyReportLlmProperties.TokenLimitParameter; import com.devkor.ifive.nadab.domain.question.core.entity.DailyQuestion; import com.devkor.ifive.nadab.domain.user.core.entity.User; @@ -56,12 +56,17 @@ class DailyReportLlmClientTest { ChatClient.CallResponseSpec callResponseSpec; @Test - void generate_uses_default_options_and_returns_content_with_token_usage() { + void generate_uses_gpt_4o_mini_options_and_returns_content_with_token_usage() { // given - DailyReportLlmProperties properties = new DailyReportLlmProperties(); + ModelCandidate modelCandidate = modelCandidate( + "gpt-4o-mini", + 0.3, + TokenLimitParameter.MAX_TOKENS, + null + ); // when - GenerationExecution execution = generateWith(properties); + GenerationExecution execution = generateWith(modelCandidate); // then assertThat(execution.result().content()).isEqualTo(new com.devkor.ifive.nadab.domain.dailyreport.core.dto.AiDailyReportResultDto( @@ -79,17 +84,17 @@ void generate_uses_default_options_and_returns_content_with_token_usage() { } @Test - void generate_uses_dev_experiment_options() { + void generate_uses_gpt_5_6_luna_options() { // given - DailyReportLlmProperties properties = new DailyReportLlmProperties(); - properties.setModel("gpt-5.6-luna"); - properties.setTemperature(1.0); - properties.setMaxOutputTokens(512); - properties.setTokenLimitParameter(TokenLimitParameter.MAX_COMPLETION_TOKENS); - properties.setReasoningEffort("none"); + ModelCandidate modelCandidate = modelCandidate( + "gpt-5.6-luna", + 1.0, + TokenLimitParameter.MAX_COMPLETION_TOKENS, + "none" + ); // when - GenerationExecution execution = generateWith(properties); + GenerationExecution execution = generateWith(modelCandidate); // then assertThat(execution.options().getModel()).isEqualTo("gpt-5.6-luna"); @@ -99,13 +104,12 @@ void generate_uses_dev_experiment_options() { assertThat(execution.options().getReasoningEffort()).isEqualTo("none"); } - private GenerationExecution generateWith(DailyReportLlmProperties properties) { + private GenerationExecution generateWith(ModelCandidate modelCandidate) { DailyReportLlmClient client = new DailyReportLlmClient( dailyReportPromptLoader, new ObjectMapper(), llmRouter, - profileImageUrlBuilder, - properties + profileImageUrlBuilder ); AnswerEntry answerEntry = AnswerEntry.create( User.createUser("test@test.com", "hashed_password"), @@ -127,7 +131,7 @@ private GenerationExecution generateWith(DailyReportLlmProperties properties) { new DefaultUsage(100, 50, 150) )); - LlmGenerationResult result = client.generate("question", answerEntry); + LlmGenerationResult result = client.generate("question", answerEntry, modelCandidate); ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(OpenAiChatOptions.class); verify(requestSpec).options(optionsCaptor.capture()); @@ -135,6 +139,21 @@ private GenerationExecution generateWith(DailyReportLlmProperties properties) { return new GenerationExecution(result, optionsCaptor.getValue()); } + private ModelCandidate modelCandidate( + String model, + double temperature, + TokenLimitParameter tokenLimitParameter, + String reasoningEffort + ) { + ModelCandidate modelCandidate = new ModelCandidate(); + modelCandidate.setModel(model); + modelCandidate.setTemperature(temperature); + modelCandidate.setMaxOutputTokens(512); + modelCandidate.setTokenLimitParameter(tokenLimitParameter); + modelCandidate.setReasoningEffort(reasoningEffort); + return modelCandidate; + } + private ChatResponse chatResponse(String content, DefaultUsage usage) { return new ChatResponse( List.of(new Generation(new AssistantMessage(content))),