From c460aabaa70d31d343067f37b4055ba5f3472f2d Mon Sep 17 00:00:00 2001 From: doswalt Date: Wed, 8 Jul 2026 15:35:20 -0400 Subject: [PATCH 01/28] removed mooclet infrastructure, implemented native thompson-sampling algorithm, added weight estimation mechanism Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/skills/setup-perftrace/SKILL.md | 2 +- CLAUDE.md | 75 + .../js/src/UpGradeClient/UpgradeClient.ts | 2 +- clientlibs/js/src/types/Interfaces.ts | 7 - clientlibs/python/BUILD_PLAN.md | 2 +- .../python/src/upgrade_client_lib/client.py | 2 +- .../src/upgrade_client_lib/types/__init__.py | 2 - .../src/upgrade_client_lib/types/responses.py | 9 - clientlibs/python/tests/test_api_service.py | 5 +- clientlibs/python/tests/test_client.py | 5 +- packages/backend/.env.docker.local.example | 9 - packages/backend/.env.example | 9 - packages/backend/CLAUDE.md | 5 +- .../rest-client-vscode/MoocletAPI.http | 204 -- packages/backend/src/api/DTO/ExperimentDTO.ts | 29 +- .../ExperimentClientController.v6.ts | 23 +- .../api/controllers/ExperimentController.ts | 125 +- .../backend/src/api/errors/MoocletError.ts | 14 - .../api/middlewares/ErrorHandlerMiddleware.ts | 8 - .../src/api/models/ConditionPosteriorState.ts | 43 + .../src/api/models/MoocletExperimentRef.ts | 37 - .../api/models/MoocletVersionConditionMap.ts | 28 - .../ThompsonSamplingExperimentConfig.ts | 34 + .../src/api/models/ThompsonSamplingReward.ts | 31 + .../ConditionPosteriorStateRepository.ts | 17 + .../MoocletExperimentRefRepository.ts | 31 - ...mpsonSamplingExperimentConfigRepository.ts | 38 + .../ThompsonSamplingRewardRepository.ts | 17 + .../services/ExperimentAssignmentService.ts | 115 +- .../src/api/services/ExperimentService.ts | 16 +- .../src/api/services/ImportExportService.ts | 45 +- .../src/api/services/MoocletDataService.ts | 394 ---- .../api/services/MoocletExperimentService.ts | 1630 --------------- .../src/api/services/MoocletRewardsService.ts | 353 ---- .../ThompsonSamplingExperimentCrudService.ts | 142 ++ .../services/ThompsonSamplingRewardService.ts | 162 ++ .../api/services/ThompsonSamplingService.ts | 183 ++ .../1781222400000-thompsonSamplingEntities.ts | 118 ++ .../1781308800000-cleanupMoocletEntities.ts | 46 + ...200000-bootstrapThompsonSamplingConfigs.ts | 44 + packages/backend/src/env.ts | 6 - packages/backend/src/types/Mooclet.ts | 111 - .../controllers/ExperimentController.test.ts | 33 +- .../mocks/MoocletExperimentServiceMock.ts | 9 - .../mocks/MoocletRewardsServiceMock.ts | 25 - .../ExperimentAssignmentService.test.ts | 20 +- .../unit/services/ExperimentService.test.ts | 12 +- .../unit/services/MoocletDataService.test.ts | 571 ----- .../services/MoocletExperimentService.test.ts | 1855 ----------------- .../services/MoocletRewardsService.test.ts | 1345 ------------ .../services/ThompsonSamplingService.test.ts | 296 +++ .../src/app/core/api-endpoints.constants.ts | 2 +- .../experiments/experiments.data.service.ts | 10 +- .../core/experiments/experiments.service.ts | 10 +- .../mooclet-helper.service.spec.ts | 483 ----- .../experiments/mooclet-helper.service.ts | 232 --- .../store/experiments.effects.spec.ts | 74 - .../experiments/store/experiments.effects.ts | 10 +- .../experiments/store/experiments.model.ts | 19 +- .../store/experiments.selectors.ts | 9 +- .../thompson-sampling-helper.service.ts | 110 + .../edit-condition-prior-modal.component.ts | 6 +- ...able-policy-parameters-form.component.html | 22 +- ...urable-policy-parameters-form.component.ts | 100 +- .../upsert-experiment-modal.component.html | 15 +- .../upsert-experiment-modal.component.ts | 108 +- ...ent-conditions-section-card.component.html | 4 +- ...iment-conditions-section-card.component.ts | 10 +- ...experiment-conditions-table.component.html | 10 +- .../experiment-conditions-table.component.ts | 4 +- ...periment-details-page-content.component.ts | 9 +- ...nt-condition-expandable-row.component.html | 16 +- ...ment-condition-expandable-row.component.ts | 11 +- ...igurable-reward-count-table.component.html | 16 + ...igurable-reward-count-table.component.scss | 4 + ...nfigurable-reward-count-table.component.ts | 4 +- .../projects/upgrade/src/assets/i18n/en.json | 18 +- .../src/environments/environment-types.ts | 3 +- .../src/environments/environment.bsnl.ts | 1 - .../src/environments/environment.demo.prod.ts | 1 - .../environments/environment.local.example.ts | 1 - .../src/environments/environment.prod.ts | 1 - .../src/environments/environment.qa.ts | 1 - .../src/environments/environment.staging.ts | 1 - .../upgrade/src/environments/environment.ts | 1 - packages/types/CLAUDE.md | 1 - packages/types/src/Experiment/enums.ts | 6 +- packages/types/src/Experiment/interfaces.ts | 29 + .../src/Mooclet/MoocletPolicyParametersDTO.ts | 9 - ...oocletTSConfigurablePolicyParametersDTO.ts | 56 - packages/types/src/Mooclet/index.ts | 49 - packages/types/src/index.ts | 7 +- postman/ClientAPI.postman_collection.json | 2 +- 93 files changed, 1737 insertions(+), 8092 deletions(-) delete mode 100644 packages/backend/rest-client-vscode/MoocletAPI.http delete mode 100644 packages/backend/src/api/errors/MoocletError.ts create mode 100644 packages/backend/src/api/models/ConditionPosteriorState.ts delete mode 100644 packages/backend/src/api/models/MoocletExperimentRef.ts delete mode 100644 packages/backend/src/api/models/MoocletVersionConditionMap.ts create mode 100644 packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts create mode 100644 packages/backend/src/api/models/ThompsonSamplingReward.ts create mode 100644 packages/backend/src/api/repositories/ConditionPosteriorStateRepository.ts delete mode 100644 packages/backend/src/api/repositories/MoocletExperimentRefRepository.ts create mode 100644 packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts create mode 100644 packages/backend/src/api/repositories/ThompsonSamplingRewardRepository.ts delete mode 100644 packages/backend/src/api/services/MoocletDataService.ts delete mode 100644 packages/backend/src/api/services/MoocletExperimentService.ts delete mode 100644 packages/backend/src/api/services/MoocletRewardsService.ts create mode 100644 packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts create mode 100644 packages/backend/src/api/services/ThompsonSamplingRewardService.ts create mode 100644 packages/backend/src/api/services/ThompsonSamplingService.ts create mode 100644 packages/backend/src/database/migrations/1781222400000-thompsonSamplingEntities.ts create mode 100644 packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts create mode 100644 packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts delete mode 100644 packages/backend/src/types/Mooclet.ts delete mode 100644 packages/backend/test/unit/controllers/mocks/MoocletExperimentServiceMock.ts delete mode 100644 packages/backend/test/unit/controllers/mocks/MoocletRewardsServiceMock.ts delete mode 100644 packages/backend/test/unit/services/MoocletDataService.test.ts delete mode 100644 packages/backend/test/unit/services/MoocletExperimentService.test.ts delete mode 100644 packages/backend/test/unit/services/MoocletRewardsService.test.ts create mode 100644 packages/backend/test/unit/services/ThompsonSamplingService.test.ts delete mode 100644 packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.spec.ts delete mode 100644 packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.ts create mode 100644 packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts delete mode 100644 packages/types/src/Mooclet/MoocletPolicyParametersDTO.ts delete mode 100644 packages/types/src/Mooclet/MoocletTSConfigurablePolicyParametersDTO.ts delete mode 100644 packages/types/src/Mooclet/index.ts diff --git a/.claude/skills/setup-perftrace/SKILL.md b/.claude/skills/setup-perftrace/SKILL.md index 3ff573ccf1..92ef619f57 100644 --- a/.claude/skills/setup-perftrace/SKILL.md +++ b/.claude/skills/setup-perftrace/SKILL.md @@ -387,7 +387,7 @@ file they live in. Verify against the branch rather than trusting this table if | `PATCH /v6/groupmembership` | `experimentUserService.updateGroupMembership` | controller | | `PATCH /v6/workinggroup` | `experimentUserService.updateWorkingGroup` | controller | | `PATCH /v6/useraliases` | `experimentUserService.setAliasesForUser(aliases=N)` | controller | -| `POST /v6/reward` | `moocletRewardsService.sendReward` | controller | +| `POST /v6/reward` | `thompsonSamplingRewardService.recordReward` | controller | | `POST /v6/mark` | `experimentAssignmentService.markExperimentPoint` | controller | | " | 8 spans inside `markExperimentPoint` — `previewUserService.findOneFromCache`, `getCachedExperiments`, `checkUserOrGroupIsGloballyExcluded`, `experimentLevelExclusionInclusion`, `monitoredDecisionPointRepository.findOne`, `saveGroupExclusionDoc`, `updateEnrollmentExclusionDocumentsAndCheckEndingCriteria`, `monitoredDecisionPointRepository.saveRawJson` | `services/ExperimentAssignmentService.ts` | | `POST /v6/assign` | `formatAssignments` (sync) | controller | diff --git a/CLAUDE.md b/CLAUDE.md index b9efa9cda6..4989900d09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,3 +65,78 @@ Environment files: copy `packages/backend/.env.example` to `packages/backend/.en ## Package Versions All packages are versioned together (currently v6.5.0). Keep versions in sync when bumping. + +--- + +## Native Thompson Sampling Migration + +**Goal:** Remove the third-party MoocLet integration and replace it with a native TypeScript Thompson Sampling implementation. Retain UI visually as-is. No use of the term "mooclet" anywhere in the new codebase. + +**Reference:** MoocLet engine source at `~/Code/mooclet-engine` (read-only reference — do NOT copy logic from it). + +### Algorithm parameters carried over (renamed) + +| Old (MoocLet) | New | Purpose | +|---|---|---| +| `prior` | `prior` | Beta(α, β) priors per condition | +| `uniform_threshold` | `warmupThreshold` | Uniform random during cold-start | +| `batch_size` | `batchSize` | Batch posterior updates | +| `tspostdiff_thresh` | `minimumDrawDifference` | Fall back to uniform when arms converge | + +Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. + +### Progress + +**Phase 1 — Core algorithm (backend, pure functions)** +- [x] `ThompsonSamplingService` with `selectCondition()` — no DB or HTTP dependencies +- [x] Unit tests (14 passing) — warmup, priors, multi-arm, thresholds + +**Phase 2 — Data model** +- [x] New entity: `ThompsonSamplingExperimentConfig` (replaces `MoocletExperimentRef`) +- [x] New entity: `ConditionPosteriorState` per condition (stores current α/β) +- [x] New entity: `ThompsonSamplingReward` (raw reward events — audit trail + recalculation) +- [x] Repositories: `ThompsonSamplingExperimentConfigRepository`, `ConditionPosteriorStateRepository` +- [x] Add `ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING = 'thompson_sampling'` to `upgrade_types` +- [x] DB migration: `1781222400000-thompsonSamplingEntities` + +**Phase 3 — Assignment integration** +- [x] Wire `ThompsonSamplingService` into `ExperimentAssignmentService` — `assignThompsonSampling()` method handles THOMPSON_SAMPLING algorithm +- [x] `ThompsonSamplingRewardService` replaces `MoocletRewardsService` — stores `ThompsonSamplingReward` + increments `ConditionPosteriorState` counts +- [x] `POST /v6/reward` rewired to `ThompsonSamplingRewardService.recordReward()`; mooclet guard removed + +**Phase 4 — Experiment CRUD** +- [x] `ThompsonSamplingExperimentCrudService` — `createConfig`, `updateConfig`, `syncConditions` +- [x] `ExperimentController` create/update/delete handlers use native TS flow; mooclet routing removed +- [x] `ExperimentAssignmentService` mooclet branch removed; `handleEnrollCondition` mooclet path removed +- [x] `ImportExportService` mooclet routing removed from bulk create and export +- [x] Deleted: `MoocletExperimentService`, `MoocletDataService`, `MoocletRewardsService` + their tests/mocks + +**Phase 5 — Frontend** +- [x] `ThompsonSamplingHelperService` replaces `MoocletExperimentHelperService` across all components +- [x] NgRx state: `moocletPolicyParameters` → `thompsonSamplingConfig`; `ThompsonSamplingConfigDTO` added to model +- [x] Form fields renamed: `batch_size`→`batchSize`, `uniform_threshold`→`warmupThreshold`, `tspostdiff_thresh`→`minimumDrawDifference` +- [x] Algorithm check: `MOOCLET_TS_CONFIGURABLE` → `THOMPSON_SAMPLING` throughout +- [x] Dead rewards effect + data service method removed; `mooclet-rewards` API endpoint removed +- [x] `moocletToggle` feature flag removed from all env files +- [x] Deleted: `mooclet-helper.service.ts`, `mooclet-helper.service.spec.ts`; rewards effect tests removed + +**Phase 6 — Cleanup** +- [x] DB migration `1781308800000-cleanupMoocletEntities`: data-migrates `ts_configurable` → `thompson_sampling`, drops mooclet tables, removes `ts_configurable` from enum +- [x] Removed `MOOCLETS_*` env vars from `env.ts` and `.env.example` +- [x] `upgrade_types`: removed `Mooclet/` directory, `MOOCLET_TS_CONFIGURABLE` enum value, `SUPPORTED_MOOCLET_ALGORITHMS`; `Prior`, `BinaryRewardAllowedValue`, `ExperimentRewardsSummary` moved to `Experiment/interfaces.ts` +- [x] Deleted: mooclet models, repository, error class, types file +- [x] Removed mooclet validation from `ExperimentDTO`, mooclet inject from `ExperimentService`, mooclet error cases from `ErrorHandlerMiddleware` + +### Post-migration fixes + +- **Reward error type**: `ThompsonSamplingRewardService.throwConflictError` sets `(error as any).type = SERVER_ERROR.ASSIGNMENT_ERROR` explicitly. Without this, a plain `HttpError(409)` falls through to the middleware's httpCode switch where 409 → `SERVER_ERROR.DUPLICATE_KEY` (whose string is the unrelated feature flag message). Added `case SERVER_ERROR.ASSIGNMENT_ERROR` to `ErrorHandlerMiddleware` outer switch. + +- **thompsonSamplingConfig missing in API responses**: `getSingleExperiment`, `create`, and `update` all return `ExperimentDTO` without loading `thompsonSamplingConfig` (it's in a separate table). Added `attachThompsonSamplingConfig()` private method to `ExperimentController` that queries the config for THOMPSON_SAMPLING experiments and attaches it; called on all three response paths. Also added `getConfigForExperiment()` to `ThompsonSamplingExperimentCrudService`. + +- **Form/overview labels still showing old terms**: Updated `packages/frontend/projects/upgrade/src/assets/i18n/en.json` — `"uniform-threshold.label.text"` → "Warmup Threshold", `"tspostdiff-thresh.label.text"` → "Minimum Draw Difference" (with updated hints). The translation keys themselves are unchanged; only the values were updated. + +### Architecture notes + +- **conditionId as algorithm key**: `ThompsonSamplingService.selectCondition()` uses condition UUIDs (not `conditionCode`) as identifiers, since `conditionCode` is nullable. `ConditionPosteriorState` rows are keyed by `conditionId`. The `priors` field in `ThompsonSamplingConfigDTO` is therefore also keyed by conditionId. + +- **Reward summary endpoint**: `GET /experiments/rewards/:id` delegates to `ThompsonSamplingExperimentCrudService.getRewardsSummary()`, which queries `ConditionPosteriorState` rows joined to conditions, computes `successes`, `failures`, `successRate`, `priorSuccess`, `priorFailure` per condition, and sorts by condition order. Fully implemented. diff --git a/clientlibs/js/src/UpGradeClient/UpgradeClient.ts b/clientlibs/js/src/UpGradeClient/UpgradeClient.ts index 2bffb52fea..6740d58c49 100644 --- a/clientlibs/js/src/UpGradeClient/UpgradeClient.ts +++ b/clientlibs/js/src/UpGradeClient/UpgradeClient.ts @@ -630,7 +630,7 @@ export default class UpgradeClient { } /** - * Sends a binary reward signal for an adaptive experiment (Mooclet). + * Sends a binary reward signal for an adaptive experiment. * * This method allows sending reward feedback (SUCCESS or FAILURE) for adaptive experiments. * The reward is used by the adaptive algorithm to update its learning model and improve future assignments. diff --git a/clientlibs/js/src/types/Interfaces.ts b/clientlibs/js/src/types/Interfaces.ts index 28a2df222b..f6c83340b4 100644 --- a/clientlibs/js/src/types/Interfaces.ts +++ b/clientlibs/js/src/types/Interfaces.ts @@ -109,13 +109,6 @@ export namespace UpGradeClientInterfaces { context?: string; decisionPoint?: IDecisionPoint; }; - reward: { - variable: string; - value: number; - mooclet: number; - version: number; - learner: string; - }; } export interface IHttpClientWrapperRequestConfig { diff --git a/clientlibs/python/BUILD_PLAN.md b/clientlibs/python/BUILD_PLAN.md index c87951bbe7..32fdd26ea9 100644 --- a/clientlibs/python/BUILD_PLAN.md +++ b/clientlibs/python/BUILD_PLAN.md @@ -103,7 +103,7 @@ class BinaryRewardValue(str, Enum): - `MarkDecisionPointResponse` — userId, site, target, experimentId - `LogEventResponse` — id, uniquifier, timeStamp, data - `UserAliasResponse` — userId, aliases -- `SendRewardResponse` — message, request (rewardValue, experimentId, context, decisionPoint), reward (variable, value, mooclet, version, learner) +- `SendRewardResponse` — message, request (rewardValue, experimentId, context, decisionPoint) - `ErrorResponse` — message, httpStatusCode, type ### Deliverable diff --git a/clientlibs/python/src/upgrade_client_lib/client.py b/clientlibs/python/src/upgrade_client_lib/client.py index b7598b418a..c1bb28484c 100644 --- a/clientlibs/python/src/upgrade_client_lib/client.py +++ b/clientlibs/python/src/upgrade_client_lib/client.py @@ -330,7 +330,7 @@ async def send_reward( context: str | None = None, decision_point: dict[str, str] | None = None, ) -> SendRewardResponse: - """Send a binary reward signal for an adaptive (Mooclet) experiment.""" + """Send a binary reward signal for an adaptive experiment.""" return await self._api_service.send_reward( reward_value=reward_value, experiment_id=experiment_id, diff --git a/clientlibs/python/src/upgrade_client_lib/types/__init__.py b/clientlibs/python/src/upgrade_client_lib/types/__init__.py index ca47c22eb3..54f42dfa5e 100644 --- a/clientlibs/python/src/upgrade_client_lib/types/__init__.py +++ b/clientlibs/python/src/upgrade_client_lib/types/__init__.py @@ -33,7 +33,6 @@ LogEventResponse, MarkDecisionPointResponse, Payload, - RewardDetails, RewardRequest, SendRewardResponse, UserAliasResponse, @@ -71,7 +70,6 @@ "LogEventResponse", "MarkDecisionPointResponse", "Payload", - "RewardDetails", "RewardRequest", "SendRewardResponse", "UserAliasResponse", diff --git a/clientlibs/python/src/upgrade_client_lib/types/responses.py b/clientlibs/python/src/upgrade_client_lib/types/responses.py index 1771196a7a..174f864d61 100644 --- a/clientlibs/python/src/upgrade_client_lib/types/responses.py +++ b/clientlibs/python/src/upgrade_client_lib/types/responses.py @@ -74,18 +74,9 @@ class RewardRequest(BaseModel): decisionPoint: dict[str, str] | None = None -class RewardDetails(BaseModel): - variable: str - value: float - mooclet: int - version: int - learner: str - - class SendRewardResponse(BaseModel): message: str request: RewardRequest - reward: RewardDetails class ErrorResponse(BaseModel): diff --git a/clientlibs/python/tests/test_api_service.py b/clientlibs/python/tests/test_api_service.py index 71add228ad..ab5fefb991 100644 --- a/clientlibs/python/tests/test_api_service.py +++ b/clientlibs/python/tests/test_api_service.py @@ -404,7 +404,6 @@ async def test_sends_aliases(self) -> None: REWARD_RESPONSE = { "message": "Reward sent", "request": {"rewardValue": "SUCCESS", "experimentId": "exp-1"}, - "reward": {"variable": "score", "value": 1.0, "mooclet": 42, "version": 3, "learner": USER_ID}, } @@ -414,13 +413,13 @@ async def test_async_minimal(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_RESPONSE)) result = await make_service().send_reward(BinaryRewardValue.SUCCESS) assert result.message == "Reward sent" - assert result.reward.variable == "score" + assert result.request.experimentId == "exp-1" @respx.mock def test_sync(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_RESPONSE)) result = make_service().send_reward_sync(BinaryRewardValue.SUCCESS) - assert result.reward.mooclet == 42 + assert result.message == "Reward sent" @respx.mock async def test_full_params(self) -> None: diff --git a/clientlibs/python/tests/test_client.py b/clientlibs/python/tests/test_client.py index 4d878172ac..40eb978509 100644 --- a/clientlibs/python/tests/test_client.py +++ b/clientlibs/python/tests/test_client.py @@ -68,7 +68,6 @@ REWARD_PAYLOAD = { "message": "ok", "request": {"rewardValue": "SUCCESS"}, - "reward": {"variable": "score", "value": 1.0, "mooclet": 1, "version": 1, "learner": USER}, } @@ -596,7 +595,7 @@ async def test_async_success(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_PAYLOAD)) result = await make_client().send_reward(BinaryRewardValue.SUCCESS) assert result.message == "ok" - assert result.reward.variable == "score" + assert result.request.rewardValue == BinaryRewardValue.SUCCESS @respx.mock async def test_passes_all_params(self) -> None: @@ -617,4 +616,4 @@ async def test_passes_all_params(self) -> None: def test_sync(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_PAYLOAD)) result = make_client().send_reward_sync(BinaryRewardValue.SUCCESS) - assert result.reward.mooclet == 1 + assert result.message == "ok" diff --git a/packages/backend/.env.docker.local.example b/packages/backend/.env.docker.local.example index 4ed43d5a38..ad3122a644 100644 --- a/packages/backend/.env.docker.local.example +++ b/packages/backend/.env.docker.local.example @@ -88,15 +88,6 @@ EMAIL_EXPIRE_AFTER_SECONDS=36000 EMAIL_BUCKET="s3_bucket" EMAIL_TMP_WRITEABLE_FILE_PATH="src/api/assets/files/" -# -# Mooclets -# - -MOOCLETS_ENABLED = false -MOOCLETS_HOST_URL = mooclet_host_url -MOOCLETS_API_ROUTE = /engine/api/v1 -MOOCLETS_API_TOKEN = some_token - # # Initialization # diff --git a/packages/backend/.env.example b/packages/backend/.env.example index f00e631b02..5d7e529ceb 100644 --- a/packages/backend/.env.example +++ b/packages/backend/.env.example @@ -90,15 +90,6 @@ EMAIL_EXPIRE_AFTER_SECONDS=36000 EMAIL_BUCKET="s3_bucket" EMAIL_TMP_WRITEABLE_FILE_PATH="src/api/assets/files/" -# -# Mooclets -# - -MOOCLETS_ENABLED=false -MOOCLETS_HOST_URL=mooclet_host_url -MOOCLETS_API_ROUTE=/engine/api/v1 -MOOCLETS_API_TOKEN=some_token - # # Initialization # diff --git a/packages/backend/CLAUDE.md b/packages/backend/CLAUDE.md index 3621c8c259..a4337cebe5 100644 --- a/packages/backend/CLAUDE.md +++ b/packages/backend/CLAUDE.md @@ -140,14 +140,13 @@ Experiment join tables (`ExperimentSegmentInclusion` / `ExperimentSegmentExclusi | Segment list members updated | `ExperimentService.updateList` → `withRecompute` (passes `skipScheduleRecompute=true` to the segment upsert) | | Segment list removed | `ExperimentService.deleteList` → delegates to `SegmentService.deleteSegment` (which owns the recompute) | | Experiment created | `ExperimentService.create` → `await recomputeForExperiment` after lists attach (only when it owns the commit; recompute yields empty arrays for list-less experiments, so no separate empty-seed) | -| Experiment created inside a Mooclet transaction | `MoocletExperimentService.syncCreate` → `await recomputeForExperiment` after the transaction commits (create deferred it because it ran inside the transaction) | | Experiment lists imported | `ExperimentService.importExperimentLists` → `await recomputeForExperiment` after the import transaction commits | -| Experiment context changed (deletes all its lists) | `ExperimentService.updateExperimentInDB` → `scheduleRecomputeForExperiments` after commit (recomputes to empty; `deleteAllListsFromExperiment` deletes the private segments directly, so the precomputed row would otherwise keep stale IDs). When a caller owns the transaction (`MoocletExperimentService.syncUpdate` / `syncUpdateWithMoocletAlgorithmTransition`), those methods recompute after their own commit — mirrors the flag side's `updateFeatureFlagInDB` → `withRecompute` | +| Experiment context changed (deletes all its lists) | `ExperimentService.updateExperimentInDB` → `scheduleRecomputeForExperiments` after commit (recomputes to empty; `deleteAllListsFromExperiment` deletes the private segments directly, so the precomputed row would otherwise keep stale IDs) | | Shared segment members/structure changed | `SegmentService.addList` / `deleteList` / `addSegmentDataWithPipeline` → `scheduleRecomputeForSegment` for **both** the flag and experiment services | | Segment deleted entirely | `SegmentService.deleteSegment` → collects affected experiment IDs **before** the delete, recomputes **after** commit (flags use `withRecompute` in the same method) | | Server startup | `app.ts` → `backfillExperimentPrecomputedSegments` (guarded by `.catch` — a missing table never crashes startup) | -Same invariant as feature flags: recompute **after** the change commits; for deletes, collect affected experiment IDs **before**. All write-path recomputes are fire-and-forget except the `create` / import / Mooclet paths, which `await` so "done" means the row is ready. +Same invariant as feature flags: recompute **after** the change commits; for deletes, collect affected experiment IDs **before**. All write-path recomputes are fire-and-forget except the `create` / import paths, which `await` so "done" means the row is ready. ### INCLUDE_ALL semantics (resolved) diff --git a/packages/backend/rest-client-vscode/MoocletAPI.http b/packages/backend/rest-client-vscode/MoocletAPI.http deleted file mode 100644 index 3f2207cf38..0000000000 --- a/packages/backend/rest-client-vscode/MoocletAPI.http +++ /dev/null @@ -1,204 +0,0 @@ -# Mooclet -# To be used with the Rest Client extension for Visual Studio Code -# https://marketplace.visualstudio.com/items?itemName=humao.rest-client - -# This is not an exhaustive list, but running down the list emulates the process of -# of creating and running mooclet policies like UpGrade does - -# Other CRUD operations like GET / DELETE / PUT will work as expected -# https://docs.google.com/spreadsheets/d/1eL3z5zJfpraqEONYMVhiWgGYj6PSm8T8KFKFyeOMnHU/edit?gid=0#gid=0 - -### Creating a Mooclet -# 1. Get policy id by assignment algorithm name -# 2. Create mooclet -# 3. Create policyparameters -# 4. Create version 1 -# 5. Create version 2 -# 6. Create outcome variable - -#### Assigning and sending rewards -# 7. Create a learner (optional) -# 8. Get a new assignment -# 9. Check the policy parameters (optional to view changes to policy parameters) -# 10. Send reward -# 11. Repeat steps 8-10 as needed - -############ env variables -@host = https://apps.qa-cli.net/mooclet-service - -# Replace with your token, i.e. -@token = Token abc123 -# @token = - -@apiEndpoint = /engine/api/v1 - -############ request variables (change as needed) -@moocletId = 196 -@moocletName = newmooc4 -@policyId = 17 -@policyParametersId = 2 -@version1Name = controlz -@version1Id = -@version2Name = variant -@version2Id = 6 - -@outcomeVariableName = mooc_4_variable -@learnerName = dave - -########### Get policy id by assignment algorithm name -GET {{host}}{{apiEndpoint}}/policy HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -########### create mooclet -POST {{host}}{{apiEndpoint}}/mooclet HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{moocletName}}", - "policy": {{policyId}} -} - -########### create policyparameters -POST {{host}}{{apiEndpoint}}/policyparameters HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "policy": {{policyId}}, - "parameters": { - "prior": { - "failure": 1, - "success": 1 - }, - "batch_size": 1, - "max_rating": 1, - "min_rating": 0, - "uniform_threshold": 0, - "tspostdiff_thresh": 0, - "outcome_variable_name": "{{outcomeVariableName}}" - } -} - -########### create version 1 -POST {{host}}{{apiEndpoint}}/version HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "{{version1Name}}" -} - -########### create version 2 -POST {{host}}{{apiEndpoint}}/version HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "{{version2Name}}" -} - -########### create outcome variable -POST {{host}}{{apiEndpoint}}/variable HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{outcomeVariableName}}", - "value": 0 -} - -########### create a learner? -POST {{host}}{{apiEndpoint}}/learner HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{ learnerName }}" -} - -########### get a new assignment -GET {{host}}{{apiEndpoint}}/mooclet/{{moocletId}}/run HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -########### check the policy parameters -GET {{host}}{{apiEndpoint}}/policyparameters/{{policyParametersId}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -########### send reward -POST {{host}}{{apiEndpoint}}/value HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "variable": "{{outcomeVariableName}}", - "value": 0, - "mooclet": {{moocletId}}, - "version": {{version1Id}}, - "policy": {{policyId}} -} - -########## query rewards by mooclet id: -GET {{host}}{{apiEndpoint}}/value?mooclet={{moocletId}}&variable__name={{outcomeVariableName}} -Authorization: {{token}} -Content-type: application/json - -##### EDITS: - -########### create policyparameters -PUT {{host}}{{apiEndpoint}}/policyparameters/{{policyParametersId}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "policy": {{policyId}}, - "parameters": { - "prior": { - "failure": 1, - "success": 1 - }, - "batch_size": 1, - "max_rating": 1, - "min_rating": 0, - "uniform_threshold": 0, - "tspostdiff_thresh": 0, - "outcome_variable_name": "{{outcomeVariableName}}" - } -} - -########### update version 1 -PUT {{host}}{{apiEndpoint}}/version/{{version1Id}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "qwerty" -} - -########### update version 1 -GET {{host}}{{apiEndpoint}}/version/{{version1Id}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "qwerty" -} - -########### create outcome variable -PUT {{host}}{{apiEndpoint}}/variable HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{outcomeVariableName}}", -} - diff --git a/packages/backend/src/api/DTO/ExperimentDTO.ts b/packages/backend/src/api/DTO/ExperimentDTO.ts index 9cacd480dd..8120c75cea 100644 --- a/packages/backend/src/api/DTO/ExperimentDTO.ts +++ b/packages/backend/src/api/DTO/ExperimentDTO.ts @@ -3,7 +3,6 @@ import { IsArray, IsBoolean, IsDateString, - IsDefined, IsEnum, IsInt, IsNotEmpty, @@ -37,9 +36,6 @@ import { REPEATED_MEASURE, EXPERIMENT_TYPE, ASSIGNMENT_ALGORITHM, - MoocletTSConfigurablePolicyParametersDTO, - MoocletPolicyParametersDTO, - SUPPORTED_MOOCLET_ALGORITHMS, } from 'upgrade_types'; import { Type, Transform } from 'class-transformer'; @@ -471,9 +467,6 @@ abstract class BaseExperimentWithoutPayload { public type: EXPERIMENT_TYPE; } -const isMoocletAssignmentAlgorithm = (experiment: ExperimentDTO) => - experiment.assignmentAlgorithm && SUPPORTED_MOOCLET_ALGORITHMS.includes(experiment.assignmentAlgorithm); - function IsAssignmentUnitGroupConsistent(validationOptions?: ValidationOptions) { return function (object: any, propertyName: string) { registerDecorator({ @@ -509,21 +502,13 @@ export class ExperimentDTO extends BaseExperimentWithoutPayload { @Type(() => ConditionPayloadValidator) public conditionPayloads?: ConditionPayloadValidator[]; - // This should be validated when assignmentAlgorithm is not RANDOM or STRATIFIED_RANDOM_SAMPLING - @ValidateIf(isMoocletAssignmentAlgorithm) - @IsDefined() - @ValidateNested() - @Type(() => MoocletPolicyParametersDTO, { - discriminator: { - property: 'assignmentAlgorithm', - subTypes: [ - { value: MoocletTSConfigurablePolicyParametersDTO, name: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE }, - // Other policy types can be added here - ], - }, - keepDiscriminatorProperty: true, - }) - public moocletPolicyParameters?: MoocletPolicyParametersDTO; + @IsOptional() + public thompsonSamplingConfig?: { + warmupThreshold?: number; + minimumDrawDifference?: number; + batchSize?: number; + priors?: Record; + }; } export class OldExperimentDTO extends BaseExperimentWithoutPayload { diff --git a/packages/backend/src/api/controllers/ExperimentClientController.v6.ts b/packages/backend/src/api/controllers/ExperimentClientController.v6.ts index 527122a343..7132519602 100644 --- a/packages/backend/src/api/controllers/ExperimentClientController.v6.ts +++ b/packages/backend/src/api/controllers/ExperimentClientController.v6.ts @@ -8,7 +8,6 @@ import { Delete, Patch, Authorized, - BadRequestError, } from 'routing-controllers'; import { ExperimentService } from '../services/ExperimentService'; import { ExperimentAssignmentService } from '../services/ExperimentAssignmentService'; @@ -29,8 +28,10 @@ import { Log } from '../models/Log'; import { ExperimentUserValidatorv6 } from './validators/ExperimentUserValidator'; import { UserCheckMiddleware } from '../middlewares/UserCheckMiddleware'; import { RewardValidator } from './validators/RewardValidator'; -import { IRewardResponse, MoocletRewardsService } from '../services/MoocletRewardsService'; -import { env } from '../../env'; +import { + IThompsonSamplingRewardResponse, + ThompsonSamplingRewardService, +} from '../services/ThompsonSamplingRewardService'; interface IMonitoredDecisionPoint { id: string; @@ -98,7 +99,7 @@ export class ExperimentClientController { public experimentUserService: ExperimentUserService, public featureFlagService: FeatureFlagService, public metricService: MetricService, - public moocletRewardsService: MoocletRewardsService + public thompsonSamplingRewardService: ThompsonSamplingRewardService ) {} /** @@ -829,7 +830,7 @@ export class ExperimentClientController { * /v6/reward: * post: * description: | - * Send a reward signal for an adaptive experiment (Mooclet). + * Send a reward signal for an adaptive (Thompson Sampling) experiment. * * This endpoint allows sending binary reward feedback (SUCCESS or FAILURE) for adaptive experiments. * The reward is used by the adaptive algorithm to update its learning model and improve future assignments. @@ -934,9 +935,6 @@ export class ExperimentClientController { * type: string * target: * type: string - * reward: - * type: object - * description: The reward data that was sent to the Mooclet API * '400': * description: BadRequestError - Invalid parameters (e.g., missing required fields, invalid rewardValue) * '401': @@ -952,14 +950,9 @@ export class ExperimentClientController { request: AppRequest, @Body({ validate: true }) rewardData: RewardValidator - ): Promise { + ): Promise { request.logger.info({ message: 'Starting the sendReward call for user' }); - if (!env.mooclets?.enabled) { - throw new BadRequestError('Failed to send reward: mooclet is not currently enabled on backend.'); - } - - const experimentUserDoc = request.userDoc; - return this.moocletRewardsService.sendReward(experimentUserDoc, rewardData, request.logger); + return this.thompsonSamplingRewardService.recordReward(request.userDoc, rewardData, request.logger); } /** diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index fa4cb0a5ca..a839fac5a2 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -27,17 +27,17 @@ import { AssignmentStateUpdateValidator } from './validators/AssignmentStateUpda import { AppRequest, PaginationResponse } from '../../types'; import { ExperimentDTO, ExperimentFile, ValidatedExperimentError } from '../DTO/ExperimentDTO'; import { ExperimentIds } from './validators/ExperimentIdsValidator'; -import { MoocletExperimentService } from '../services/MoocletExperimentService'; -import { env } from '../../env'; +import { ThompsonSamplingExperimentCrudService } from '../services/ThompsonSamplingExperimentCrudService'; import { Response } from 'express'; import { NotFoundException } from '@nestjs/common/exceptions'; import { ExperimentIdValidator } from '../DTO/ExperimentDTO'; import { + ASSIGNMENT_ALGORITHM, CACHE_PREFIX, IImportError, LIST_FILTER_MODE, SERVER_ERROR, - SUPPORTED_MOOCLET_ALGORITHMS, + ExperimentRewardsSummary, } from 'upgrade_types'; import { ImportExportService } from '../services/ImportExportService'; import { getInstanceId } from '../../lib/instanceIdentity'; @@ -46,8 +46,6 @@ import { SegmentInputValidator } from './validators/SegmentInputValidator'; import { ExperimentSegmentExclusion } from '../models/ExperimentSegmentExclusion'; import { IdValidator } from './validators/ExperimentUserValidator'; import { Segment } from '../models/Segment'; -import { MoocletRewardsService } from '../services/MoocletRewardsService'; -import { ExperimentRewardsSummary } from 'upgrade_types'; import { CacheService } from '../services/CacheService'; interface ExperimentPaginationInfo extends PaginationResponse { @@ -661,10 +659,9 @@ export class ExperimentController { constructor( public experimentService: ExperimentService, public experimentAssignmentService: ExperimentAssignmentService, - public moocletExperimentService: MoocletExperimentService, - public moocletRewardService: MoocletRewardsService, public importExportService: ImportExportService, - public cacheService: CacheService + public cacheService: CacheService, + public thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService ) {} /** @@ -919,19 +916,16 @@ export class ExperimentController { @Params({ validate: true }) { id }: ExperimentIdValidator, @Req() request: AppRequest ): Promise { - let experiment = await this.experimentService.getSingleExperiment(id, request.logger); - - if (SUPPORTED_MOOCLET_ALGORITHMS.includes(experiment?.assignmentAlgorithm)) { - if (!env.mooclets?.enabled) { - throw new BadRequestError( - 'MoocletPolicyParameters are present in the experiment but Mooclet is not enabled in the environment' - ); - } else { - experiment = await this.moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, request.logger); - } - } + const experiment = await this.experimentService.getSingleExperiment(id, request.logger); + return this.attachThompsonSamplingConfig(experiment); + } - return experiment; + @Get('/rewards/:id') + public async getRewardsSummary( + @Params({ validate: true }) { id }: ExperimentIdValidator, + @Req() request: AppRequest + ): Promise { + return this.thompsonSamplingCrudService.getRewardsSummary(id); } /** @@ -1046,7 +1040,7 @@ export class ExperimentController { */ @Post() - public create( + public async create( @Body({ validate: true }) experiment: ExperimentDTO, @CurrentUser() currentUser: UserDTO, @Req() request: AppRequest @@ -1058,21 +1052,17 @@ export class ExperimentController { throw new BadRequestError(contextValidationError); } - if ('moocletPolicyParameters' in experiment) { - if (!env.mooclets?.enabled) { - throw new BadRequestError( - 'Failed to create Experiment: moocletPolicyParameters was provided but mooclets are not enabled on backend.' - ); - } else { - return this.moocletExperimentService.syncCreate({ - experimentDTO: experiment, - currentUser, - logger: request.logger, - }); - } + const createdExperiment = await this.experimentService.create(experiment, currentUser, request.logger); + + if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + await this.thompsonSamplingCrudService.createConfig( + createdExperiment.id, + createdExperiment.conditions, + experiment.thompsonSamplingConfig ?? {} + ); } - return this.experimentService.create(experiment, currentUser, request.logger); + return this.attachThompsonSamplingConfig(createdExperiment); } /** @@ -1155,20 +1145,6 @@ export class ExperimentController { ): Promise { request.logger.child({ user: currentUser }); - // Manually check if the experiment has a mooclet ref - if (env.mooclets.enabled) { - const moocletExperimentRef = await this.moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId(id); - - if (moocletExperimentRef) { - return await this.moocletExperimentService.syncDelete({ - moocletExperimentRef, - experimentId: id, - currentUser, - logger: request.logger, - }); - } - } - const experiment = await this.experimentService.delete(id, currentUser, { logger: request.logger }); if (!experiment) { @@ -1274,29 +1250,16 @@ export class ExperimentController { throw new BadRequestError(contextValidationError); } - if (env.mooclets.enabled) { - // if mooclet is enabled, we must check for potential assignment algorithm changes in all experiments - const updatedMoocletExperiment = - await this.moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - { ...experiment, id }, - currentUser, - request.logger - ); + const updatedExperiment = await this.experimentService.update({ ...experiment, id }, currentUser, request.logger); - if (updatedMoocletExperiment) { - return updatedMoocletExperiment; - } - } else { - // if mooclet is not enabled, but experiment has mooclet params, throw error - if ('moocletPolicyParameters' in experiment) { - throw new BadRequestError( - 'Failed to update Experiment: moocletPolicyParameters was provided but mooclets are not enabled on backend.' - ); + if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + await this.thompsonSamplingCrudService.syncConditions(id, updatedExperiment.conditions); + if (experiment.thompsonSamplingConfig) { + await this.thompsonSamplingCrudService.updateConfig(id, experiment.thompsonSamplingConfig); } } - // else, if mooclet is not involved, we can do a normal update - return this.experimentService.update({ ...experiment, id }, currentUser, request.logger); + return this.attachThompsonSamplingConfig(updatedExperiment); } /** @@ -1937,20 +1900,6 @@ export class ExperimentController { return lists; } - /** - * Get Mooclet Rewards Feedback data - */ - @Get('/mooclet-rewards/:id') - public getMoocletRewards( - @Params({ validate: true }) { id }: IdValidator, - @Req() request: AppRequest - ): Promise { - if (!env.mooclets?.enabled) { - throw new BadRequestError('Mooclet is not enabled in the environment'); - } - return this.moocletRewardService.getRewardsSummaryForExperiment(id, request.logger); - } - /** * Debugging endpoint: a cache report for THIS instance. * @@ -2018,4 +1967,18 @@ export class ExperimentController { summary, }; } + + private async attachThompsonSamplingConfig(experiment: ExperimentDTO): Promise { + if (experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + const config = await this.thompsonSamplingCrudService.getConfigForExperiment(experiment.id); + if (config) { + experiment.thompsonSamplingConfig = { + warmupThreshold: config.warmupThreshold, + minimumDrawDifference: config.minimumDrawDifference, + batchSize: config.batchSize, + }; + } + } + return experiment; + } } diff --git a/packages/backend/src/api/errors/MoocletError.ts b/packages/backend/src/api/errors/MoocletError.ts deleted file mode 100644 index a503939764..0000000000 --- a/packages/backend/src/api/errors/MoocletError.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { SERVER_ERROR } from 'upgrade_types'; -import { ErrorWithType } from './ErrorWithType'; - -export class MoocletError extends ErrorWithType { - public httpCode: number; - - constructor(message: string, httpCode = 500) { - super(); - this.message = message; - this.type = SERVER_ERROR.MOOCLET_ERROR; - this.httpCode = httpCode; - this.name = 'MoocletError'; - } -} diff --git a/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts b/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts index 673fd0866d..c1668fc078 100644 --- a/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts +++ b/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts @@ -79,14 +79,6 @@ export class ErrorHandlerMiddleware implements ExpressErrorMiddlewareInterface { type = SERVER_ERROR.EMAIL_SEND_ERROR; message = errorMessage; break; - case SERVER_ERROR.MOOCLET_REWARD_ERROR: - type = SERVER_ERROR.MOOCLET_REWARD_ERROR; - message = errorMessage; - break; - case SERVER_ERROR.MOOCLET_ERROR: - type = SERVER_ERROR.MOOCLET_ERROR; - message = errorMessage; - break; default: switch (error.httpCode) { case 400: diff --git a/packages/backend/src/api/models/ConditionPosteriorState.ts b/packages/backend/src/api/models/ConditionPosteriorState.ts new file mode 100644 index 0000000000..c4e5689f79 --- /dev/null +++ b/packages/backend/src/api/models/ConditionPosteriorState.ts @@ -0,0 +1,43 @@ +import { Entity, Column, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Unique } from 'typeorm'; +import { ThompsonSamplingExperimentConfig } from './ThompsonSamplingExperimentConfig'; +import { ExperimentCondition } from './ExperimentCondition'; +import { BaseModel } from './base/BaseModel'; + +@Entity() +@Unique(['configId', 'conditionId']) +export class ConditionPosteriorState extends BaseModel { + @PrimaryGeneratedColumn('uuid') + public id: string; + + @ManyToOne(() => ThompsonSamplingExperimentConfig, (config) => config.conditionPosteriorStates, { + onDelete: 'CASCADE', + }) + @JoinColumn({ name: 'configId' }) + config: ThompsonSamplingExperimentConfig; + + @Column() + configId: string; + + @ManyToOne(() => ExperimentCondition, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'conditionId' }) + condition: ExperimentCondition; + + @Column() + conditionId: string; + + /** α₀ — Beta prior success count for this condition. */ + @Column({ type: 'float', default: 1 }) + priorSuccess: number; + + /** β₀ — Beta prior failure count for this condition. */ + @Column({ type: 'float', default: 1 }) + priorFailure: number; + + /** Accumulated successes since the experiment started. */ + @Column({ type: 'int', default: 0 }) + successCount: number; + + /** Total rewards received for this condition (successes + failures). */ + @Column({ type: 'int', default: 0 }) + totalCount: number; +} diff --git a/packages/backend/src/api/models/MoocletExperimentRef.ts b/packages/backend/src/api/models/MoocletExperimentRef.ts deleted file mode 100644 index 64e75762a1..0000000000 --- a/packages/backend/src/api/models/MoocletExperimentRef.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Entity, Column, OneToMany, PrimaryColumn, JoinColumn, OneToOne } from 'typeorm'; -import { MoocletVersionConditionMap } from './MoocletVersionConditionMap'; -import { Experiment } from './Experiment'; - -@Entity() -export class MoocletExperimentRef { - @PrimaryColumn('uuid') - public id?: string; - - @OneToOne(() => Experiment, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'experimentId' }) - experiment: Experiment; - - @Column({ nullable: true }) - experimentId?: string; - - @Column() - moocletId?: number; - - @OneToMany(() => MoocletVersionConditionMap, (versionConditionMap) => versionConditionMap.moocletExperimentRef, { - cascade: true, - onDelete: 'CASCADE', - }) - versionConditionMaps: MoocletVersionConditionMap[]; - - @Column({ nullable: true }) - policyId?: number; - - @Column({ nullable: true }) - policyParametersId?: number; - - @Column({ nullable: true }) - variableId?: number; - - @Column({ nullable: true }) - outcomeVariableName?: string; -} diff --git a/packages/backend/src/api/models/MoocletVersionConditionMap.ts b/packages/backend/src/api/models/MoocletVersionConditionMap.ts deleted file mode 100644 index 8eda1e474f..0000000000 --- a/packages/backend/src/api/models/MoocletVersionConditionMap.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Entity, Column, JoinColumn, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'; -import { MoocletExperimentRef } from './MoocletExperimentRef'; -import { ExperimentCondition } from './ExperimentCondition'; - -@Entity() -export class MoocletVersionConditionMap { - @PrimaryGeneratedColumn() - public id?: number; - - @ManyToOne(() => MoocletExperimentRef, (moocletExperimentRef) => moocletExperimentRef.versionConditionMaps, { - onDelete: 'CASCADE', - }) - @JoinColumn({ name: 'moocletExperimentRefId' }) - moocletExperimentRef: MoocletExperimentRef; - - @Column() - moocletExperimentRefId?: string; - - @ManyToOne(() => ExperimentCondition, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'experimentConditionId' }) - experimentCondition: ExperimentCondition; - - @Column() - experimentConditionId?: string; - - @Column() - moocletVersionId?: number; -} diff --git a/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts new file mode 100644 index 0000000000..629994ec88 --- /dev/null +++ b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts @@ -0,0 +1,34 @@ +import { Entity, Column, OneToMany, PrimaryGeneratedColumn, JoinColumn, OneToOne } from 'typeorm'; +import { ConditionPosteriorState } from './ConditionPosteriorState'; +import { Experiment } from './Experiment'; +import { BaseModel } from './base/BaseModel'; + +@Entity() +export class ThompsonSamplingExperimentConfig extends BaseModel { + @PrimaryGeneratedColumn('uuid') + public id: string; + + @OneToOne(() => Experiment, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'experimentId' }) + experiment: Experiment; + + @Column({ nullable: true }) + experimentId?: string; + + /** Use uniform random selection until total enrollments exceed this count. */ + @Column({ nullable: true }) + warmupThreshold?: number; + + /** Fall back to uniform when the top two sampled draws differ by less than this value. */ + @Column({ nullable: true, type: 'float' }) + minimumDrawDifference?: number; + + /** Update posteriors every N reward events rather than on every reward. */ + @Column({ nullable: true }) + batchSize?: number; + + @OneToMany(() => ConditionPosteriorState, (state) => state.config, { + cascade: true, + }) + conditionPosteriorStates: ConditionPosteriorState[]; +} diff --git a/packages/backend/src/api/models/ThompsonSamplingReward.ts b/packages/backend/src/api/models/ThompsonSamplingReward.ts new file mode 100644 index 0000000000..46d75ad237 --- /dev/null +++ b/packages/backend/src/api/models/ThompsonSamplingReward.ts @@ -0,0 +1,31 @@ +import { Entity, Column, ManyToOne, PrimaryGeneratedColumn, JoinColumn, Index } from 'typeorm'; +import { Experiment } from './Experiment'; +import { ExperimentCondition } from './ExperimentCondition'; +import { BaseModel } from './base/BaseModel'; + +@Entity() +@Index(['experimentId', 'conditionId']) +export class ThompsonSamplingReward extends BaseModel { + @PrimaryGeneratedColumn('uuid') + public id: string; + + @ManyToOne(() => Experiment, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'experimentId' }) + experiment: Experiment; + + @Column() + experimentId: string; + + @ManyToOne(() => ExperimentCondition, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'conditionId' }) + condition: ExperimentCondition; + + @Column() + conditionId: string; + + @Column() + userId: string; + + @Column({ type: 'boolean' }) + success: boolean; +} diff --git a/packages/backend/src/api/repositories/ConditionPosteriorStateRepository.ts b/packages/backend/src/api/repositories/ConditionPosteriorStateRepository.ts new file mode 100644 index 0000000000..373a5f33b0 --- /dev/null +++ b/packages/backend/src/api/repositories/ConditionPosteriorStateRepository.ts @@ -0,0 +1,17 @@ +import { Repository } from 'typeorm'; +import { EntityRepository } from '../../typeorm-typedi-extensions'; +import { ConditionPosteriorState } from '../models/ConditionPosteriorState'; + +@EntityRepository(ConditionPosteriorState) +export class ConditionPosteriorStateRepository extends Repository { + public async findByConfigId(configId: string): Promise { + return this.createQueryBuilder('state') + .leftJoinAndSelect('state.condition', 'condition') + .where('state.configId = :configId', { configId }) + .getMany(); + } + + public async findByConditionId(conditionId: string): Promise { + return this.createQueryBuilder('state').where('state.conditionId = :conditionId', { conditionId }).getOne(); + } +} diff --git a/packages/backend/src/api/repositories/MoocletExperimentRefRepository.ts b/packages/backend/src/api/repositories/MoocletExperimentRefRepository.ts deleted file mode 100644 index 7f911162a0..0000000000 --- a/packages/backend/src/api/repositories/MoocletExperimentRefRepository.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Repository } from 'typeorm'; -import { EntityRepository } from '../../typeorm-typedi-extensions'; -import { MoocletExperimentRef } from '../models/MoocletExperimentRef'; -import { EXPERIMENT_STATE } from 'upgrade_types'; - -@EntityRepository(MoocletExperimentRef) -export class MoocletExperimentRefRepository extends Repository { - public async getRefsForActivelyEnrollingExperiments(): Promise { - return this.createQueryBuilder('moocletExperimentRef') - .leftJoinAndSelect('moocletExperimentRef.versionConditionMaps', 'versionConditionMaps') - .leftJoinAndSelect('moocletExperimentRef.experiment', 'experiment') - .where('experiment.state = :status', { status: EXPERIMENT_STATE.ENROLLING }) - .getMany(); - } - - public async findActivelyEnrollingMoocletExperimentsByContextSiteTarget( - context: string, - site: string, - target: string - ): Promise { - return this.createQueryBuilder('moocletExperimentRef') - .leftJoinAndSelect('moocletExperimentRef.versionConditionMaps', 'versionConditionMaps') - .leftJoinAndSelect('moocletExperimentRef.experiment', 'experiment') - .leftJoinAndSelect('experiment.partitions', 'decisionPoint') - .where('experiment.state = :status', { status: EXPERIMENT_STATE.ENROLLING }) - .andWhere(':context = ANY(experiment.context)', { context }) - .andWhere('decisionPoint.site = :site', { site }) - .andWhere('decisionPoint.target = :target', { target }) - .getMany(); - } -} diff --git a/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts new file mode 100644 index 0000000000..0faa50d21c --- /dev/null +++ b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts @@ -0,0 +1,38 @@ +import { Repository } from 'typeorm'; +import { EntityRepository } from '../../typeorm-typedi-extensions'; +import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; +import { EXPERIMENT_STATE } from 'upgrade_types'; + +@EntityRepository(ThompsonSamplingExperimentConfig) +export class ThompsonSamplingExperimentConfigRepository extends Repository { + public async findByExperimentId(experimentId: string): Promise { + return this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .where('config.experimentId = :experimentId', { experimentId }) + .getOne(); + } + + public async findByDecisionPoint( + context: string, + site: string, + target: string + ): Promise { + return this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .leftJoinAndSelect('config.experiment', 'experiment') + .leftJoinAndSelect('experiment.partitions', 'decisionPoint') + .where('experiment.state = :state', { state: EXPERIMENT_STATE.ENROLLING }) + .andWhere(':context = ANY(experiment.context)', { context }) + .andWhere('decisionPoint.site = :site', { site }) + .andWhere('decisionPoint.target = :target', { target }) + .getMany(); + } + + public async findConfigsForActivelyEnrollingExperiments(): Promise { + return this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .leftJoinAndSelect('config.experiment', 'experiment') + .where('experiment.state = :state', { state: EXPERIMENT_STATE.ENROLLING }) + .getMany(); + } +} diff --git a/packages/backend/src/api/repositories/ThompsonSamplingRewardRepository.ts b/packages/backend/src/api/repositories/ThompsonSamplingRewardRepository.ts new file mode 100644 index 0000000000..12dc2ea21e --- /dev/null +++ b/packages/backend/src/api/repositories/ThompsonSamplingRewardRepository.ts @@ -0,0 +1,17 @@ +import { Repository } from 'typeorm'; +import { EntityRepository } from '../../typeorm-typedi-extensions'; +import { ThompsonSamplingReward } from '../models/ThompsonSamplingReward'; + +@EntityRepository(ThompsonSamplingReward) +export class ThompsonSamplingRewardRepository extends Repository { + public async findByExperimentAndCondition( + experimentId: string, + conditionId: string + ): Promise { + return this.createQueryBuilder('reward') + .where('reward.experimentId = :experimentId', { experimentId }) + .andWhere('reward.conditionId = :conditionId', { conditionId }) + .orderBy('reward.createdAt', 'ASC') + .getMany(); + } +} diff --git a/packages/backend/src/api/services/ExperimentAssignmentService.ts b/packages/backend/src/api/services/ExperimentAssignmentService.ts index 3ebb824a92..105f99a915 100644 --- a/packages/backend/src/api/services/ExperimentAssignmentService.ts +++ b/packages/backend/src/api/services/ExperimentAssignmentService.ts @@ -69,12 +69,17 @@ import { UserStratificationFactorRepository } from '../repositories/UserStratifi import { UserStratificationFactor } from '../models/UserStratificationFactor'; import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; import { Brackets, In } from 'typeorm'; -import { env } from '../../env'; -import { MoocletExperimentService } from './MoocletExperimentService'; import { ExperimentPrecomputedSegmentService } from './ExperimentPrecomputedSegmentService'; import { ExperimentPrecomputedSegment } from '../models/ExperimentPrecomputedSegment'; import { precomputedGroupKey } from './precomputedSegmentHelpers'; import { EntitySegmentMembers, EntitySegmentResolutionInput, SegmentGroupMember } from '../../types'; +import { + ThompsonSamplingService, + ConditionPrior, + ThompsonSamplingConfig, + ConditionRewardSummary, +} from './ThompsonSamplingService'; +import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; export interface FactorialConditionResult { factorialCondition: Omit; @@ -117,6 +122,9 @@ export class ExperimentAssignmentService { @InjectRepository() private userStratificationFactorRepository: UserStratificationFactorRepository, + @InjectRepository() + private thompsonSamplingConfigRepository: ThompsonSamplingExperimentConfigRepository, + public previewUserService: PreviewUserService, public experimentUserService: ExperimentUserService, public errorService: ErrorService, @@ -124,8 +132,8 @@ export class ExperimentAssignmentService { public segmentService: SegmentService, public experimentService: ExperimentService, public cacheService: CacheService, - public moocletExperimentService: MoocletExperimentService, - public experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService + public experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService, + public thompsonSamplingService: ThompsonSamplingService ) {} /** @@ -1941,26 +1949,14 @@ export class ExperimentAssignmentService { }; await this.repeatedEnrollmentRepository.save(RepeatedEnrollmentDocument); } else { - let conditionAssigned: ExperimentCondition | void; - - const isMoocletExperiment = this.moocletExperimentService.isMoocletExperiment(experiment.assignmentAlgorithm); - - if (isMoocletExperiment) { - conditionAssigned = await this.moocletExperimentService.handleEnrollCondition( - experiment.id, - condition, - logger - ); - } else { - conditionAssigned = await this.assignExperiment( - user, - experiment, - individualEnrollment, - groupEnrollment, - individualExclusion, - groupExclusion - ); - } + const conditionAssigned = await this.assignExperiment( + user, + experiment, + individualEnrollment, + groupEnrollment, + individualExclusion, + groupExclusion + ); if (!individualEnrollment && !individualExclusion && conditionAssigned) { const individualEnrollmentDocument: Omit = { @@ -2068,34 +2064,65 @@ export class ExperimentAssignmentService { logger: UpgradeLogger, enrollmentCount?: { conditionId: string; userCount: number }[] ): Promise { - const isMoocletExperiment = this.moocletExperimentService.isMoocletExperiment(experiment.assignmentAlgorithm); - - if (isMoocletExperiment && !env.mooclets.enabled) { - logger.error({ - message: 'Mooclet experiment algorithm is indicated but mooclets are not enabled', - experiment, - user, - }); - return undefined; + if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return this.assignThompsonSampling(experiment, user, logger); } - if (isMoocletExperiment && env.mooclets.enabled) { - return this.getConditionFromMoocletProxy(experiment, user, logger); - } else { - return this.assignRandom(experiment, user, enrollmentCount); - } + return this.assignRandom(experiment, user, enrollmentCount); } - private async getConditionFromMoocletProxy(experiment: Experiment, user: ExperimentUser, logger: UpgradeLogger) { - const userId = user.id; - + private async assignThompsonSampling( + experiment: Experiment, + user: ExperimentUser, + logger: UpgradeLogger + ): Promise { try { - return await this.moocletExperimentService.getConditionFromMoocletProxy(experiment, userId, logger); + const config = await this.thompsonSamplingConfigRepository.findByExperimentId(experiment.id); + + if (!config) { + logger.error({ + message: 'Thompson Sampling config not found for experiment; no condition assigned', + experimentId: experiment.id, + userId: user.id, + }); + return undefined; + } + + // Use conditionId as the identifier throughout — conditionCode is nullable + const conditionIds = experiment.conditions.map((c) => c.id); + + const rewardSummaries: ConditionRewardSummary[] = config.conditionPosteriorStates.map((state) => ({ + conditionCode: state.conditionId, + successCount: state.successCount, + totalCount: state.totalCount, + })); + + const priors: Record = {}; + config.conditionPosteriorStates.forEach((state) => { + priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; + }); + + const totalEnrollments = config.conditionPosteriorStates.reduce((sum, s) => sum + s.totalCount, 0); + + const tsConfig: ThompsonSamplingConfig = { + priors, + warmupThreshold: config.warmupThreshold, + minimumDrawDifference: config.minimumDrawDifference, + }; + + const selectedConditionId = this.thompsonSamplingService.selectCondition( + conditionIds, + rewardSummaries, + totalEnrollments, + tsConfig + ); + + return experiment.conditions.find((c) => c.id === selectedConditionId); } catch (err) { logger.error({ - message: 'Error getting condition from Mooclet proxy; experiment will return no condition for this user', + message: 'Error in Thompson Sampling assignment; no condition assigned', experimentId: experiment.id, - userId, + userId: user.id, error: err, }); return undefined; diff --git a/packages/backend/src/api/services/ExperimentService.ts b/packages/backend/src/api/services/ExperimentService.ts index 33e72238a4..d3300fbc6e 100644 --- a/packages/backend/src/api/services/ExperimentService.ts +++ b/packages/backend/src/api/services/ExperimentService.ts @@ -97,7 +97,6 @@ import { plainToClass } from 'class-transformer'; import { StratificationFactorRepository } from '../repositories/StratificationFactorRepository'; import { ExperimentDetailsForCSVData } from '../repositories/AnalyticsRepository'; import { MetricService } from './MetricService'; -import { MoocletExperimentRefRepository } from '../repositories/MoocletExperimentRefRepository'; import { ExperimentAuditLog } from '../models/ExperimentAuditLog'; import { SegmentRepository } from '../repositories/SegmentRepository'; import { NotFoundException } from '@nestjs/common/exceptions'; @@ -135,7 +134,6 @@ export class ExperimentService { @InjectRepository() protected levelCombinationElementsRepository: LevelCombinationElementRepository, @InjectRepository() protected archivedStatsRepository: ArchivedStatsRepository, @InjectRepository() protected stratificationRepository: StratificationFactorRepository, - @InjectRepository() protected moocletExperimentRefRepository: MoocletExperimentRefRepository, @InjectDataSource() protected dataSource: DataSource, protected previewUserService: PreviewUserService, protected segmentService: SegmentService, @@ -393,7 +391,7 @@ export class ExperimentService { // Populate the experiment_precomputed_segment row from the just-attached lists. When an // existingEntityManager was provided the caller owns the (still-open) transaction, so it — not - // create — must recompute after commit (see MoocletExperimentService); recomputing here would + // create — must recompute after commit; recomputing here would // read uncommitted writes. recomputeForExperiment yields empty arrays when the experiment has no // lists, so this also seeds list-less experiments (no separate empty-seed needed). if (!existingEntityManager) { @@ -1268,7 +1266,7 @@ export class ExperimentService { // filterMode was forced to EXCLUDE_ALL, but the experiment_precomputed_segment row still holds the // old member IDs. Recompute it (to empty) after commit so the assignment read path never serves // stale inclusion/exclusion data. Mirrors FeatureFlagService.updateFeatureFlagInDB's withRecompute. - // When a caller owns the transaction (existingEntityManager, e.g. MoocletExperimentService), the + // When a caller owns the transaction (existingEntityManager), the // writes are not committed yet, so the caller recomputes after its own commit (see syncUpdate) — // the same deferral create() uses. if (isChangingContext && !existingEntityManager) { @@ -1583,14 +1581,6 @@ export class ExperimentService { const experimentJSONValidationError = await this.validateExperimentJSON(newExperiment); const fileName = experimentFile.fileName; - if ('moocletPolicyParameters' in newExperiment && !env.mooclets?.enabled) { - return { - fileName, - error: 'moocletPolicyParameters was provided but mooclets are not enabled on backend.', - compatibilityType: IMPORT_COMPATIBILITY_TYPE.INCOMPATIBLE, - }; - } - try { experiment = this.autoFillSomeMissingProperties(experiment); experiment = this.deduceExperimentDetails(experiment); @@ -2091,7 +2081,7 @@ export class ExperimentService { // The caller owns the outer transaction. We must NOT recompute here: recomputeForExperiment // reads through its own repositories and cannot see this transaction's uncommitted writes, so // it would persist a stale experiment_precomputed_segment row that never self-heals. The caller - // (create / importExperimentLists / MoocletExperimentService) recomputes after it commits. + // (create / importExperimentLists) recomputes after it commits. return await executeTransaction(transactionalEntityManager); } else { // withRecompute runs the mutation in its own transaction, then fires a fire-and-forget diff --git a/packages/backend/src/api/services/ImportExportService.ts b/packages/backend/src/api/services/ImportExportService.ts index 5d9079757e..1d077d93c4 100644 --- a/packages/backend/src/api/services/ImportExportService.ts +++ b/packages/backend/src/api/services/ImportExportService.ts @@ -3,9 +3,8 @@ import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; import { ExperimentService } from './ExperimentService'; import { ExperimentDTO, ExperimentFile } from '../DTO/ExperimentDTO'; import { env } from '../../env'; -import { MoocletExperimentService } from './MoocletExperimentService'; import { UserDTO } from '../DTO/UserDTO'; -import { LOG_TYPE, SUPPORTED_MOOCLET_ALGORITHMS } from 'upgrade_types'; +import { LOG_TYPE } from 'upgrade_types'; import { In } from 'typeorm'; import { InjectRepository } from '../../typeorm-typedi-extensions'; import { ExperimentRepository } from '../repositories/ExperimentRepository'; @@ -16,8 +15,7 @@ export class ImportExportService { constructor( @InjectRepository() protected experimentRepository: ExperimentRepository, @InjectRepository() protected experimentAuditLogRepository: ExperimentAuditLogRepository, - protected experimentService: ExperimentService, - protected moocletExperimentService: MoocletExperimentService + protected experimentService: ExperimentService ) {} public async importExperiments(experiments: ExperimentFile[], user: UserDTO, logger: UpgradeLogger) { @@ -41,19 +39,8 @@ export class ImportExportService { await Promise.all( experiments.map(async (experiment) => { try { - if (this.moocletExperimentService.isMoocletExperiment(experiment.assignmentAlgorithm)) { - if (!env.mooclets.enabled) { - throw new Error('Attempting to import a moclet experiment, but mooclets are not enabled'); - } - await this.moocletExperimentService.syncCreate({ - experimentDTO: experiment, - currentUser, - logger, - }); - } else { - const result = await this.experimentService.create(experiment, currentUser, logger); - createdExperiments.push(result); - } + const result = await this.experimentService.create(experiment, currentUser, logger); + createdExperiments.push(result); } catch (error) { logger.error({ message: 'Failed to create experiment during import', @@ -133,32 +120,10 @@ export class ImportExportService { return a.order - b.order; }); - let experimentRecord = this.experimentService.reducedConditionPayload( + const experimentRecord = this.experimentService.reducedConditionPayload( this.experimentService.formattingPayload(this.experimentService.formattingConditionPayload(experiment)) ); - // If it's a mooclet experiment, policy parameters - if (SUPPORTED_MOOCLET_ALGORITHMS.includes(experiment.assignmentAlgorithm)) { - try { - experimentRecord = await this.moocletExperimentService.attachPolicyParamsToExperimentDTO( - experimentRecord, - logger - ); - } catch (error) { - logger.error({ - message: 'Failed to get mooclet data for experiment', - error: error, - experiment: experiment, - user: user, - }); - throw error; - } - // remove currentPosteriors from moocletPolicyParameters for export - const { current_posteriors: _, ...filteredPolictParameters } = experimentRecord.moocletPolicyParameters; - - experimentRecord.moocletPolicyParameters = filteredPolictParameters; - } - this.experimentAuditLogRepository.saveRawJson( LOG_TYPE.EXPERIMENT_DESIGN_EXPORTED, { experimentId: experiment.id, experimentName: experiment.name }, diff --git a/packages/backend/src/api/services/MoocletDataService.ts b/packages/backend/src/api/services/MoocletDataService.ts deleted file mode 100644 index 5cf609a606..0000000000 --- a/packages/backend/src/api/services/MoocletDataService.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { Service } from 'typedi'; -import { env } from '../../env'; - -import axios, { AxiosRequestConfig } from 'axios'; -import { - MoocletBatchResponse, - MoocletPolicyResponseDetails, - MoocletProxyRequestParams, - MoocletRequestBody, - MoocletResponseDetails, - MoocletVersionRequestBody, - MoocletVersionResponseDetails, - MoocletPolicyParametersRequestBody, - MoocletPolicyParametersResponseDetails, - MoocletVariableRequestBody, - MoocletVariableResponseDetails, - MoocletValueRequestBody, - MoocletValueResponseDetails, - MoocletRewardCountRequestBody, - MoocletPaginatedResponse, -} from '../../types/Mooclet'; -import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; -import { MoocletError } from '../errors/MoocletError'; - -@Service() -export class MoocletDataService { - private apiUrl = env.mooclets.hostUrl + env.mooclets.apiRoute; - private apiToken = 'Token ' + env.mooclets.apiToken; - - /************************************************************************************************* - * EXTERNAL DATA FETCHING METHODS - */ - - public async getMoocletIdByName(policyName: string, logger: UpgradeLogger): Promise { - const response: MoocletBatchResponse = await this.getPoliciesList(logger); - let matchedPolicy: MoocletPolicyResponseDetails = null; - - if (response?.results.length) { - matchedPolicy = response.results.find((policy) => policy.name === policyName); - } - - if (matchedPolicy) { - return matchedPolicy.id; - } - - return null; - } - - public async getPoliciesList(logger: UpgradeLogger): Promise> { - const endpoint = '/policy'; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response: MoocletBatchResponse = await this.fetchExternalMoocletsData( - requestParams, - logger - ); - - return response; - } - - public async postNewMooclet(requestBody: MoocletRequestBody, logger: UpgradeLogger): Promise { - const endpoint = '/mooclet'; - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deleteMooclet(moocletId: number, logger: UpgradeLogger): Promise { - const endpoint = `/mooclet/${moocletId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewVersion( - requestBody: MoocletVersionRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = '/version'; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getVersion(versionId: number, logger: UpgradeLogger): Promise { - const endpoint = `/version/${versionId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async updateVersion( - versionId: number, - requestBody: MoocletVersionRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/version/${versionId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'PUT', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deleteVersion(versionId: number, logger: UpgradeLogger): Promise { - const endpoint = `/version/${versionId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewPolicyParameters( - requestBody: MoocletPolicyParametersRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = '/policyparameters'; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getPolicyParameters( - policyParametersId: number, - logger: UpgradeLogger - ): Promise { - const endpoint = `/policyparameters/${policyParametersId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async updatePolicyParameters( - policyParametersId: number, - policyParameters: MoocletPolicyParametersRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/policyparameters/${policyParametersId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'PUT', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: policyParameters, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deletePolicyParameters(policyParametersId: number, logger: UpgradeLogger): Promise { - const endpoint = `/policyparameters/${policyParametersId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewReward( - requestBody: MoocletValueRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/value?learner=${requestBody.learner}`; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getRewardsForExperiment( - requestBody: MoocletRewardCountRequestBody, - logger: UpgradeLogger, - nextPageUrl?: string - ): Promise> { - // this endpoint serves a paginated response - // if there are more results "pages" mooclet api sends the exact url to use for "next" page - // else it is nul/undefined and we'll fetch from the beginning - const url = - nextPageUrl || `${this.apiUrl}/value?mooclet=${requestBody.moocletId}&variable__name=${requestBody.variableName}`; - - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewVariable( - requestBody: MoocletVariableRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = '/variable'; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getVariable(variableId: number, logger: UpgradeLogger): Promise { - const endpoint = `/variable/${variableId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async updateVariable( - variableId: number, - requestBody: MoocletVariableRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/variable/${variableId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'PUT', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deleteVariable(variableId: number, logger: UpgradeLogger): Promise { - const endpoint = `/variable/${variableId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getVersionForNewLearner( - moocletId: number, - userId: string, - logger: UpgradeLogger - ): Promise { - const endpoint = `/mooclet/${moocletId}/run?learner=${userId}`; - - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - /** - * Generic Requests to Mooclets API - */ - - public async fetchExternalMoocletsData( - requestParams: MoocletProxyRequestParams, - logger: UpgradeLogger - ): Promise { - const { method, url, body } = requestParams; - if (method && url) { - const headers: HeadersInit = { - Authorization: this.apiToken, - 'Content-Type': 'application/json', - }; - - const JSONbody = JSON.stringify(body); - - logger.info({ message: `Fetching data from Mooclets API: ${url}`, body: JSONbody }); - - try { - const options: AxiosRequestConfig = { - method, - data: JSONbody, - headers, - url, - }; - - const res = await axios.request(options); - return res.data; - } catch (err) { - if (err instanceof MoocletError) { - throw err; - } - if (axios.isAxiosError(err)) { - logger.error({ - message: 'Error fetching data from Mooclets API', - url, - method, - status: err.response?.status, - responseBody: err.response?.data, - error: err, - }); - throw new MoocletError(`Mooclet server returned non-2xx status: ${err.response?.status}`); - } else { - logger.error({ - message: 'Error fetching data from Mooclets API', - url, - method, - error: err, - }); - throw new MoocletError('Failed to communicate with Mooclet server'); - } - } - } - } -} diff --git a/packages/backend/src/api/services/MoocletExperimentService.ts b/packages/backend/src/api/services/MoocletExperimentService.ts deleted file mode 100644 index 3133a51d3e..0000000000 --- a/packages/backend/src/api/services/MoocletExperimentService.ts +++ /dev/null @@ -1,1630 +0,0 @@ -import { Service } from 'typedi'; -import { MoocletDataService } from './MoocletDataService'; -import { - MoocletPolicyParametersRequestBody, - MoocletPolicyParametersResponseDetails, - MoocletRequestBody, - MoocletResponseDetails, - MoocletVariableRequestBody, - MoocletVariableResponseDetails, - MoocletVersionRequestBody, - MoocletVersionResponseDetails, -} from '../../types/Mooclet'; -import { ExperimentService } from './ExperimentService'; -import { ExperimentPrecomputedSegmentService } from './ExperimentPrecomputedSegmentService'; -import { MoocletError } from '../errors/MoocletError'; -import { ExperimentRepository } from '../repositories/ExperimentRepository'; -import { ExperimentConditionRepository } from '../repositories/ExperimentConditionRepository'; -import { InjectRepository, InjectDataSource } from '../../typeorm-typedi-extensions'; -import { DataSource, EntityManager } from 'typeorm'; -import { ArchivedStatsRepository } from '../repositories/ArchivedStatsRepository'; -import { ConditionPayloadRepository } from '../repositories/ConditionPayloadRepository'; -import { SegmentRepository } from '../repositories/SegmentRepository'; -import { DecisionPointRepository } from '../repositories/DecisionPointRepository'; -import { ExperimentAuditLogRepository } from '../repositories/ExperimentAuditLogRepository'; -import { ExperimentSegmentExclusionRepository } from '../repositories/ExperimentSegmentExclusionRepository'; -import { ExperimentSegmentInclusionRepository } from '../repositories/ExperimentSegmentInclusionRepository'; -import { ExperimentUserRepository } from '../repositories/ExperimentUserRepository'; -import { FactorRepository } from '../repositories/FactorRepository'; -import { GroupExclusionRepository } from '../repositories/GroupExclusionRepository'; -import { IndividualExclusionRepository } from '../repositories/IndividualExclusionRepository'; -import { LevelCombinationElementRepository } from '../repositories/LevelCombinationElements'; -import { LevelRepository } from '../repositories/LevelRepository'; -import { MetricRepository } from '../repositories/MetricRepository'; -import { MonitoredDecisionPointRepository } from '../repositories/MonitoredDecisionPointRepository'; -import { QueryRepository } from '../repositories/QueryRepository'; -import { StateTimeLogsRepository } from '../repositories/StateTimeLogsRepository'; -import { StratificationFactorRepository } from '../repositories/StratificationFactorRepository'; -import { CacheService } from './CacheService'; -import { ErrorService } from './ErrorService'; -import { PreviewUserService } from './PreviewUserService'; -import { QueryService } from './QueryService'; -import { SegmentService } from './SegmentService'; -import { MoocletExperimentRef } from '../models/MoocletExperimentRef'; -import { MoocletVersionConditionMap } from '../models/MoocletVersionConditionMap'; - -import { MoocletExperimentRefRepository } from '../repositories/MoocletExperimentRefRepository'; -import { ConditionValidator, ExperimentDTO } from '../DTO/ExperimentDTO'; -import { UserDTO } from '../DTO/UserDTO'; -import { Experiment } from '../models/Experiment'; -import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; -import { - ASSIGNMENT_ALGORITHM, - EXPERIMENT_STATE, - MoocletPolicyParametersDTO, - MoocletTSConfigurablePolicyParametersDTO, - SUPPORTED_MOOCLET_ALGORITHMS, -} from 'upgrade_types'; -import { ExperimentCondition } from '../models/ExperimentCondition'; -import { MetricService } from './MetricService'; -import { env } from '../../env'; -import { ExperimentSchedulerService } from './ExperimentSchedulerService'; - -export interface SyncCreateParams { - experimentDTO: ExperimentDTO; - currentUser: UserDTO; - createType?: string; - logger: UpgradeLogger; -} - -export interface SyncEditParams { - experimentDTO: ExperimentDTO; - currentUser: UserDTO; - moocletRefToDelete?: MoocletExperimentRef; - logger: UpgradeLogger; -} - -export interface SyncDeleteParams { - moocletExperimentRef: MoocletExperimentRef; - experimentId: string; - currentUser: UserDTO; - logger: UpgradeLogger; -} - -export interface AllowedInactiveStateChanges { - addedConditions: ConditionValidator[] | false; - removedConditions: MoocletVersionConditionMap[] | false; - modifiedConditions: MoocletVersionConditionMap[] | false; -} - -export interface EditRollbackRef { - revertPolicyParameters: MoocletPolicyParametersDTO; - restoreVersions: ExperimentCondition[]; - revertVersionModifications: MoocletVersionConditionMap[]; - removeVersions: MoocletVersionConditionMap[]; - currentMoocletExperimentRef: MoocletExperimentRef; - currentExperiment: Experiment; -} - -@Service() -export class MoocletExperimentService extends ExperimentService { - constructor( - private moocletDataService: MoocletDataService, - @InjectRepository() experimentRepository: ExperimentRepository, - @InjectRepository() experimentConditionRepository: ExperimentConditionRepository, - @InjectRepository() decisionPointRepository: DecisionPointRepository, - @InjectRepository() experimentAuditLogRepository: ExperimentAuditLogRepository, - @InjectRepository() individualExclusionRepository: IndividualExclusionRepository, - @InjectRepository() groupExclusionRepository: GroupExclusionRepository, - @InjectRepository() monitoredDecisionPointRepository: MonitoredDecisionPointRepository, - @InjectRepository() userRepository: ExperimentUserRepository, - @InjectRepository() metricRepository: MetricRepository, - @InjectRepository() queryRepository: QueryRepository, - @InjectRepository() stateTimeLogsRepository: StateTimeLogsRepository, - @InjectRepository() experimentSegmentInclusionRepository: ExperimentSegmentInclusionRepository, - @InjectRepository() experimentSegmentExclusionRepository: ExperimentSegmentExclusionRepository, - @InjectRepository() conditionPayloadRepository: ConditionPayloadRepository, - @InjectRepository() factorRepository: FactorRepository, - @InjectRepository() levelRepository: LevelRepository, - @InjectRepository() levelCombinationElementsRepository: LevelCombinationElementRepository, - @InjectRepository() archivedStatsRepository: ArchivedStatsRepository, - @InjectRepository() stratificationRepository: StratificationFactorRepository, - @InjectRepository() segmentRepository: SegmentRepository, - @InjectRepository() - moocletExperimentRefRepository: MoocletExperimentRefRepository, - @InjectDataSource() dataSource: DataSource, - previewUserService: PreviewUserService, - segmentService: SegmentService, - experimentSchedulerService: ExperimentSchedulerService, - errorService: ErrorService, - cacheService: CacheService, - queryService: QueryService, - metricService: MetricService, - experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService - ) { - super( - experimentRepository, - experimentConditionRepository, - decisionPointRepository, - experimentAuditLogRepository, - individualExclusionRepository, - groupExclusionRepository, - monitoredDecisionPointRepository, - userRepository, - metricRepository, - queryRepository, - stateTimeLogsRepository, - experimentSegmentInclusionRepository, - experimentSegmentExclusionRepository, - segmentRepository, - conditionPayloadRepository, - factorRepository, - levelRepository, - levelCombinationElementsRepository, - archivedStatsRepository, - stratificationRepository, - moocletExperimentRefRepository, - dataSource, - previewUserService, - segmentService, - experimentSchedulerService, - errorService, - cacheService, - queryService, - metricService, - experimentPrecomputedSegmentService - ); - } - - public async syncCreate(params: SyncCreateParams): Promise { - const experiment = await this.dataSource.transaction(async (manager) => { - const experimentResponse = await this.createUpgradeExperiment(manager, params); - params.experimentDTO = experimentResponse; - return this.handleCreateMoocletTransaction(manager, params); - }); - - // create() ran inside the transaction above (via existingEntityManager), so it deferred the - // precomputed recompute to us. Now that the transaction has committed, populate the - // experiment_precomputed_segment row (awaited, mirroring create()). - await this.experimentPrecomputedSegmentService.recomputeForExperiment(experiment.id, params.logger); - return experiment; - } - - public async syncUpdate(params: SyncEditParams): Promise { - const experiment = await this.dataSource.transaction((manager) => - this.handleEditMoocletTransaction(manager, params) - ); - - // super.update() ran inside the transaction above (caller-owned entityManager), so it deferred the - // precomputed recompute to us. Recompute after commit so a context change — which deletes all - // segment lists — can't leave a stale experiment_precomputed_segment row. Fire-and-forget. - this.experimentPrecomputedSegmentService.scheduleRecomputeForExperiments([experiment.id], params.logger); - return experiment; - } - - public async syncUpdateWithMoocletAlgorithmTransition(params: SyncEditParams): Promise { - const experiment = await this.dataSource.transaction(async (manager) => { - const updatedExperiment = await this.updateUpgradeExperiment(manager, params); - - // when transitioning away from Mooclet, delete the old refs after successful experiment update - if (params.moocletRefToDelete) { - await this.moocletExperimentRefRepository.delete(params.moocletRefToDelete.id); - } - - // when transitioning to Mooclet, create the Mooclet resources - if (this.isMoocletExperiment(updatedExperiment.assignmentAlgorithm)) { - params.experimentDTO = { ...updatedExperiment }; - return await this.handleCreateMoocletTransaction(manager, params); - } - - return updatedExperiment; - }); - - // Same deferral as syncUpdate: super.update() ran inside the transaction, so recompute after commit. - this.experimentPrecomputedSegmentService.scheduleRecomputeForExperiments([experiment.id], params.logger); - return experiment; - } - - public async syncDelete(params: SyncDeleteParams): Promise { - return this.dataSource.transaction((manager) => this.handleDeleteMoocletTransaction(manager, params)); - } - - /** - * handleCreateMoocletTransaction - * - * 1. Save the upgrade experiment - * 2. Create and save the Mooclet experiment resources (outputs MoocletExperimentRef) - * 3. Save the MoocletExperimentRef and VersionConditionMaps - * - * On any error, rollback the Mooclet resources and abort the transaction - */ - - private async handleCreateMoocletTransaction( - manager: EntityManager, - params: SyncCreateParams - ): Promise { - const logger = params.logger; - const { moocletPolicyParameters } = params.experimentDTO; - const experiment = params.experimentDTO; - - // Auto-generate outcome variable name for ts_configurable experiments (always generate new) - if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { - const generatedName = this.generateUniqueOutcomeVariableName(experiment.name); - (moocletPolicyParameters as MoocletTSConfigurablePolicyParametersDTO).outcome_variable_name = generatedName; - logger.info({ - message: 'Generated outcome variable name for new TS Configurable Mooclet experiment', - experimentId: experiment.id, - generatedName, - }); - } - - // create Mooclet resources. If this fails, it will internally rollback any mooclet resources created, and the UpGrade experiment transaction will abort - const moocletExperimentRefResponse = await this.orchestrateMoocletCreation( - experiment, - moocletPolicyParameters, - logger - ); - - logger.info({ - message: 'Mooclet experiment created successfully:', - moocletExperimentRef: JSON.stringify(moocletExperimentRefResponse), - }); - - // additionally, create MoocletExperimentRef and VersionConditionMaps. - // If either of THESE fail, we will need to rollback the mooclet resources here also before aborting the UpGrade experiment transaction - try { - await this.saveMoocletExperimentRef(manager, moocletExperimentRefResponse); - await this.createAndSaveVersionConditionMapEntities( - manager, - moocletExperimentRefResponse.id, - moocletExperimentRefResponse.versionConditionMaps, - logger - ); - - experiment.moocletPolicyParameters = moocletPolicyParameters; - - return experiment; - } catch (error) { - await this.orchestrateDeleteMoocletResources(moocletExperimentRefResponse, logger); - throw error; - } - } - - private async createUpgradeExperiment(manager: EntityManager, params: SyncCreateParams): Promise { - const { experimentDTO, currentUser, createType, logger } = params; - return await super.create(experimentDTO, currentUser, logger, { - existingEntityManager: manager, - createType, - }); - } - - private async updateUpgradeExperiment(manager: EntityManager, params: SyncEditParams): Promise { - const { experimentDTO, currentUser, logger } = params; - return await super.update(experimentDTO, currentUser, logger, manager); - } - - private async saveMoocletExperimentRef( - manager: EntityManager, - moocletExperimentRefResponse: MoocletExperimentRef - ): Promise { - await manager.save(MoocletExperimentRef, moocletExperimentRefResponse); - } - - private async createAndSaveVersionConditionMapEntities( - manager: EntityManager, - moocletExperimentRefId: string, - versionConditionMaps: MoocletVersionConditionMap[], - logger: UpgradeLogger - ): Promise { - for (const versionConditionMap of versionConditionMaps) { - const versionConditionMapEntity = manager.create(MoocletVersionConditionMap, { - ...versionConditionMap, - moocletExperimentRefId, - experimentConditionId: versionConditionMap.experimentConditionId, - }); - const map = await manager.save(MoocletVersionConditionMap, versionConditionMapEntity); - logger.info({ - message: 'MoocletVersionConditionMap created successfully:', - versionConditionMap: JSON.stringify(map), - }); - } - } - - /** - * Handles a transaction to edit Mooclet experiment resources with rollback capability. - * - * The function performs the following sequential operations: - * 1. Validates experiment state eligibility (INACTIVE, PREVIEW, SCHEDULED, or RUNNING) - * 2. Fetches current resources for comparison with incoming changes - * 3. Detects experiment design changes that affect Mooclet resources and validates if they're allowed in current state - * 4. Updates the base experiment via parent class - * 5. Updates policy parameters (allowed in all eligible states) - * 6. If version edits exist, applies them with rollback capability - * - * If any step fails, the transaction will abort and we will automatically attempt to roll back any Mooclet resources to previous state - * - * @param manager - Entity manager passed for transaction handling - * @param params - Object containing experimentDTO, currentUser, and logger - * @returns Promise resolving to updated ExperimentDTO - * @throws Error if experiment state is ineligible or if disallowed changes are detected - */ - private async handleEditMoocletTransaction(manager: EntityManager, params: SyncEditParams): Promise { - const { experimentDTO: incomingExperiment, currentUser, logger } = params; - const rollbackRef: EditRollbackRef = { - revertPolicyParameters: null, - restoreVersions: null, - revertVersionModifications: null, - removeVersions: null, - currentMoocletExperimentRef: null, - currentExperiment: null, - }; - - // bail if the experiment is in not in an eligible edit state for mooclet resources - if ( - ![ - EXPERIMENT_STATE.INACTIVE, - EXPERIMENT_STATE.PREVIEW, - EXPERIMENT_STATE.SCHEDULED, - EXPERIMENT_STATE.RUNNING, - EXPERIMENT_STATE.PAUSED, - ].includes(incomingExperiment.state) - ) { - logger.error({ - message: '[Mooclet Edit] Ineligible experiment state for Mooclet edits', - state: incomingExperiment.state, - }); - throw new MoocletError('[Mooclet Edit] Ineligible experiment state for Mooclet edits', 400); - } - - // NOTE: Currently allowed states for updating mooclet resources: - // conditions (versions): PREVIEW, SCHEDULED, INACTIVE, PAUSED - // outcome variable name: IMMUTABLE after creation (no changes allowed) - // policy parameters: RUNNING, PREVIEW, SCHEDULED, INACTIVE, PAUSED - - try { - // ---------- fetch current resources for comparison to incoming ------------------------------- - const { currentMoocletExperimentRef, currentPolicyParametersResponse, currentExperiment } = - await this.fetchCurrentResources(incomingExperiment, logger); - - const versionEdits = this.detectExperimentDesignChanges(incomingExperiment, currentMoocletExperimentRef); - - // bail if disallowed changes are detected for running experiments - if (versionEdits && EXPERIMENT_STATE.RUNNING === incomingExperiment.state) { - logger.error({ - message: '[Mooclet Edit] Ineligible version edits detected for an active Mooclet experiment', - changes: versionEdits, - }); - throw new MoocletError( - '[Mooclet Edit] Ineligible version edits detected for an active Mooclet experiment', - 400 - ); - } - - logger.debug({ - message: '[Mooclet Edit] Experiment changes affect Mooclet resources, begin sync.', - changes: versionEdits, - }); - - // ---------- update the experiment first --------------------------------------- - const updatedExperiment = await super.update(incomingExperiment, currentUser, logger, manager); - - rollbackRef.currentExperiment = currentExperiment; - rollbackRef.currentMoocletExperimentRef = currentMoocletExperimentRef; - - // ---------- update policy parameters ------------------------------------------ - // NOTE: the rollbackRef is mutated within these methods and will be used to revert changes if any step fails - - // go ahead and PUT policy parameters (these are allowed when running and not really worth checking for changes) - const policyParameterResponse = await this.doRevertablePolicyParameterChange({ - incomingExperiment, - currentMoocletExperimentRef, - currentPolicyParametersResponse, - rollbackRef, - logger, - }); - - updatedExperiment.moocletPolicyParameters = policyParameterResponse.parameters; - - // Transform prior keys from Mooclet version IDs back to UpGrade condition codes, - // so the PUT response is consistent with the GET response from attachPolicyParamsToExperimentDTO. - const updatedTsParams = updatedExperiment.moocletPolicyParameters as MoocletTSConfigurablePolicyParametersDTO; - if (updatedTsParams?.prior) { - updatedTsParams.prior = this.translateVersionIdsToConditionCodes( - updatedTsParams.prior, - currentMoocletExperimentRef.versionConditionMaps - ); - } - - // --------- update versions ---------------------- - - if (!versionEdits) { - return updatedExperiment; - } - - await this.doRevertableVersionEdits({ - versionEdits, - currentMoocletExperimentRef, - incomingExperiment, - rollbackRef, - manager, - logger, - }); - - return updatedExperiment; - } catch (error) { - logger.error({ message: '[Mooclet Edits] Error updating experiment', error }); - await this.rollbackMoocletEdits(rollbackRef, logger); - throw error; - } - } - - /** - * Identifies conditions that exist in the incoming experiment but not in the current Mooclet experiment. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Array of new conditions if any are found, or null if no new conditions - */ - private detectNewConditions( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): ConditionValidator[] | null { - const newConditions = incomingExperimentDTO.conditions.filter( - (condition) => - !currentMoocletExperimentRef.versionConditionMaps.find((map) => map.experimentCondition.id === condition.id) - ); - - return newConditions.length ? newConditions : null; - } - - /** - * Identifies conditions that exist in the current Mooclet experiment but not in the incoming experiment. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Array of version-condition maps for removed conditions if any are found, or null if no conditions were removed - */ - private detectRemovedConditions( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): MoocletVersionConditionMap[] | null { - const mapsForRemovedConditions = currentMoocletExperimentRef.versionConditionMaps.filter( - (map) => !incomingExperimentDTO.conditions.find((condition) => condition.id === map.experimentCondition.id) - ); - - return mapsForRemovedConditions.length ? mapsForRemovedConditions : null; - } - - /** - * Identifies conditions whose code has been modified between the incoming experiment and the current Mooclet experiment. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Array of version-condition maps for modified conditions if any are found, or null if no conditions were modified - */ - private detectModifiedConditions( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): MoocletVersionConditionMap[] | null { - const versionsToUpdate = currentMoocletExperimentRef.versionConditionMaps.filter((map) => { - const condition = incomingExperimentDTO.conditions.find( - (condition) => condition.id === map.experimentCondition.id - ); - - if (!condition) { - return null; - } - - return condition && condition.conditionCode !== map.experimentCondition.conditionCode; - }); - - return versionsToUpdate.length ? versionsToUpdate : null; - } - - /** - * Aggregates all experiment design changes by calling individual detection methods. - * - * Checks for added conditions, removed conditions, and modified conditions. - * Returns an object containing all detected changes, or null if no changes were detected. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Object with all detected changes if any exist, or null if no changes were detected - */ - private detectExperimentDesignChanges( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): AllowedInactiveStateChanges | null { - const changes = { - addedConditions: this.detectNewConditions(incomingExperimentDTO, currentMoocletExperimentRef), - removedConditions: this.detectRemovedConditions(incomingExperimentDTO, currentMoocletExperimentRef), - modifiedConditions: this.detectModifiedConditions(incomingExperimentDTO, currentMoocletExperimentRef), - }; - - const hasChanges = Object.values(changes).some((change) => !!change); - - return hasChanges ? changes : null; - } - - /** - * Updates policy parameters for a Mooclet experiment. - * - * This method updates the policy parameters of a Mooclet experiment in response to - * experiment design changes. It forwards the request to the - * Mooclet data service. - * - * @param newPolicyParameters - The updated policy parameters to apply - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param logger - Logger instance for recording operation details - * @returns Promise resolving to the updated policy parameters response from Mooclet - */ - private async handleUpdatePolicyParameters( - newPolicyParameters: MoocletPolicyParametersDTO, - currentMoocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - const tsParams = newPolicyParameters as MoocletTSConfigurablePolicyParametersDTO; - if (tsParams.prior) { - // Translate conditionCode keys to Mooclet version IDs before sending to the Mooclet API - newPolicyParameters = { - ...tsParams, - prior: this.translateConditionCodesToVersionIds( - tsParams.prior, - currentMoocletExperimentRef.versionConditionMaps - ), - } as MoocletPolicyParametersDTO; - } - - return this.moocletDataService.updatePolicyParameters( - currentMoocletExperimentRef.policyParametersId, - { - mooclet: currentMoocletExperimentRef.moocletId, - policy: currentMoocletExperimentRef.policyId, - parameters: newPolicyParameters, - }, - logger - ); - } - - /** - * Removes conditions from a Mooclet experiment. - * - * This method deletes versions from Mooclet that correspond to experiment conditions - * that have been removed from the experiment design. It - * processes each removal in parallel. - * - * @param removedConditions - Array of version-condition maps for conditions to remove - * @param logger - Logger instance for recording operation details - * @returns Promise resolving to an array of void results, one for each deleted version - */ - private async handleRemoveConditions( - removedConditions: MoocletVersionConditionMap[], - logger: UpgradeLogger - ): Promise { - return Promise.all( - removedConditions.map(async (map) => { - return await this.moocletDataService.deleteVersion(map.moocletVersionId, logger); - }) - ); - } - - /** - * Modifies conditions in a Mooclet experiment. - * - * This method updates versions in Mooclet to reflect changes to their corresponding - * experiment conditions. It retrieves each version, updates its name and text fields - * if they differ from the incoming condition, and updates the version in Mooclet. - * - * @param versionMapsToUpdate - Array of version-condition maps for conditions to modify - * @param incomingExperiment - The updated experiment data containing modified conditions - * @param logger - Logger instance for recording operation details - * @returns Promise resolving to the array of updated version-condition maps - */ - private async handleModifyConditions( - versionMapsToUpdate: MoocletVersionConditionMap[], - incomingExperiment: ExperimentDTO | Experiment, - logger: UpgradeLogger - ): Promise { - return Promise.all( - versionMapsToUpdate.map(async (versionMap) => { - const version = await this.moocletDataService.getVersion(versionMap.moocletVersionId, logger); - const condition = incomingExperiment.conditions.find( - (changedCondition) => changedCondition.id === versionMap.experimentConditionId - ); - - if (version && condition && version.name !== condition.conditionCode) { - version.name = condition.conditionCode; - version.text = condition.conditionCode; // this could be mapped to payload - await this.moocletDataService.updateVersion(versionMap.moocletVersionId, version, logger); - } - - return versionMap; - }) - ); - } - - /** - * Adds new conditions to a Mooclet experiment. - * - * This method creates new versions in Mooclet for conditions that have been added to - * the experiment design. It creates each version in parallel, and - * then creates mappings between the new versions and their corresponding conditions. - * - * @param addedConditions - Array of conditions to add to the Mooclet experiment - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @returns Promise resolving to an array of newly created version-condition maps - */ - private async handleAddConditions( - addedConditions: ExperimentCondition[] | ConditionValidator[], - currentMoocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - const newVersions: MoocletVersionResponseDetails[] = []; - - await Promise.all( - addedConditions.map(async (condition) => { - const newVersionRequest: MoocletVersionRequestBody = { - mooclet: currentMoocletExperimentRef.moocletId, - name: condition.conditionCode, - text: condition.conditionCode, - }; - - const version = await this.moocletDataService.postNewVersion(newVersionRequest, logger); - newVersions.push(version); - }) - ); - - return this.createMoocletVersionConditionMaps(newVersions, addedConditions); - } - - /** - * Updates policy parameters with rollback capability. - * - * @param options - Object containing: - * @param incomingExperiment - The updated experiment data containing new policy parameters - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param currentPolicyParametersResponse - Current policy parameters for rollback - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves to the updated policy parameters response - */ - async doRevertablePolicyParameterChange({ - incomingExperiment, - currentMoocletExperimentRef, - currentPolicyParametersResponse, - rollbackRef, - logger, - }) { - logger.debug({ - message: '[Mooclet Edit] Upserting policy parameters due to experiment design change', - incomingPolicyParameters: incomingExperiment.moocletPolicyParameters, - }); - const policyParametersResponse = await this.handleUpdatePolicyParameters( - incomingExperiment.moocletPolicyParameters, - currentMoocletExperimentRef, - logger - ); - - rollbackRef.revertPolicyParameters = currentPolicyParametersResponse.parameters; - return policyParametersResponse; - } - - /** - * Executes revertable edits to versions within a Mooclet experiment. - * Handles condition modifications, additions, and removals - * with built-in rollback capability if any step fails. - * - * @param options - Object containing: - * @param versionEdits - Changes to make to versions - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param incomingExperiment - New experiment configuration - * @param manager - Entity manager for database operations - * @param logger - Logger instance - * @param rollbackRef - Reference object to store rollback information - * @returns Promise that resolves when all edits are complete - * @throws Will throw the first encountered error if any task fails - */ - async doRevertableVersionEdits({ - versionEdits, - currentMoocletExperimentRef, - incomingExperiment, - manager, - logger, - rollbackRef, - }) { - const { addedConditions, removedConditions, modifiedConditions } = versionEdits; - - const taskPromises = []; - - if (removedConditions) { - taskPromises.push(this.doRevertableRemovedConditions({ removedConditions, rollbackRef, logger })); - } - - if (modifiedConditions) { - taskPromises.push( - this.doRevertableModifiedConditions({ - modifiedConditions, - incomingExperiment, - experimentRefId: currentMoocletExperimentRef.id, - manager, - rollbackRef, - logger, - }) - ); - } - - if (addedConditions) { - taskPromises.push( - this.doRevertableAddConditions({ addedConditions, currentMoocletExperimentRef, manager, rollbackRef, logger }) - ); - } - - // Use Promise.allSettled to let all tasks complete or fail independently - const results = await Promise.allSettled(taskPromises); - - // Check if any tasks failed - const failures = results.filter((result): result is PromiseRejectedResult => result.status === 'rejected'); - - if (failures.length > 0) { - // Log all the errors that occurred - failures.forEach((failure, index) => { - logger.error({ - message: `[Mooclet Edit] Task ${index} failed during version edits.`, - error: failure.reason, - }); - }); - - // Throw the first error to signal failure to the caller - // The rollbackRef will already be populated with all successful operations - throw failures[0].reason; - } - } - - /** - * Removes conditions from a Mooclet experiment with rollback capability. - * - * @param options - Object containing: - * @param removedConditions - Conditions to be removed - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves when condition removal is complete - */ - async doRevertableRemovedConditions({ removedConditions, rollbackRef, logger }) { - logger.debug({ - message: '[Mooclet Edit] Removing versions from Mooclet due to experiment design change.', - removedConditions, - }); - - await this.handleRemoveConditions(removedConditions, logger); - rollbackRef.restoreVersions = removedConditions.map((map: MoocletVersionConditionMap) => map.experimentCondition); - } - - /** - * Modifies conditions in a Mooclet experiment with rollback capability. - * - * @param options - Object containing: - * @param modifiedConditions - Conditions to be modified - * @param incomingExperiment - New experiment configuration - * @param experimentRefId - ID of the experiment reference - * @param manager - Entity manager for database operations - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves when condition modifications are complete - */ - async doRevertableModifiedConditions({ - modifiedConditions, - incomingExperiment, - experimentRefId, - manager, - rollbackRef, - logger, - }) { - logger.debug({ - message: '[Mooclet Edit] Modifying versions from Mooclet due to experiment design change.', - modifiedConditions, - }); - - const modifiedVersionConditionMaps = await this.handleModifyConditions( - modifiedConditions, - incomingExperiment, - logger - ); - - await this.createAndSaveVersionConditionMapEntities(manager, experimentRefId, modifiedVersionConditionMaps, logger); - - rollbackRef.revertVersionModifications = modifiedConditions; - } - - /** - * Adds conditions to a Mooclet experiment with rollback capability. - * - * @param options - Object containing: - * @param addedConditions - Conditions to be added - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param manager - Entity manager for database operations - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves when condition additions are complete - */ - async doRevertableAddConditions({ addedConditions, currentMoocletExperimentRef, manager, rollbackRef, logger }) { - logger.debug({ - message: '[Mooclet Edit] Adding conditions to Mooclet due to experiment design change.', - addedConditions, - }); - - const addedVersionConditionMaps = await this.handleAddConditions( - addedConditions, - currentMoocletExperimentRef, - logger - ); - - await this.createAndSaveVersionConditionMapEntities( - manager, - currentMoocletExperimentRef.id, - addedVersionConditionMaps, - logger - ); - - rollbackRef.removeVersions = addedVersionConditionMaps; - } - - /** - * Fetches all current resources related to a Mooclet experiment. - * - * @param incomingExperiment - The experiment configuration to retrieve resources for - * @param logger - Logger instance for operation logging - * @returns Promise that resolves to an object containing the current Mooclet experiment reference, - * policy parameters, and current experiment - */ - async fetchCurrentResources(incomingExperiment: ExperimentDTO, logger: UpgradeLogger) { - const currentMoocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(incomingExperiment.id); - - const [currentPolicyParametersResponse, currentExperiment] = await Promise.all([ - this.moocletDataService.getPolicyParameters(currentMoocletExperimentRef.policyParametersId, logger), - this.experimentRepository.findOne({ - where: { id: incomingExperiment.id }, - relations: { - conditions: true, - }, - }), - ]); - - return { - currentMoocletExperimentRef, - currentPolicyParametersResponse, - currentExperiment, - }; - } - - /** - * Rolls back changes made to Mooclet experiment resources in case of transaction failure. - * - * This method attempts to restore the Mooclet experiment to its previous state when - * an error occurs during the edit process. It systematically reverses each type of - * change that may have been made: - * - * 1. Reverts policy parameters to their original values - * 2. Recreates versions that were deleted - * 3. Reverts modifications to existing versions - * 4. Removes versions that were newly added - * - * If an error occurs during the rollback process itself, a critical error is logged - * and thrown, indicating that the experiment resources may be in an inconsistent state. - * - * @param rollbackRef - Reference object containing all the data needed for rollback operations - * @param logger - Logger instance for recording rollback operation details - * @returns Promise that resolves when rollback is complete - * @throws Error if the rollback itself fails, indicating resources are likely out of sync - */ - private async rollbackMoocletEdits(rollbackRef: EditRollbackRef, logger: UpgradeLogger): Promise { - logger.error({ - message: '[Mooclet Edit] Rolling back Mooclet edits', - rollbackRef, - }); - - const { - revertPolicyParameters, - restoreVersions, - revertVersionModifications, - removeVersions, - currentMoocletExperimentRef, - currentExperiment, - } = rollbackRef; - - try { - if (revertPolicyParameters) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to reverting policy parameters to previous state', - revertPolicyParameters, - currentMoocletExperimentRef, - }); - await this.handleUpdatePolicyParameters(revertPolicyParameters, currentMoocletExperimentRef, logger); - } - - if (restoreVersions) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to restore deleted versions to previous state', - restoreVersions, - }); - await this.handleAddConditions(restoreVersions, currentMoocletExperimentRef, logger); - } - - if (revertVersionModifications) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to revert modified versions to previous state', - revertVersionModifications, - }); - await this.handleModifyConditions(revertVersionModifications, currentExperiment, logger); - } - - if (removeVersions) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to remove added versions to previous state', - removeVersions, - }); - await this.handleRemoveConditions(removeVersions, logger); - } - } catch (rollbackError) { - logger.error({ - message: - '[Mooclet Edit] Error during rollback, Mooclet resources are likely out of sync for this experiment! Check the resources noted in the rollback ref', - error: rollbackError, - rollbackRef, - }); - throw new MoocletError( - '[Mooclet Edit] Error during rollback, Mooclet resources are likely out of sync for this experiment' - ); - } - } - - private async handleDeleteMoocletTransaction(manager: EntityManager, params: SyncDeleteParams): Promise { - const { moocletExperimentRef, currentUser, logger } = params; - let deleteResponse: Experiment | undefined; - - try { - // delete the upgrade experiment. If this fails, the Mooclet resources will not be deleted, and the transaction will abort - deleteResponse = await super.delete(params.experimentId, currentUser, { - existingEntityManager: manager, - }); - - // delete the mooclet resources. If this fails, the transaction will abort and the upgrade experiment will not be deleted, - // but the Mooclet resources may not be deleted either - const removedResources = await this.orchestrateDeleteMoocletResources(moocletExperimentRef, logger); - if (!removedResources) { - logger.error({ - message: 'Failed to delete Mooclet resources, aborting transaction to preserve upgrade experiment', - experimentId: params.experimentId, - moocletExperimentRef, - }); - throw new MoocletError( - 'Failed to delete Mooclet resources, aborting transaction to preserve upgrade experiment' - ); - } - - return deleteResponse; - } catch (error) { - logger.error({ - message: 'Failed to delete experiment with Mooclet', - error: error, - experimentId: params.experimentId, - user: currentUser, - }); - throw error; - } - } - - async handleRemoveAnyMoocletResources(experimentId: string, logger: UpgradeLogger): Promise { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experimentId); - if (moocletExperimentRef) { - this.orchestrateDeleteMoocletResources(moocletExperimentRef, logger); - } else { - logger.info({ - message: `No MoocletExperimentRef found for experiment ${experimentId}, skipping Mooclet resource deletion.`, - }); - } - } - - /** - * orchestrateMoocletCreation: Sequential calls needed to create of a new Mooclet experiment from experiment data and supplied policy parameters - * - * 1. Retrieves the Mooclet policy ID based on the assignment algorithm. - * 2. Creates a new Mooclet experiment with name an policy ID. - * 3. Creates Mooclet versions for each Experiment condition - * 4. Creates Policy Parameters - * 5. Creates a variable if needed based on the assignment algorithm and policy parameters. - * 6. Constructs and returns the Mooclet experiment reference if all steps were successful. - */ - - public async orchestrateMoocletCreation( - upgradeExperiment: ExperimentDTO, - moocletPolicyParameters: MoocletPolicyParametersDTO, - logger: UpgradeLogger - ): Promise { - const newMoocletRequest: MoocletRequestBody = { - name: upgradeExperiment.name + '-' + upgradeExperiment.id.slice(0, 8), //using part of exp uuid so this will make sure mooclet name is unique - policy: null, - }; - const moocletExperimentRef = new MoocletExperimentRef(); - moocletExperimentRef.experimentId = upgradeExperiment.id; - - logger.debug({ - message: '[Mooclet Creation] 0. Starting Mooclet creation process', - upgradeExperiment: JSON.stringify(upgradeExperiment), - }); - - try { - newMoocletRequest.policy = await this.getMoocletPolicy(upgradeExperiment.assignmentAlgorithm, logger); - logger.debug({ - message: `[Mooclet Creation] 1. Policy id fetched:`, - newMoocletRequest: JSON.stringify(newMoocletRequest), - }); - - const moocletResponse = await this.createMooclet(newMoocletRequest, logger); - moocletExperimentRef.moocletId = moocletResponse.id; - - logger.debug({ - message: `[Mooclet Creation] 2. Mooclet created:`, - moocletResponse: JSON.stringify(moocletResponse), - }); - - const moocletVersionsResponse = await this.createMoocletVersions(upgradeExperiment, moocletResponse, logger); - moocletExperimentRef.versionConditionMaps = this.createMoocletVersionConditionMaps( - moocletVersionsResponse, - upgradeExperiment.conditions - ); - logger.debug({ - message: `[Mooclet Creation] 3. Mooclet versions created:`, - moocletVersionsResponse: JSON.stringify(moocletVersionsResponse), - }); - - const moocletPolicyParametersResponse = await this.createPolicyParameters( - moocletResponse, - moocletPolicyParameters, - logger - ); - moocletExperimentRef.policyParametersId = moocletPolicyParametersResponse.id; - logger.debug({ - message: `[Mooclet Creation] 4. Policy parameters created:`, - moocletPolicyParametersResponse: JSON.stringify(moocletPolicyParametersResponse), - }); - - const moocletVariableResponse = await this.createVariableIfNeeded( - moocletPolicyParameters, - upgradeExperiment.assignmentAlgorithm, - logger - ); - logger.debug({ - message: `[Mooclet Creation] 5. Variable created (if needed):`, - moocletVariableResponse: JSON.stringify(moocletVariableResponse), - }); - - moocletExperimentRef.variableId = moocletVariableResponse?.id; - moocletExperimentRef.policyId = newMoocletRequest.policy; - moocletExperimentRef.outcomeVariableName = ( - moocletPolicyParameters as MoocletTSConfigurablePolicyParametersDTO - ).outcome_variable_name; - } catch (err) { - await this.orchestrateDeleteMoocletResources(moocletExperimentRef, logger); - throw err; - } - - moocletExperimentRef.id = crypto.randomUUID(); - - return moocletExperimentRef; - } - - private createMoocletVersionConditionMaps( - versions: MoocletVersionResponseDetails[], - conditions: ConditionValidator[] | ExperimentCondition[] - ): MoocletVersionConditionMap[] { - const versionConditionMaps: MoocletVersionConditionMap[] = conditions.map((condition) => { - const versionConditionMap = new MoocletVersionConditionMap(); - versionConditionMap.moocletVersionId = versions.find((version) => version.name === condition.conditionCode)?.id; - versionConditionMap.experimentConditionId = condition.id; - - return versionConditionMap; - }); - return versionConditionMaps; - } - - public async orchestrateDeleteMoocletResources( - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - try { - if (!moocletExperimentRef) { - logger.error({ message: 'MoocletExperimentRef not defined for deletion' }); - throw new MoocletError('MoocletExperimentRef not defined'); - } - - logger.debug({ message: '[Mooclet Deletion]: Starting deletion of Mooclet resources', moocletExperimentRef }); - - // Delete Mooclet resources if they exist - logger.debug({ message: '[Mooclet Deletion]: Deleting Mooclet', moocletId: moocletExperimentRef.moocletId }); - if (moocletExperimentRef.moocletId) { - await this.moocletDataService.deleteMooclet(moocletExperimentRef.moocletId, logger); - logger.debug({ message: '[Mooclet Deletion]: Deleted Mooclet', moocletId: moocletExperimentRef.moocletId }); - } - - await this.deleteMoocletVersions(moocletExperimentRef, logger); - - logger.debug({ - message: '[Mooclet Deletion]: Deleting policy parameters', - policyParametersId: moocletExperimentRef.policyParametersId, - }); - if (moocletExperimentRef.policyParametersId) { - await this.moocletDataService.deletePolicyParameters(moocletExperimentRef.policyParametersId, logger); - logger.debug({ - message: '[Mooclet Deletion]: Deleted policy parameters', - policyParametersId: moocletExperimentRef.policyParametersId, - }); - } - - logger.debug({ message: '[Mooclet Deletion]: Deleting variable', variableId: moocletExperimentRef.variableId }); - if (moocletExperimentRef.variableId) { - await this.moocletDataService.deleteVariable(moocletExperimentRef.variableId, logger); - logger.debug({ message: '[Mooclet Deletion]: Deleted variable', variableId: moocletExperimentRef.variableId }); - } - - logger.info({ message: '[Mooclet Deletion]: Completed deletion of Mooclet resources', moocletExperimentRef }); - return true; // Return true to indicate successful deletion of all resources - } catch (err) { - const error = { - message: - '[Mooclet Deletion]: Failed to delete Mooclet resources, please check manually for out of sync resources', - error: err, - moocletExperimentRef: moocletExperimentRef, - }; - - logger.error(error); - return false; // Return false to indicate failure, but do not throw an error to allow the transaction to complete - } - } - - private async deleteMoocletVersions( - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - if (moocletExperimentRef.versionConditionMaps) { - for (const versionConditionMap of moocletExperimentRef.versionConditionMaps) { - logger.debug({ - message: '[Mooclet Deletion]: Deleting Mooclet version', - moocletVersionId: versionConditionMap.moocletVersionId, - }); - if (versionConditionMap.moocletVersionId) { - await this.moocletDataService.deleteVersion(versionConditionMap.moocletVersionId, logger); - logger.debug({ - message: '[Mooclet Deletion]: Deleted Mooclet version', - moocletVersionId: versionConditionMap.moocletVersionId, - }); - } - } - } - } - - private async getMoocletPolicy(assignmentAlgorithm: string, logger: UpgradeLogger): Promise { - try { - return await this.moocletDataService.getMoocletIdByName(assignmentAlgorithm, logger); - } catch (err) { - logger.error({ message: 'Failed to get Mooclet policy', error: err, assignmentAlgorithm }); - throw new MoocletError('Failed to get Mooclet policy'); - } - } - - public async attachPolicyParamsToExperimentDTO( - experiment: ExperimentDTO, - logger: UpgradeLogger - ): Promise { - try { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experiment.id); - const policyParameters = await this.moocletDataService.getPolicyParameters( - moocletExperimentRef.policyParametersId, - logger - ); - - // Transform current_posteriors and prior keys from Mooclet version IDs to UpGrade condition codes - const tsConfigurableParams = policyParameters.parameters as MoocletTSConfigurablePolicyParametersDTO; - if (tsConfigurableParams.current_posteriors) { - tsConfigurableParams.current_posteriors = this.translateVersionIdsToConditionCodes( - tsConfigurableParams.current_posteriors, - moocletExperimentRef.versionConditionMaps - ); - } - - if (tsConfigurableParams.prior) { - tsConfigurableParams.prior = this.translateVersionIdsToConditionCodes( - tsConfigurableParams.prior, - moocletExperimentRef.versionConditionMaps - ); - } - - experiment.moocletPolicyParameters = policyParameters.parameters; - - return experiment; - } catch (err) { - logger.error({ message: 'Failed to get Mooclet policy parameters', error: err, experimentId: experiment.id }); - throw new MoocletError('Failed to get Mooclet policy parameters'); - } - } - - private async createMooclet( - newMoocletRequest: MoocletRequestBody, - logger: UpgradeLogger - ): Promise { - try { - return await this.moocletDataService.postNewMooclet(newMoocletRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet', error: err, newMoocletRequest }); - throw new MoocletError('Failed to create Mooclet'); - } - } - - private async createMoocletVersions( - experiment: ExperimentDTO, - moocletResponse: MoocletResponseDetails, - logger: UpgradeLogger - ): Promise { - if (!moocletResponse?.id || !experiment.conditions) return null; - - try { - return await Promise.all( - experiment.conditions.map(async (condition) => this.createNewVersion(condition, moocletResponse.id, logger)) - ); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet versions', error: err, experimentId: experiment.id }); - throw new MoocletError('Failed to create Mooclet versions'); - } - } - - private async createNewVersion( - upgradeCondition: ConditionValidator | ExperimentCondition, - moocletId: number, - logger: UpgradeLogger - ): Promise { - const newVersionRequest: MoocletVersionRequestBody = { - mooclet: moocletId, - name: upgradeCondition.conditionCode, - text: upgradeCondition.conditionCode, - }; - - try { - return await this.moocletDataService.postNewVersion(newVersionRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create new version for Mooclet', error: err, newVersionRequest }); - throw new MoocletError('Failed to create new version for Mooclet'); - } - } - - private async createPolicyParameters( - moocletResponse: MoocletResponseDetails, - moocletPolicyParameters: MoocletPolicyParametersDTO, - logger: UpgradeLogger - ): Promise { - if (!moocletResponse) return null; - - const policyParametersRequest: MoocletPolicyParametersRequestBody = { - mooclet: moocletResponse.id, - policy: moocletResponse.policy, - parameters: moocletPolicyParameters, - }; - - try { - return await this.moocletDataService.postNewPolicyParameters(policyParametersRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet policy parameters', error: err, policyParametersRequest }); - throw new MoocletError('Failed to create Mooclet policy parameters'); - } - } - - private async createVariableIfNeeded( - moocletPolicyParametersResponse: MoocletPolicyParametersDTO, - assignmentAlgorithm: string, - logger: UpgradeLogger - ): Promise { - if (!moocletPolicyParametersResponse || assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { - return null; - } - - const variableRequest: MoocletVariableRequestBody = { - name: (moocletPolicyParametersResponse as MoocletTSConfigurablePolicyParametersDTO)?.outcome_variable_name, - }; - - try { - return await this.moocletDataService.postNewVariable(variableRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet variable', error: err, variableRequest }); - throw new MoocletError('Failed to create variable'); - } - } - - public async getMoocletExperimentRefByUpgradeExperimentId( - upgradeExperimentId: string - ): Promise { - const moocletExperimentRef = await this.moocletExperimentRefRepository.findOne({ - where: { experimentId: upgradeExperimentId }, - relations: { - versionConditionMaps: { - experimentCondition: true, - }, - }, - }); - return moocletExperimentRef; - } - - public async getConditionFromMoocletProxy( - experiment: Experiment, - userId: string, - logger: UpgradeLogger - ): Promise { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experiment.id); - - if (!moocletExperimentRef) { - throw new MoocletError(`MoocletExperimentRef not found for experiment id ${experiment.id}`); - } - - const versionResponse = await this.moocletDataService.getVersionForNewLearner( - moocletExperimentRef.moocletId, - userId, - logger - ); - const experimentCondition = this.mapMoocletVersionToUpgradeCondition(versionResponse, moocletExperimentRef, logger); - return experimentCondition; - } - - private mapMoocletVersionToUpgradeCondition( - versionResponse: MoocletVersionResponseDetails, - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): ExperimentCondition { - // Find the corresponding versionConditionMap - const versionConditionMap = moocletExperimentRef.versionConditionMaps.find( - (map) => map.moocletVersionId === versionResponse.id - ); - - if (!versionConditionMap) { - logger.error({ - message: 'Version ID not found in version condition maps', - version: versionResponse, - versionConditionMaps: moocletExperimentRef.versionConditionMaps, - }); - throw new MoocletError('Version ID not found in version condition maps'); - } - - // Get the experiment condition from the versionConditionMap - const experimentCondition = versionConditionMap.experimentCondition; - - if (!experimentCondition) { - logger.error({ - message: 'Experiment condition not found in version condition map', - version: versionResponse, - versionConditionMap, - }); - throw new MoocletError('Experiment condition not found in version condition map'); - } - - return experimentCondition; - } - - public isMoocletExperiment(assignmentAlgorithm: ASSIGNMENT_ALGORITHM): boolean { - return SUPPORTED_MOOCLET_ALGORITHMS.includes(assignmentAlgorithm); - } - - private translateConditionCodesToVersionIds( - record: Record, - versionConditionMaps: MoocletVersionConditionMap[] - ): Record { - const result: Record = {}; - for (const [conditionCode, value] of Object.entries(record)) { - const map = versionConditionMaps?.find((m) => m.experimentCondition?.conditionCode === conditionCode); - if (!map?.moocletVersionId) { - throw new MoocletError(`No version mapping found for condition code ${conditionCode}`); - } - if (map?.moocletVersionId) { - result[String(map.moocletVersionId)] = value; - } - } - return result; - } - - private translateVersionIdsToConditionCodes( - record: Record, - versionConditionMaps: MoocletVersionConditionMap[] - ): Record { - const result: Record = {}; - for (const [versionId, value] of Object.entries(record)) { - const map = versionConditionMaps?.find((m) => String(m.moocletVersionId) === versionId); - if (!map?.experimentCondition?.conditionCode) { - throw new MoocletError(`No condition mapping found for Mooclet version ${versionId}`); - } - result[map.experimentCondition.conditionCode] = value; - } - return result; - } - - /** - * Generate a unique outcome variable name based on experiment name and timestamp. - * Returns a string in this format: - * "[first 10 chars of sanitized experiment name]_[ISO_timestamp]_REWARD_VARIABLE" - * - * This method mirrors the frontend logic but ensures outcome variable names - * are generated server-side and remain immutable after creation. - */ - private generateUniqueOutcomeVariableName(experimentName: string): string { - // first 10 chars of experiment name, sanitized - const baseName = experimentName - .toLowerCase() - .replace(/[^a-z0-9]+/g, '_') - .replace(/^_+|_+$/g, '') - .substring(0, 10); - - // append ISO date timestamp suffix to ensure uniqueness - const timestamp = new Date().toISOString(); - return `${baseName}_${timestamp}_REWARD_VARIABLE`; - } - - public async checkForMoocletAssignmentAlgorithmChange( - experiment: ExperimentDTO, - logger: UpgradeLogger - ): Promise<{ - hasChanged: boolean; - wasMooclet: boolean; - isNowMooclet: boolean; - oldAlgorithm: ASSIGNMENT_ALGORITHM; - }> { - logger.info({ message: `Check for assignment algorithm change for experiment id => ${experiment.id}` }); - const oldExperiment = await this.findOne(experiment.id, logger); - - if (!oldExperiment) { - logger.error({ message: 'Experiment unexpectedly not found', experimentId: experiment.id }); - throw new MoocletError(`Experiment unexpectedly not found for id ${experiment.id}`); - } - - const hasChanged = oldExperiment.assignmentAlgorithm !== experiment.assignmentAlgorithm; - const wasMooclet = this.isMoocletExperiment(oldExperiment.assignmentAlgorithm); - const isNowMooclet = this.isMoocletExperiment(experiment.assignmentAlgorithm); - return { - hasChanged, - wasMooclet, - isNowMooclet, - oldAlgorithm: oldExperiment.assignmentAlgorithm, - }; - } - - public async handlePotentialMoocletAssignmentAlgorithmChange( - experiment: ExperimentDTO, - currentUser: UserDTO, - logger: UpgradeLogger - ): Promise { - const { hasChanged, wasMooclet, isNowMooclet, oldAlgorithm } = await this.checkForMoocletAssignmentAlgorithmChange( - experiment, - logger - ); - let moocletRefToDelete: MoocletExperimentRef | undefined; - - if (hasChanged) { - if (experiment.state !== EXPERIMENT_STATE.INACTIVE) { - logger.error({ - message: 'Cannot edit assignment algorithm for non-INACTIVE experiment', - experimentId: experiment.id, - state: experiment.state, - }); - throw new MoocletError( - `Experiment state is ${experiment.state}. Can only edit assignment algorithm for experiments yet to begin (INACTIVE state)`, - 400 - ); - } - - // 1. Cache old mooclet ref BEFORE update, but don't delete stuff from mooclet db yet until after successful update in upgrade db - if (wasMooclet) { - moocletRefToDelete = await this.getMoocletExperimentRefByUpgradeExperimentId(experiment.id); - logger.debug({ - message: '[Algorithm Change] Fetched mooclet resources for deletion after update', - moocletRefId: moocletRefToDelete?.id, - experimentId: experiment.id, - }); - } - - // 2. Update experiment with new assignment algorithm, creating new mooclet resources when "isNowMooclet" - let message = `[Algorithm Change] Assignment algorithm changed from ${oldAlgorithm} to ${experiment.assignmentAlgorithm}.`; - if (wasMooclet) message += ' Will delete old Mooclet resources after successful update.'; - if (isNowMooclet) message += ' Will create new Mooclet resources as part of update.'; - - logger.info({ - message, - experimentId: experiment.id, - newAlgorithm: experiment.assignmentAlgorithm, - moocletRefToDelete, - }); - const updatedExperiment = await this.syncUpdateWithMoocletAlgorithmTransition({ - experimentDTO: experiment, - currentUser, - logger, - moocletRefToDelete, - }); - - // 3. AFTER successful update, delete old mooclet resources when "wasMooclet" - if (wasMooclet && moocletRefToDelete) { - try { - logger.info({ - message: '[Algorithm Change] Deleting old mooclet resources after successful update', - moocletRefId: moocletRefToDelete.id, - experimentId: experiment.id, - }); - await this.orchestrateDeleteMoocletResources(moocletRefToDelete, logger); - logger.info({ - message: '[Algorithm Change] Successfully deleted old mooclet resources', - experimentId: experiment.id, - }); - } catch (error) { - // Log error but don't throw - the experiment update was successful - // Orphaned mooclet resources are less critical than rolling back the experiment - logger.error({ - message: - '[Algorithm Change] Failed to delete old mooclet resources after algorithm change. Resources may be orphaned.', - error, - moocletRefToDelete: { - id: moocletRefToDelete.id, - moocletId: moocletRefToDelete.moocletId, - experimentId: moocletRefToDelete.experimentId, - }, - experimentId: experiment.id, - }); - } - } - - return updatedExperiment; - } - - // Handle regular mooclet experiment updates (no algorithm change) - if (!hasChanged && isNowMooclet) { - return await this.syncUpdate({ - experimentDTO: experiment, - currentUser, - logger, - }); - } - - // No mooclet involvement, return null to let controller handle via normal path - return Promise.resolve(null); - } - - public async handleEnrollCondition( - experimentId: string, - condition: string, - logger: UpgradeLogger - ): Promise { - if (!env.mooclets.enabled) { - logger.error({ - message: 'Mooclet experiment algorithm is indicated but mooclets are not enabled', - }); - return undefined; - } - - // Note: Unlike regular experiments where we infer the condition the user "should" have gotten, - // we have to enroll the condition the client has marked, because the Mooclet assignment won't persist on a user basis. - // This means that outside factors can potentially influence the intended balance. - - try { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experimentId); - - if (!moocletExperimentRef) { - logger.error({ - message: '[Mooclet Mark Condition] No MoocletExperimentRef found for experiment', - experimentId, - }); - throw new MoocletError( - `[Mooclet Mark Condition] No MoocletExperimentRef found for experiment id ${experimentId}` - ); - } - const versionConditionMap = moocletExperimentRef.versionConditionMaps.find( - (expCondition) => expCondition.experimentCondition.conditionCode === condition - ); - - if (!versionConditionMap) { - logger.error({ - message: '[Mooclet Mark Condition] No version found for condition', - condition, - moocletExperimentRef, - }); - throw new MoocletError(`[Mooclet Mark Condition] No version found for condition ${condition}`); - } - return versionConditionMap.experimentCondition; - } catch (err) { - logger.error({ - message: '[Mooclet Mark Condition] There was an error processing marked condition.', - err, - }); - throw err; - } - } -} diff --git a/packages/backend/src/api/services/MoocletRewardsService.ts b/packages/backend/src/api/services/MoocletRewardsService.ts deleted file mode 100644 index cdde84d90a..0000000000 --- a/packages/backend/src/api/services/MoocletRewardsService.ts +++ /dev/null @@ -1,353 +0,0 @@ -import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; -import { - EXPERIMENT_STATE, - SERVER_ERROR, - BinaryRewardValueMap, - MoocletTSConfigurablePolicyParametersDTO, - Prior, -} from 'upgrade_types'; -import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; -import { MoocletExperimentRef } from '../models/MoocletExperimentRef'; -import { MoocletDataService } from './MoocletDataService'; -import { MoocletExperimentRefRepository } from '../repositories/MoocletExperimentRefRepository'; -import { IndividualEnrollment } from '../models/IndividualEnrollment'; -import { IndividualEnrollmentRepository } from '../repositories/IndividualEnrollmentRepository'; -import { Service } from 'typedi'; -import { InjectRepository } from '../../typeorm-typedi-extensions'; -import { HttpError } from 'routing-controllers'; -import { - MoocletPaginatedResponse, - MoocletRewardCountRequestBody, - MoocletValueRequestBody, - MoocletValueResponseDetails, -} from '../../types/Mooclet'; -import { RewardValidator } from '../controllers/validators/RewardValidator'; -import { ExperimentRewardsByCondition, ExperimentRewardsSummary } from 'upgrade_types'; -import { MoocletExperimentService } from './MoocletExperimentService'; - -export interface IRewardResponse { - message: string; - request: RewardValidator; - reward: MoocletValueRequestBody; -} - -@Service() -export class MoocletRewardsService { - constructor( - @InjectRepository() - private moocletExperimentRefRepository: MoocletExperimentRefRepository, - @InjectRepository() - private individualEnrollmentRepository: IndividualEnrollmentRepository, - private moocletDataService: MoocletDataService, - private moocletExperimentService: MoocletExperimentService - ) {} - - /** - * Attempt to send a reward to the external mooclet API. - * This is intended to be a "fire-and-forget" operation; we do not wait for - * confirmation that the reward was received successfully. - * - * Several criteria must be met to validate that a reward can be sent: - * - * 1. Mooclets feature must be currently enabled - * - * 2. The unique experiment must be ascertained before sending a reward. - * `experimentId` is preferred, but if not provided, `context`, `site`, and `target` can - * be used to identify the experiment. - * - * 3. A complete synced Mooclet experiment reference must exist. - * - * 4. The user must have marked and have a unique enrollment. - * - * 5. The condition enrolled must match a `version` in the mooclet experiment ref. - * - * - If any of these criteria are not met, a 409 data-conflict error is thrown. - */ - public async sendReward( - user: RequestedExperimentUser, - request: RewardValidator, - logger: UpgradeLogger - ): Promise { - const { experimentId, context, decisionPoint, rewardValue } = request; - - try { - // Find the mooclet experiment ref by ID or decision point - const moocletExperimentRef = experimentId - ? await this.findMoocletExperimentRefById(experimentId, request, logger) - : await this.findMoocletExperimentRefByDecisionPoint(context, decisionPoint, request, logger); - - // Find user's enrollment - const enrollments = await this.individualEnrollmentRepository.findEnrollments(user.id, [ - moocletExperimentRef.experimentId, - ]); - - if (!enrollments.length || enrollments.length > 1) { - this.throwConflictError( - `Could not find unique user enrollment for experiment (userId: ${user.id}, enrollments: ${enrollments}), no reward sent.`, - request, - logger - ); - } - - // Get version ID for the user's condition - const enrollment = enrollments[0]; - const versionId = this.getVersionIdByConditionId(enrollment, moocletExperimentRef, request, logger); - - if (!versionId) { - this.throwConflictError( - `Could not find version id for user enrollment (userId: ${user.id}, experimentId: ${moocletExperimentRef.experimentId}, conditionId: ${enrollment.conditionId}).`, - request, - logger - ); - } - - // Prepare and send reward - const reward: MoocletValueRequestBody = { - variable: moocletExperimentRef.outcomeVariableName, - value: BinaryRewardValueMap[rewardValue], - mooclet: moocletExperimentRef.moocletId, - version: versionId, - learner: user.id, - }; - - logger.info({ message: 'Sending reward to mooclet', reward, user }); - - // Fire-and-forget operation - // NOTE: in the future we may want to batch these, the mooclet API technically supports it by adding "/create_many" to an endpoint but it's not documented - this.moocletDataService.postNewReward(reward, logger); - - return { message: `Reward sent to mooclet successfully.`, request, reward }; - } catch (error) { - if (error instanceof HttpError) { - throw error; - } - - // Log and wrap unexpected errors - this.throwConflictError( - `Failed to process reward request due to unexpected error (userId: ${user.id}, experimentId: ${ - experimentId || 'not provided' - }, rewardValue: ${rewardValue}).`, - request, - logger - ); - } - } - - /** - * Finds mooclet experiment ref by experiment ID - */ - private async findMoocletExperimentRefById( - experimentId: string, - request: RewardValidator, - logger: UpgradeLogger - ): Promise { - const moocletExperimentRef = await this.moocletExperimentRefRepository.findOne({ - where: { experimentId }, - relations: { - versionConditionMaps: { - experimentCondition: true, - }, - - experiment: true, - }, - }); - - if (!moocletExperimentRef) { - this.throwConflictError( - `No active mooclet experiment ref found for experiment id: ${experimentId}, could not send reward.`, - request, - logger - ); - } - - if (moocletExperimentRef.experiment.state !== EXPERIMENT_STATE.ENROLLING) { - this.throwConflictError( - `Experiment with id: ${experimentId} is not actively enrolling (current state: ${moocletExperimentRef.experiment.state}), could not send reward.`, - request, - logger - ); - } - - return moocletExperimentRef; - } - - /** - * Finds mooclet experiment ref by decision point - */ - private async findMoocletExperimentRefByDecisionPoint( - context: string, - decisionPoint: { site: string; target: string }, - request: RewardValidator, - logger: UpgradeLogger - ): Promise { - const { site, target } = decisionPoint; - const moocletExperimentRefs = - await this.moocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget( - context, - site, - target - ); - - if (moocletExperimentRefs.length === 0) { - this.throwConflictError( - `No active experiment found for decision point (context: ${context}, site: ${site}, target: ${target}), could not send reward.`, - request, - logger - ); - } - - // TODO: this is a spot where we want to use shared-decision-point pooling logic, - // but for now if there are competing experiments, it will be required to use experimentId - if (moocletExperimentRefs.length > 1) { - this.throwConflictError( - `Multiple active experiments found for decision point (context: ${context}, site: ${site}, target: ${target}), cannot determine which to send reward to.`, - request, - logger - ); - } - - return moocletExperimentRefs[0]; - } - - private getVersionIdByConditionId( - enrollment: IndividualEnrollment, - moocletExperimentRef: MoocletExperimentRef, - request: RewardValidator, - logger: UpgradeLogger - ): number | null { - const map = moocletExperimentRef.versionConditionMaps.find( - (map) => enrollment.conditionId === map.experimentConditionId - ); - if (!map) { - this.throwConflictError(`Version-condition mapping not found, no reward sent.`, request, logger); - } - return map.moocletVersionId; - } - - public async getRewardsSummaryForExperiment( - experimentId: string, - logger: UpgradeLogger - ): Promise { - try { - const moocletExperimentRef = await this.moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId( - experimentId - ); - const rewards: MoocletValueResponseDetails[] = []; - logger.info({ - message: `Fetching Rewards data from mooclet server.`, - experimentId, - }); - let response = await this.fetchRewardsForExperiment(moocletExperimentRef, logger); - if (Array.isArray(response.results)) { - rewards.push(...response.results); - } - - while (response.next) { - logger.info({ - message: `But wait there's more (Fetching more Rewards data from Mooclet server for experiment...)`, - totalFound: response.count, - totalFetched: response.results.length, - next: response.next, - }); - response = await this.fetchRewardsForExperiment(moocletExperimentRef, logger, response.next); - if (Array.isArray(response.results)) { - rewards.push(...response.results); - } - } - - let tsConfigurableParams: MoocletTSConfigurablePolicyParametersDTO | undefined; - if (moocletExperimentRef.policyParametersId) { - try { - const policyParametersResponse = await this.moocletDataService.getPolicyParameters( - moocletExperimentRef.policyParametersId, - logger - ); - tsConfigurableParams = policyParametersResponse.parameters as MoocletTSConfigurablePolicyParametersDTO; - } catch (policyError) { - logger.warn({ - message: 'Could not fetch policy parameters for Thompson sampling estimate', - experimentId, - error: policyError, - }); - } - } - - return this.createExperimentRewardsSummary(moocletExperimentRef, rewards, logger, tsConfigurableParams); - } catch (error) { - logger.error({ message: 'Error fetching rewards summary for experiment', experimentId, error }); - throw error; - } - } - - public async fetchRewardsForExperiment( - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger, - nextPageUrl?: string - ): Promise> { - const requestBody: MoocletRewardCountRequestBody = { - moocletId: moocletExperimentRef.moocletId, - variableName: moocletExperimentRef.outcomeVariableName, - }; - - return await this.moocletDataService.getRewardsForExperiment(requestBody, logger, nextPageUrl); - } - - public async createExperimentRewardsSummary( - moocletExperimentRef: MoocletExperimentRef, - rewardsData: MoocletValueResponseDetails[], - logger: UpgradeLogger, - policyParameters?: MoocletTSConfigurablePolicyParametersDTO - ): Promise { - const rewards: MoocletValueResponseDetails[] = rewardsData; - - if (!rewardsData) { - logger.warn({ - message: 'No rewards data returned from Mooclet API', - experimentId: moocletExperimentRef.experimentId, - }); - return []; - } - - const DEFAULT_PRIOR: Prior = { success: 1, failure: 1 }; - - const rewardsSummaries = moocletExperimentRef.versionConditionMaps.map( - ({ experimentCondition, moocletVersionId }) => { - const conditionCode = experimentCondition.conditionCode; - const versionIdKey = String(moocletVersionId); - const versionRewards = rewards.filter((reward) => reward.version === moocletVersionId); - const successes = versionRewards.filter((reward) => reward.value === 1.0).length; - const failures = versionRewards.filter((reward) => reward.value === 0.0).length; - const total = successes + failures; - const percentSuccess = total > 0 ? (successes / total) * 100 : 0.0; - const successRate = percentSuccess.toFixed(1) + '%'; - - const conditionPrior: Prior = policyParameters?.prior?.[versionIdKey] ?? DEFAULT_PRIOR; - - const rewardsForCondition: ExperimentRewardsByCondition = { - conditionCode, - successes, - failures, - successRate, - order: experimentCondition.order, - priorSuccess: conditionPrior.success, - priorFailure: conditionPrior.failure, - }; - return rewardsForCondition; - } - ); - - const orderedRewardsSummary = rewardsSummaries.sort((a, b) => a.order - b.order); - return orderedRewardsSummary; - } - - /** - * Throws a 409 data-conflict error for most unexpected cases - */ - private throwConflictError(message: string, request: RewardValidator, logger: UpgradeLogger): never { - logger.error({ message, request }); - - const error = new HttpError(409, message); - (error as any).type = SERVER_ERROR.MOOCLET_REWARD_ERROR; - throw error; - } -} diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts new file mode 100644 index 0000000000..ef4a4db03f --- /dev/null +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -0,0 +1,142 @@ +import { Service } from 'typedi'; +import { InjectRepository } from '../../typeorm-typedi-extensions'; +import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; +import { ConditionPosteriorStateRepository } from '../repositories/ConditionPosteriorStateRepository'; +import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; +import { ThompsonSamplingService } from './ThompsonSamplingService'; +import { ExperimentRewardsSummary } from 'upgrade_types'; + +type ConditionRef = { id: string }; + +export interface ThompsonSamplingConfigParams { + warmupThreshold?: number; + minimumDrawDifference?: number; + batchSize?: number; + /** Beta priors per condition, keyed by conditionId. Defaults to Beta(1,1) for missing entries. */ + priors?: Record; +} + +@Service() +export class ThompsonSamplingExperimentCrudService { + constructor( + @InjectRepository() private configRepository: ThompsonSamplingExperimentConfigRepository, + @InjectRepository() private posteriorStateRepository: ConditionPosteriorStateRepository, + private thompsonSamplingService: ThompsonSamplingService + ) {} + + public async getConfigForExperiment(experimentId: string): Promise { + return this.configRepository.findByExperimentId(experimentId); + } + + public async createConfig( + experimentId: string, + conditions: ConditionRef[], + params: ThompsonSamplingConfigParams = {} + ): Promise { + const config = await this.configRepository.save({ + experimentId, + warmupThreshold: params.warmupThreshold ?? null, + minimumDrawDifference: params.minimumDrawDifference ?? null, + batchSize: params.batchSize ?? null, + }); + + await Promise.all( + conditions.map((condition) => + this.posteriorStateRepository.save({ + configId: config.id, + conditionId: condition.id, + priorSuccess: params.priors?.[condition.id]?.success ?? 1, + priorFailure: params.priors?.[condition.id]?.failure ?? 1, + successCount: 0, + totalCount: 0, + }) + ) + ); + + return config; + } + + public async updateConfig(experimentId: string, params: ThompsonSamplingConfigParams): Promise { + await this.configRepository.update( + { experimentId }, + { + warmupThreshold: params.warmupThreshold ?? null, + minimumDrawDifference: params.minimumDrawDifference ?? null, + batchSize: params.batchSize ?? null, + } + ); + } + + /** + * Keeps ConditionPosteriorState rows in sync with the experiment's current conditions. + * Adds rows for new conditions (using default priors) and removes rows for deleted conditions. + */ + public async getRewardsSummary(experimentId: string): Promise { + const config = await this.configRepository + .createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'states') + .leftJoinAndSelect('states.condition', 'condition') + .where('config.experimentId = :experimentId', { experimentId }) + .getOne(); + + if (!config) return []; + + const rows = config.conditionPosteriorStates.map((state) => { + const successes = state.successCount; + const failures = state.totalCount - state.successCount; + const successRate = state.totalCount > 0 ? ((successes / state.totalCount) * 100).toFixed(1) + '%' : '0.0%'; + const alpha = state.priorSuccess + state.successCount; + const beta = state.priorFailure + (state.totalCount - state.successCount); + return { + code: state.condition?.conditionCode ?? state.conditionId, + alpha, + beta, + conditionCode: state.condition?.conditionCode ?? state.conditionId, + successes, + failures, + successRate, + order: state.condition?.order ?? 0, + priorSuccess: state.priorSuccess, + priorFailure: state.priorFailure, + }; + }); + + const weightMap = this.thompsonSamplingService.estimateConditionWeights( + rows.map((r) => ({ code: r.code, alpha: r.alpha, beta: r.beta })) + ); + + return rows + .map(({ code: _code, alpha: _alpha, beta: _beta, ...rest }) => ({ + ...rest, + estimatedWeight: weightMap[rest.conditionCode], + })) + .sort((a, b) => a.order - b.order); + } + + public async syncConditions(experimentId: string, currentConditions: ConditionRef[]): Promise { + const config = await this.configRepository.findByExperimentId(experimentId); + if (!config) return; + + const existingIds = new Set(config.conditionPosteriorStates.map((s) => s.conditionId)); + const currentIds = new Set(currentConditions.map((c) => c.id)); + + const toAdd = currentConditions.filter((c) => !existingIds.has(c.id)); + await Promise.all( + toAdd.map((condition) => + this.posteriorStateRepository.save({ + configId: config.id, + conditionId: condition.id, + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + totalCount: 0, + }) + ) + ); + + const toRemove = config.conditionPosteriorStates.filter((s) => !currentIds.has(s.conditionId)); + if (toRemove.length > 0) { + await this.posteriorStateRepository.remove(toRemove); + } + } +} diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts new file mode 100644 index 0000000000..80582b9b6c --- /dev/null +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -0,0 +1,162 @@ +import { Service } from 'typedi'; +import { HttpError } from 'routing-controllers'; +import { InjectRepository } from '../../typeorm-typedi-extensions'; +import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; +import { BinaryRewardAllowedValue, EXPERIMENT_STATE, SERVER_ERROR } from 'upgrade_types'; +import { ThompsonSamplingRewardRepository } from '../repositories/ThompsonSamplingRewardRepository'; +import { ConditionPosteriorStateRepository } from '../repositories/ConditionPosteriorStateRepository'; +import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; +import { IndividualEnrollmentRepository } from '../repositories/IndividualEnrollmentRepository'; +import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; +import { RewardValidator } from '../controllers/validators/RewardValidator'; +import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; + +export interface IThompsonSamplingRewardResponse { + message: string; + request: RewardValidator; +} + +@Service() +export class ThompsonSamplingRewardService { + constructor( + @InjectRepository() + private tsRewardRepository: ThompsonSamplingRewardRepository, + @InjectRepository() + private posteriorStateRepository: ConditionPosteriorStateRepository, + @InjectRepository() + private tsConfigRepository: ThompsonSamplingExperimentConfigRepository, + @InjectRepository() + private individualEnrollmentRepository: IndividualEnrollmentRepository + ) {} + + public async recordReward( + user: RequestedExperimentUser, + request: RewardValidator, + logger: UpgradeLogger + ): Promise { + const { experimentId, context, decisionPoint, rewardValue } = request; + const success = rewardValue === BinaryRewardAllowedValue.SUCCESS; + + try { + const config = experimentId + ? await this.findConfigById(experimentId, request, logger) + : await this.findConfigByDecisionPoint(context, decisionPoint, request, logger); + + if (config.experiment.state !== EXPERIMENT_STATE.ENROLLING) { + this.throwConflictError( + `Experiment ${config.experimentId} is not actively enrolling (state: ${config.experiment.state}), reward not recorded.`, + request, + logger + ); + } + + const enrollments = await this.individualEnrollmentRepository.findEnrollments(user.id, [config.experimentId]); + + if (!enrollments.length || enrollments.length > 1) { + this.throwConflictError( + `Could not find unique enrollment for user ${user.id} in experiment ${config.experimentId}, reward not recorded.`, + request, + logger + ); + } + + const { conditionId } = enrollments[0]; + + await this.tsRewardRepository.save({ + experimentId: config.experimentId, + conditionId, + userId: user.id, + success, + }); + + const state = await this.posteriorStateRepository.findByConditionId(conditionId); + + if (!state) { + this.throwConflictError( + `No posterior state found for condition ${conditionId} in experiment ${config.experimentId}, reward not recorded.`, + request, + logger + ); + } + + // Increment counts atomically; successCount only increments on success + await this.posteriorStateRepository.increment({ id: state.id }, 'totalCount', 1); + if (success) { + await this.posteriorStateRepository.increment({ id: state.id }, 'successCount', 1); + } + + logger.info({ + message: 'Thompson Sampling reward recorded', + experimentId: config.experimentId, + conditionId, + userId: user.id, + success, + }); + + return { message: 'Reward recorded successfully.', request }; + } catch (error) { + if (error instanceof HttpError) throw error; + this.throwConflictError( + `Failed to record reward (userId: ${user.id}, experimentId: ${experimentId ?? 'not provided'}).`, + request, + logger + ); + } + } + + private async findConfigById( + experimentId: string, + request: RewardValidator, + logger: UpgradeLogger + ): Promise { + const config = await this.tsConfigRepository.findOne({ + where: { experimentId }, + relations: { experiment: true }, + }); + + if (!config) { + this.throwConflictError( + `No Thompson Sampling config found for experiment ${experimentId}, reward not recorded.`, + request, + logger + ); + } + + return config; + } + + private async findConfigByDecisionPoint( + context: string, + decisionPoint: { site: string; target: string }, + request: RewardValidator, + logger: UpgradeLogger + ): Promise { + const { site, target } = decisionPoint; + const configs = await this.tsConfigRepository.findByDecisionPoint(context, site, target); + + if (configs.length === 0) { + this.throwConflictError( + `No active Thompson Sampling experiment found for decision point (context: ${context}, site: ${site}, target: ${target}).`, + request, + logger + ); + } + + if (configs.length > 1) { + this.throwConflictError( + `Multiple active Thompson Sampling experiments found for decision point (context: ${context}, site: ${site}, target: ${target}); use experimentId to disambiguate.`, + request, + logger + ); + } + + return configs[0]; + } + + private throwConflictError(message: string, request: RewardValidator, logger: UpgradeLogger): never { + logger.error({ message, request }); + const error = new HttpError(409, message); + (error as any).type = SERVER_ERROR.ASSIGNMENT_ERROR; + throw error; + } +} diff --git a/packages/backend/src/api/services/ThompsonSamplingService.ts b/packages/backend/src/api/services/ThompsonSamplingService.ts new file mode 100644 index 0000000000..1d81aea769 --- /dev/null +++ b/packages/backend/src/api/services/ThompsonSamplingService.ts @@ -0,0 +1,183 @@ +import { Service } from 'typedi'; + +export interface ConditionPrior { + success: number; + failure: number; +} + +export interface ConditionRewardSummary { + conditionCode: string; + successCount: number; + totalCount: number; +} + +export interface ThompsonSamplingConfig { + /** Per-condition Beta distribution priors. Conditions without an entry use DEFAULT_PRIOR. */ + priors?: Record; + /** Use uniform random selection until total enrollments exceed this count. */ + warmupThreshold?: number; + /** Fall back to uniform when the top two sampled draws differ by less than this value. */ + minimumDrawDifference?: number; +} + +export const DEFAULT_PRIOR: ConditionPrior = { success: 1, failure: 1 }; + +@Service() +export class ThompsonSamplingService { + /** + * Select a condition using Thompson Sampling. + * + * @param conditionCodes - All eligible condition codes for this experiment + * @param rewardSummaries - Accumulated reward counts per condition + * @param totalEnrollments - Total number of enrollments across all conditions + * @param config - Optional algorithm parameters (priors, warmup, thresholds) + */ + /** + * Estimate how often each condition would "win" a Thompson Sampling draw given current posteriors. + * + * Runs `numDraws` simulated rounds. Each round samples Beta(alpha, beta) for every condition and + * awards the round to the highest draw. Win counts are converted to integer percentages via the + * Largest Remainder Method so the result always sums to exactly 100. + * + * @param conditions - Each condition with its current posterior alpha and beta parameters + * @param numDraws - Number of simulated draws (default 10 000) + * @returns Map of conditionCode → integer percentage in [0, 100]; sums to 100 + */ + estimateConditionWeights( + conditions: Array<{ code: string; alpha: number; beta: number }>, + numDraws = 10_000 + ): Record { + if (conditions.length === 0) return {}; + if (conditions.length === 1) return { [conditions[0].code]: 100 }; + + const wins = new Map(conditions.map((c) => [c.code, 0])); + + for (let i = 0; i < numDraws; i++) { + let bestDraw = -1; + let winner = conditions[0].code; + for (const { code, alpha, beta } of conditions) { + const draw = this.sampleBeta(alpha, beta); + if (draw > bestDraw) { + bestDraw = draw; + winner = code; + } + } + wins.set(winner, (wins.get(winner) ?? 0) + 1); + } + + // Largest Remainder Method: floor each raw percentage then distribute the + // remaining integer points to the conditions with the largest fractional parts. + const rawPcts = conditions.map((c) => { + const raw = ((wins.get(c.code) ?? 0) / numDraws) * 100; + return { code: c.code, floor: Math.floor(raw), remainder: raw - Math.floor(raw) }; + }); + const pointsLeft = 100 - rawPcts.reduce((sum, r) => sum + r.floor, 0); + rawPcts.sort((a, b) => b.remainder - a.remainder); + + const result: Record = {}; + rawPcts.forEach((r, i) => { + result[r.code] = r.floor + (i < pointsLeft ? 1 : 0); + }); + return result; + } + + selectCondition( + conditionCodes: string[], + rewardSummaries: ConditionRewardSummary[], + totalEnrollments: number, + config: ThompsonSamplingConfig = {} + ): string { + if (conditionCodes.length === 0) { + throw new Error('Cannot select from an empty condition list'); + } + if (conditionCodes.length === 1) { + return conditionCodes[0]; + } + + // Warmup phase: use uniform random until sufficient data has been collected + if (config.warmupThreshold !== undefined && totalEnrollments <= config.warmupThreshold) { + return this.uniformRandom(conditionCodes); + } + + const summaryMap = new Map(rewardSummaries.map((s) => [s.conditionCode, s])); + + const draws = conditionCodes.map((code) => { + const summary = summaryMap.get(code); + const prior = config.priors?.[code] ?? DEFAULT_PRIOR; + const alpha = prior.success + (summary?.successCount ?? 0); + const beta = prior.failure + (summary ? summary.totalCount - summary.successCount : 0); + return { code, draw: this.sampleBeta(alpha, beta) }; + }); + + draws.sort((a, b) => b.draw - a.draw); + + // Fall back to uniform when the top two draws are too close to distinguish + if ( + config.minimumDrawDifference !== undefined && + draws.length >= 2 && + draws[0].draw - draws[1].draw < config.minimumDrawDifference + ) { + return this.uniformRandom(conditionCodes); + } + + return draws[0].code; + } + + private uniformRandom(conditionCodes: string[]): string { + return conditionCodes[Math.floor(Math.random() * conditionCodes.length)]; + } + + // Beta(α, β) sampled as the ratio of two independent Gamma samples + private sampleBeta(alpha: number, beta: number): number { + const x = this.sampleGamma(alpha); + const y = this.sampleGamma(beta); + return x / (x + y); + } + + // Gamma(α, 1) via Marsaglia–Tsang's squeeze method. + // Source: Marsaglia, G. & Tsang, W.W. (2000). "A Simple Method for Generating Gamma Variables." + // ACM Transactions on Mathematical Software, 26(3), pp. 363–372. DOI: 10.1145/358407.358414 + // The same algorithm is used by d3-random (https://github.com/d3/d3-random/blob/main/src/gamma.js) + // and jStat (https://github.com/jstat/jstat). This implementation follows the paper directly. + private sampleGamma(alpha: number): number { + if (alpha < 1) { + // Reduction: Gamma(α) = Gamma(α+1) · U^(1/α) where U ~ Uniform(0,1) + return this.sampleGamma(alpha + 1) * Math.pow(Math.random(), 1 / alpha); + } + const d = alpha - 1 / 3; + const c = 1 / Math.sqrt(9 * d); + for (;;) { + let x: number; + let v: number; + do { + x = this.sampleNormal(); + v = 1 + c * x; + } while (v <= 0); + v = v * v * v; + const u = Math.random(); + // Fast accept path (avoids log when safe) + if (u < 1 - 0.0331 * x * x * x * x) { + return d * v; + } + if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) { + return d * v; + } + } + } + + // N(0,1) via the Marsaglia polar method (polar form of Box–Muller). + // Source: Knuth, D.E. (1997). The Art of Computer Programming, Vol. 2, §3.4.1, Algorithm P. + // Original derivation: Marsaglia, G. & Bray, T.A. (1964). "A Convenient Method for Generating + // Normal Variables." SIAM Review, 6(3), pp. 260–264. DOI: 10.1137/1006063 + // Preferred over the basic Box–Muller form because it avoids evaluating trig functions and + // handles the degenerate case where Math.random() returns exactly 0 (log(0) = -Infinity). + private sampleNormal(): number { + let u: number, v: number, s: number; + do { + u = Math.random() * 2 - 1; + v = Math.random() * 2 - 1; + s = u * u + v * v; + } while (s >= 1 || s === 0); + return u * Math.sqrt((-2 * Math.log(s)) / s); + } +} diff --git a/packages/backend/src/database/migrations/1781222400000-thompsonSamplingEntities.ts b/packages/backend/src/database/migrations/1781222400000-thompsonSamplingEntities.ts new file mode 100644 index 0000000000..8a35988a53 --- /dev/null +++ b/packages/backend/src/database/migrations/1781222400000-thompsonSamplingEntities.ts @@ -0,0 +1,118 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ThompsonSamplingEntities1781222400000 implements MigrationInterface { + name = 'ThompsonSamplingEntities1781222400000'; + + public async up(queryRunner: QueryRunner): Promise { + // Add thompson_sampling to the assignment algorithm enum + await queryRunner.query( + `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` + ); + await queryRunner.query( + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'ts_configurable', 'thompson_sampling')` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); + await queryRunner.query( + `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); + await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); + + // thompson_sampling_experiment_config: one-to-one with experiment + await queryRunner.query( + `CREATE TABLE "thompson_sampling_experiment_config" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "experimentId" uuid, + "warmupThreshold" integer, + "minimumDrawDifference" double precision, + "batchSize" integer, + "createdAt" TIMESTAMP NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), + "versionNumber" integer NOT NULL, + CONSTRAINT "UQ_ts_config_experimentId" UNIQUE ("experimentId"), + CONSTRAINT "PK_ts_config" PRIMARY KEY ("id") + )` + ); + + // condition_posterior_state: per-condition Beta distribution state + await queryRunner.query( + `CREATE TABLE "condition_posterior_state" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "configId" uuid NOT NULL, + "conditionId" uuid NOT NULL, + "priorSuccess" double precision NOT NULL DEFAULT 1, + "priorFailure" double precision NOT NULL DEFAULT 1, + "successCount" integer NOT NULL DEFAULT 0, + "totalCount" integer NOT NULL DEFAULT 0, + "createdAt" TIMESTAMP NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), + "versionNumber" integer NOT NULL, + CONSTRAINT "UQ_posterior_config_condition" UNIQUE ("configId", "conditionId"), + CONSTRAINT "PK_condition_posterior_state" PRIMARY KEY ("id") + )` + ); + + // thompson_sampling_reward: raw reward events (audit trail + posterior recalculation) + await queryRunner.query( + `CREATE TABLE "thompson_sampling_reward" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "experimentId" uuid NOT NULL, + "conditionId" uuid NOT NULL, + "userId" character varying NOT NULL, + "success" boolean NOT NULL, + "createdAt" TIMESTAMP NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), + "versionNumber" integer NOT NULL, + CONSTRAINT "PK_ts_reward" PRIMARY KEY ("id") + )` + ); + await queryRunner.query( + `CREATE INDEX "IDX_ts_reward_experiment_condition" ON "thompson_sampling_reward" ("experimentId", "conditionId")` + ); + + await queryRunner.query( + `ALTER TABLE "thompson_sampling_experiment_config" ADD CONSTRAINT "FK_ts_config_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_config" FOREIGN KEY ("configId") REFERENCES "thompson_sampling_experiment_config"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_condition" FOREIGN KEY ("conditionId") REFERENCES "experiment_condition"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_condition" FOREIGN KEY ("conditionId") REFERENCES "experiment_condition"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "thompson_sampling_reward" DROP CONSTRAINT "FK_ts_reward_condition"`); + await queryRunner.query(`ALTER TABLE "thompson_sampling_reward" DROP CONSTRAINT "FK_ts_reward_experiment"`); + await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_condition"`); + await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_config"`); + await queryRunner.query( + `ALTER TABLE "thompson_sampling_experiment_config" DROP CONSTRAINT "FK_ts_config_experiment"` + ); + + await queryRunner.query(`DROP INDEX "IDX_ts_reward_experiment_condition"`); + await queryRunner.query(`DROP TABLE "thompson_sampling_reward"`); + await queryRunner.query(`DROP TABLE "condition_posterior_state"`); + await queryRunner.query(`DROP TABLE "thompson_sampling_experiment_config"`); + + // Remove thompson_sampling from the enum + await queryRunner.query( + `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` + ); + await queryRunner.query( + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'ts_configurable')` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); + await queryRunner.query( + `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); + await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); + } +} diff --git a/packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts b/packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts new file mode 100644 index 0000000000..5a84039807 --- /dev/null +++ b/packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts @@ -0,0 +1,46 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class CleanupMoocletEntities1781308800000 implements MigrationInterface { + name = 'CleanupMoocletEntities1781308800000'; + + public async up(queryRunner: QueryRunner): Promise { + // Migrate any existing ts_configurable experiments to thompson_sampling + await queryRunner.query( + `UPDATE "experiment" SET "assignmentAlgorithm" = 'thompson_sampling' WHERE "assignmentAlgorithm" = 'ts_configurable'` + ); + + // Drop mooclet tables (cascade handles FK references) + await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_version_condition_map"`); + await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_experiment_ref"`); + + // Remove ts_configurable from the assignment algorithm enum + await queryRunner.query( + `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` + ); + await queryRunner.query( + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'thompson_sampling')` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); + await queryRunner.query( + `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); + await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); + } + + public async down(queryRunner: QueryRunner): Promise { + // Restore ts_configurable enum value + await queryRunner.query( + `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` + ); + await queryRunner.query( + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'ts_configurable', 'thompson_sampling')` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); + await queryRunner.query( + `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); + await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); + } +} diff --git a/packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts b/packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts new file mode 100644 index 0000000000..a305917084 --- /dev/null +++ b/packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts @@ -0,0 +1,44 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class BootstrapThompsonSamplingConfigs1781395200000 implements MigrationInterface { + name = 'BootstrapThompsonSamplingConfigs1781395200000'; + + public async up(queryRunner: QueryRunner): Promise { + // Create ThompsonSamplingExperimentConfig rows for any thompson_sampling experiments + // that don't have one yet (e.g. experiments converted from ts_configurable by the cleanup migration). + await queryRunner.query(` + INSERT INTO "thompson_sampling_experiment_config" ("experimentId", "versionNumber") + SELECT e.id, 1 + FROM "experiment" e + WHERE e."assignmentAlgorithm" = 'thompson_sampling' + AND NOT EXISTS ( + SELECT 1 FROM "thompson_sampling_experiment_config" c WHERE c."experimentId" = e.id + ) + `); + + // Create ConditionPosteriorState rows for each condition of those experiments. + await queryRunner.query(` + INSERT INTO "condition_posterior_state" ("configId", "conditionId", "priorSuccess", "priorFailure", "successCount", "totalCount", "versionNumber") + SELECT c.id, ec.id, 1, 1, 0, 0, 1 + FROM "thompson_sampling_experiment_config" c + JOIN "experiment_condition" ec ON ec."experimentId" = c."experimentId" + WHERE NOT EXISTS ( + SELECT 1 FROM "condition_posterior_state" ps + WHERE ps."configId" = c.id AND ps."conditionId" = ec.id + ) + `); + } + + public async down(queryRunner: QueryRunner): Promise { + // Remove posterior states and configs that were created by this migration. + // We identify "bootstrapped" rows as those with no warmupThreshold/minimumDrawDifference/batchSize + // (all nullable, all NULL means they came from this migration with defaults only). + // This is a best-effort rollback — if configs were subsequently edited, those edits are lost. + await queryRunner.query(` + DELETE FROM "thompson_sampling_experiment_config" + WHERE "warmupThreshold" IS NULL + AND "minimumDrawDifference" IS NULL + AND "batchSize" IS NULL + `); + } +} diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 06889c67a4..089e47c49b 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -122,10 +122,4 @@ export const env = { secret: getOsEnv('CLIENT_API_SECRET'), key: getOsEnv('CLIENT_API_KEY'), }, - mooclets: { - enabled: toBool(getOsEnvOptional('MOOCLETS_ENABLED')) || false, - hostUrl: getOsEnvOptional('MOOCLETS_HOST_URL') || '', - apiRoute: getOsEnvOptional('MOOCLETS_API_ROUTE') || '', - apiToken: getOsEnvOptional('MOOCLETS_API_TOKEN') || '', - }, }; diff --git a/packages/backend/src/types/Mooclet.ts b/packages/backend/src/types/Mooclet.ts deleted file mode 100644 index 6ac16b1186..0000000000 --- a/packages/backend/src/types/Mooclet.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { MoocletPolicyParametersDTO } from 'upgrade_types'; - -export interface MoocletProxyRequestParams { - method: string; - url: string; - apiToken: string; - body?: MoocletRequestBody | MoocletPolicyParametersRequestBody | MoocletValueRequestBody | MoocletVariableRequestBody; -} - -export interface MoocletRequestBody { - name: string; - policy: number; -} - -export interface MoocletPaginatedResponse { - count: number; - next: string | null; - previous: string | null; - results: T[]; -} - -export interface MoocletResponseDetails { - id: number; - name: string; - policy: number; - mooclet_id?: string; - environment?: string; -} - -export interface MoocletVersionRequestBody { - mooclet: number; - name: string; - text?: string; - version_json?: Record; -} - -export interface MoocletVersionResponseDetails { - id: number; - name: string; - mooclet: number; - version_id?: number; - text?: string; - version_json?: Record; -} - -export interface MoocletPolicyParametersRequestBody { - mooclet: number; - policy: number; - parameters: MoocletPolicyParametersDTO; -} - -export interface MoocletPolicyParametersResponseDetails { - id: number; - mooclet: number; - policy: number; - parameters: MoocletPolicyParametersDTO; -} - -export interface MoocletVariableRequestBody { - name: string; -} - -export interface MoocletVariableResponseDetails { - id: number; - environment?: null; - variable_id?: null; - name: string; - min_value: number; - max_value: number; - value_type: 'BIN'; // binary is only supported type, must use 0 or 1 - sample_thres: number; -} - -export interface MoocletValueRequestBody { - variable: string; - value: number; - mooclet: number; - version: number; - learner?: number | string; - policy?: number; -} - -export interface MoocletRewardCountRequestBody { - moocletId: number; - variableName: string; -} - -export interface MoocletValueResponseDetails { - id: string; - variable: string; - learner: string; - mooclet: number; - version: number; - policy: number; - value: number; - text: string; - timestamp: string; -} - -export interface MoocletPolicyResponseDetails { - id: number; - name: string; - environment?: string; -} - -export interface MoocletBatchResponse { - count: number; - next: string; - previous: string; - results: T[]; -} diff --git a/packages/backend/test/unit/controllers/ExperimentController.test.ts b/packages/backend/test/unit/controllers/ExperimentController.test.ts index f5990855f0..827b5768f6 100644 --- a/packages/backend/test/unit/controllers/ExperimentController.test.ts +++ b/packages/backend/test/unit/controllers/ExperimentController.test.ts @@ -9,13 +9,9 @@ import { ExperimentService } from '../../../src/api/services/ExperimentService'; import { useContainer as classValidatorUseContainer } from 'class-validator'; import { ExperimentAssignmentService } from '../../../src/api/services/ExperimentAssignmentService'; import ExperimentAssignmentServiceMock from './mocks/ExperimentAssignmentServiceMock'; -import { MoocletExperimentService } from '../../../src/api/services/MoocletExperimentService'; -import MoocletExperimentServiceMock from './mocks/MoocletExperimentServiceMock'; -import { MoocletRewardsService } from '../../../src/api/services/MoocletRewardsService'; -import MoocletRewardsServiceMock from './mocks/MoocletRewardsServiceMock'; import { ImportExportService } from '../../../src/api/services/ImportExportService'; +import { ThompsonSamplingExperimentCrudService } from '../../../src/api/services/ThompsonSamplingExperimentCrudService'; import ImportExportServiceMock from './mocks/ImportExportServiceMock'; -import { env } from './../../../src/env'; import { ASSIGNMENT_ALGORITHM, ASSIGNMENT_UNIT, @@ -23,7 +19,6 @@ import { EXPERIMENT_STATE, EXPERIMENT_TYPE, FILTER_MODE, - MoocletTSConfigurablePolicyParametersDTO, POST_EXPERIMENT_RULE, SEGMENT_TYPE, } from 'upgrade_types'; @@ -38,9 +33,8 @@ describe('Experiment Controller Testing', () => { // set mock container Container.set(ExperimentService, new ExperimentServiceMock()); Container.set(ExperimentAssignmentService, new ExperimentAssignmentServiceMock()); - Container.set(MoocletExperimentService, new MoocletExperimentServiceMock()); - Container.set(MoocletRewardsService, new MoocletRewardsServiceMock()); Container.set(ImportExportService, new ImportExportServiceMock()); + Container.set(ThompsonSamplingExperimentCrudService, {} as any); }); afterAll(() => { @@ -113,19 +107,6 @@ describe('Experiment Controller Testing', () => { ], }; - const tsConfigurablePolicyParameters = new MoocletTSConfigurablePolicyParametersDTO(); - tsConfigurablePolicyParameters.assignmentAlgorithm = ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE; - tsConfigurablePolicyParameters.outcome_variable_name = 'test_outcome'; - - const moocletExperimentData: ExperimentDTO = { - ...experimentData, - id: crypto.randomUUID(), - moocletPolicyParameters: tsConfigurablePolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - - console.log({ moocletExperimentData }); - //for future use where user will be mocked for all testcases // const mockUser: User = { @@ -147,16 +128,6 @@ describe('Experiment Controller Testing', () => { return request(app).post('/api/experiments').send(experimentData).expect('Content-Type', /json/).expect(200); }); - test('Post request for /api/experiments with moocletPolicyParameters and mooclets enabled', () => { - env.mooclets.enabled = true; - return request(app).post('/api/experiments').send(moocletExperimentData).expect('Content-Type', /json/).expect(200); - }); - - test('Post request for /api/experiments with moocletPolicyParameters and mooclets disabled', () => { - env.mooclets.enabled = false; - return request(app).post('/api/experiments').send(moocletExperimentData).expect(500); - }); - test('Get request for /api/experiments/names', () => { return request(app).get('/api/experiments/names').expect('Content-Type', /json/).expect(200); }); diff --git a/packages/backend/test/unit/controllers/mocks/MoocletExperimentServiceMock.ts b/packages/backend/test/unit/controllers/mocks/MoocletExperimentServiceMock.ts deleted file mode 100644 index 62860a66fb..0000000000 --- a/packages/backend/test/unit/controllers/mocks/MoocletExperimentServiceMock.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { SyncCreateParams } from '../../../../src/api/services/MoocletExperimentService'; -import { Service } from 'typedi'; - -@Service() -export default class MoocletExperimentServiceMock { - public async syncCreate(params: SyncCreateParams): Promise<[]> { - return Promise.resolve([]); - } -} diff --git a/packages/backend/test/unit/controllers/mocks/MoocletRewardsServiceMock.ts b/packages/backend/test/unit/controllers/mocks/MoocletRewardsServiceMock.ts deleted file mode 100644 index 39796fefd2..0000000000 --- a/packages/backend/test/unit/controllers/mocks/MoocletRewardsServiceMock.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Service } from 'typedi'; - -@Service() -export default class MoocletRewardsServiceMock { - public async getRewardsSummaryForExperiment(experimentId: string, logger: any): Promise { - return [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; - } -} diff --git a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts index 25f6543a37..270db9395e 100644 --- a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts @@ -35,7 +35,6 @@ import { ENROLLMENT_CODE, EXPERIMENT_STATE, FILTER_MODE, MARKED_DECISION_POINT_S import { CacheService } from '../../../src/api/services/CacheService'; import { UserStratificationFactorRepository } from '../../../src/api/repositories/UserStratificationRepository'; import { configureLogger } from '../../utils/logger'; -import { MoocletExperimentService } from '../../../src/api/services/MoocletExperimentService'; import { ExperimentPrecomputedSegmentService } from '../../../src/api/services/ExperimentPrecomputedSegmentService'; import { factorialGroupExperiment, factorialIndividualExperiment } from '../mockdata/raw'; import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; @@ -72,7 +71,6 @@ describe('Experiment Assignment Service Test', () => { const segmentServiceMock = sinon.createStubInstance(SegmentService); const experimentServiceMock = sinon.createStubInstance(ExperimentService); const cacheServiceMock = sinon.createStubInstance(CacheService); - const moocletExperimentServiceMock = sinon.createStubInstance(MoocletExperimentService); const experimentPrecomputedSegmentServiceMock = sinon.createStubInstance(ExperimentPrecomputedSegmentService); // Default to "no precomputed rows" so the assignment read path exercises the on-the-fly fallback // (recursive segment resolution) these tests were written against. @@ -130,6 +128,7 @@ describe('Experiment Assignment Service Test', () => { stateTimeLogsRepositoryMock, analyticsRepositoryMock, userStratificationFactorRepositoryMock, + {} as any, // thompsonSamplingConfigRepository — not used in existing tests previewUserServiceMock, experimentUserServiceMock, errorServiceMock, @@ -137,8 +136,8 @@ describe('Experiment Assignment Service Test', () => { segmentServiceMock, experimentServiceMock, cacheServiceMock, - moocletExperimentServiceMock, - experimentPrecomputedSegmentServiceMock + experimentPrecomputedSegmentServiceMock, + {} as any // thompsonSamplingService — not used in existing tests ); testedModule.cacheService.wrap.resolves([]); @@ -2246,17 +2245,4 @@ describe('Experiment Assignment Service Test', () => { }); }); }); - - it('[getConditionFromMoocletProxy] should return undefined and log error when mooclet proxy throws', async () => { - const userDoc = { id: 'user123', group: {}, workingGroup: {} }; - const exp = structuredClone(simpleIndividualAssignmentExperiment); - const mockError = new Error('Mooclet proxy error'); - - moocletExperimentServiceMock.getConditionFromMoocletProxy.rejects(mockError); - - const result = await (testedModule as any).getConditionFromMoocletProxy(exp, userDoc, loggerMock); - - expect(result).toBeUndefined(); - sinon.assert.calledOnce(loggerMock.error); - }); }); diff --git a/packages/backend/test/unit/services/ExperimentService.test.ts b/packages/backend/test/unit/services/ExperimentService.test.ts index fcb83fb207..8865c40bd9 100644 --- a/packages/backend/test/unit/services/ExperimentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentService.test.ts @@ -23,7 +23,6 @@ import { LevelRepository } from '../../../src/api/repositories/LevelRepository'; import { LevelCombinationElementRepository } from '../../../src/api/repositories/LevelCombinationElements'; import { ArchivedStatsRepository } from '../../../src/api/repositories/ArchivedStatsRepository'; import { StratificationFactorRepository } from '../../../src/api/repositories/StratificationFactorRepository'; -import { MoocletExperimentRefRepository } from '../../../src/api/repositories/MoocletExperimentRefRepository'; import { PreviewUserService } from '../../../src/api/services/PreviewUserService'; import { SegmentService } from '../../../src/api/services/SegmentService'; import { ExperimentPrecomputedSegmentService } from '../../../src/api/services/ExperimentPrecomputedSegmentService'; @@ -32,7 +31,6 @@ import { ErrorService } from '../../../src/api/services/ErrorService'; import { CacheService } from '../../../src/api/services/CacheService'; import { QueryService } from '../../../src/api/services/QueryService'; import { MetricService } from '../../../src/api/services/MetricService'; -import { MoocletRewardsService } from '../../../src/api/services/MoocletRewardsService'; import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; import { Experiment } from '../../../src/api/models/Experiment'; import { ExperimentCondition } from '../../../src/api/models/ExperimentCondition'; @@ -399,10 +397,6 @@ describe('ExperimentService Testing', () => { provide: getRepositoryToken(StratificationFactorRepository), useValue: {}, }, - { - provide: getRepositoryToken(MoocletExperimentRefRepository), - useValue: {}, - }, { provide: CacheService, useValue: { @@ -449,10 +443,6 @@ describe('ExperimentService Testing', () => { withRecompute: jest.fn(async (_logger: any, _resolve: any, work: any) => work()), }, }, - { - provide: MoocletRewardsService, - useValue: {}, - }, ], }).compile(); @@ -965,7 +955,7 @@ describe('ExperimentService Testing', () => { it('awaits exclusion-list attachment before returning, even with no inclusion lists, under a caller-owned transaction', async () => { // Regression: the only `await Promise.all(addListPromises)` used to sit inside the inclusion-list // branch, so an exclusion-only experiment created within a caller-owned transaction - // (existingEntityManager, e.g. MoocletExperimentService) returned with its addList insert still in + // (existingEntityManager) returned with its addList insert still in // flight — racing the caller's commit. create() must now await it unconditionally. let resolveAddList: () => void; const addListPending = new Promise((res) => { diff --git a/packages/backend/test/unit/services/MoocletDataService.test.ts b/packages/backend/test/unit/services/MoocletDataService.test.ts deleted file mode 100644 index 83055080d0..0000000000 --- a/packages/backend/test/unit/services/MoocletDataService.test.ts +++ /dev/null @@ -1,571 +0,0 @@ -import { - MoocletBatchResponse, - MoocletPolicyParametersRequestBody, - MoocletPolicyParametersResponseDetails, - MoocletPolicyResponseDetails, - MoocletProxyRequestParams, - MoocletRequestBody, - MoocletResponseDetails, - MoocletVariableRequestBody, - MoocletVariableResponseDetails, - MoocletVersionRequestBody, - MoocletVersionResponseDetails, -} from '../../../src/types/Mooclet'; -import { MoocletDataService } from '../../../src/api/services/MoocletDataService'; -import { Container } from 'typedi'; -import axios from 'axios'; -import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; -import { MoocletTSConfigurablePolicyParametersDTO, ASSIGNMENT_ALGORITHM } from 'upgrade_types'; - -jest.mock('axios'); - -describe('#MoocletDataService', () => { - let moocletDataService: MoocletDataService; - const logger = { - error: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - } as unknown as UpgradeLogger; - - beforeAll(() => { - moocletDataService = Container.get(MoocletDataService); - }); - - afterAll(() => { - Container.reset(); - }); - - describe('#getMoocletIdByName', () => { - it('should return the correct Mooclet ID when the policy name matches', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 2, - next: null, - previous: null, - results: [ - { id: 1, name: 'Policy1' }, - { id: 2, name: 'Policy2' }, - ], - }; - - jest.spyOn(moocletDataService, 'getPoliciesList').mockResolvedValue(mockPoliciesList); - - const moocletId = await moocletDataService.getMoocletIdByName('Policy2', logger); - expect(moocletId).toBe(2); - }); - - it('should return null when the policy name does not match', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 2, - next: null, - previous: null, - results: [ - { id: 1, name: 'Policy1' }, - { id: 2, name: 'Policy2' }, - ], - }; - - jest.spyOn(moocletDataService, 'getPoliciesList').mockResolvedValue(mockPoliciesList); - - const moocletId = await moocletDataService.getMoocletIdByName('Policy3', logger); - expect(moocletId).toBeNull(); - }); - - it('should return null when the policies list is empty', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 0, - next: null, - previous: null, - results: [], - }; - - jest.spyOn(moocletDataService, 'getPoliciesList').mockResolvedValue(mockPoliciesList); - - const moocletId = await moocletDataService.getMoocletIdByName('Policy1', logger); - expect(moocletId).toBeNull(); - }); - - it('should throw an error when getPoliciesList fails', async () => { - jest.spyOn(moocletDataService, 'getPoliciesList').mockRejectedValue(new Error('Failed to fetch policies')); - - await expect(moocletDataService.getMoocletIdByName('Policy1', logger)).rejects.toThrow( - 'Failed to fetch policies' - ); - }); - - afterAll(() => { - jest.restoreAllMocks(); - }); - }); - - describe('#getPoliciesList', () => { - it('should return the list of policies', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 2, - next: null, - previous: null, - results: [ - { id: 1, name: 'Policy1' }, - { id: 2, name: 'Policy2' }, - ], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockPoliciesList); - - const policiesList = await moocletDataService.getPoliciesList(logger); - expect(policiesList).toEqual(mockPoliciesList); - }); - - it('should throw an error when fetchExternalMoocletsData fails', async () => { - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to fetch policies')); - - await expect(moocletDataService.getPoliciesList(logger)).rejects.toThrow('Failed to fetch policies'); - }); - }); - - describe('#postNewMooclet', () => { - it('should return the new Mooclet details when the request is successful', async () => { - const mockRequestBody: MoocletRequestBody = { - name: 'New Mooclet', - policy: 3, - }; - - const mockResponseDetails: MoocletResponseDetails = { - id: 1, - name: 'New Mooclet', - policy: 3, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewMooclet(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletRequestBody = { - name: 'New Mooclet', - policy: 3, - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create mooclet')); - - await expect(moocletDataService.postNewMooclet(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create mooclet' - ); - }); - }); - - describe('#deleteMooclet', () => { - it('should return the response when the delete request is successful', async () => { - const moocletId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deleteMooclet(moocletId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const moocletId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete mooclet')); - - await expect(moocletDataService.deleteMooclet(moocletId, logger)).rejects.toThrow('Failed to delete mooclet'); - }); - }); - - describe('#postNewVersion', () => { - it('should return the new version details when the request is successful', async () => { - const mockRequestBody: MoocletVersionRequestBody = { - mooclet: 1, - name: 'Version 1', - text: 'This is version 1', - version_json: { key: 1 }, - }; - - const mockResponseDetails: MoocletVersionResponseDetails = { - id: 1, - mooclet: 1, - name: 'Version 1', - text: 'This is version 1', - version_json: { key: 1 }, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewVersion(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletVersionRequestBody = { - mooclet: 1, - name: 'Version 1', - text: 'This is version 1', - version_json: { key: 1 }, - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create version')); - - await expect(moocletDataService.postNewVersion(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create version' - ); - }); - }); - - describe('#deleteVersion', () => { - it('should return the response when the delete request is successful', async () => { - const versionId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deleteVersion(versionId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const versionId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete version')); - - await expect(moocletDataService.deleteVersion(versionId, logger)).rejects.toThrow('Failed to delete version'); - }); - }); - - describe('#postNewPolicyParameters', () => { - const mockPolicyParameters: MoocletTSConfigurablePolicyParametersDTO = { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - prior: { - 1: { failure: 1, success: 1 }, - 2: { failure: 1, success: 1 }, - }, - batch_size: 1, - max_rating: 1, - min_rating: 0, - uniform_threshold: 0, - tspostdiff_thresh: 0, - outcome_variable_name: 'example_reward_var', - }; - it('should return the new policy parameters details when the request is successful', async () => { - const mockRequestBody: MoocletPolicyParametersRequestBody = { - mooclet: 1, - policy: 2, - parameters: { - ...mockPolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }, - }; - - const mockResponseDetails: MoocletPolicyParametersResponseDetails = { - id: 1, - mooclet: 1, - policy: 2, - parameters: { - ...mockPolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewPolicyParameters(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletPolicyParametersRequestBody = { - mooclet: 1, - policy: 2, - parameters: { - ...mockPolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }, - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create policy parameters')); - - await expect(moocletDataService.postNewPolicyParameters(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create policy parameters' - ); - }); - }); - - describe('#deletePolicyParameters', () => { - it('should return the response when the delete request is successful', async () => { - const policyParametersId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deletePolicyParameters(policyParametersId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const policyParametersId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete policy parameters')); - - await expect(moocletDataService.deletePolicyParameters(policyParametersId, logger)).rejects.toThrow( - 'Failed to delete policy parameters' - ); - }); - }); - - describe('#postNewVariable', () => { - it('should return the new variable details when the request is successful', async () => { - const mockRequestBody: MoocletVariableRequestBody = { - name: 'New Variable', - }; - - const mockResponseDetails: MoocletVariableResponseDetails = { - id: 1, - environment: null, - variable_id: null, - name: 'New Variable', - min_value: 0, - max_value: 1, - value_type: 'BIN', - sample_thres: 1, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewVariable(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletVariableRequestBody = { - name: 'New Variable', - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create variable')); - - await expect(moocletDataService.postNewVariable(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create variable' - ); - }); - }); - - describe('#deleteVariable', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should return the response when the delete request is successful', async () => { - const variableId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deleteVariable(variableId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const variableId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete variable')); - - await expect(moocletDataService.deleteVariable(variableId, logger)).rejects.toThrow('Failed to delete variable'); - }); - - afterAll(() => { - jest.restoreAllMocks(); - }); - }); - - describe('#fetchExternalMoocletsData', () => { - const mockRequest: MoocletRequestBody = { - name: 'New Mooclet', - policy: 3, - }; - it('should return the response data when the request is successful', async () => { - const mockRequestParams: MoocletProxyRequestParams = { - method: 'POST', - url: 'https://api.example.com/mooclet', - apiToken: 'test-token', - body: { ...mockRequest }, - }; - - const mockResponse = { - status: 200, - data: { success: true }, - }; - - (axios.request as jest.Mock).mockResolvedValue(mockResponse); - - const response = await moocletDataService.fetchExternalMoocletsData(mockRequestParams, logger); - expect(response).toEqual(mockResponse.data); - }); - - it('should throw a MoocletError when the request fails with a non-2xx status', async () => { - const mockRequestParams: MoocletProxyRequestParams = { - method: 'POST', - url: 'https://api.example.com/mooclet', - apiToken: 'test-token', - body: { ...mockRequest }, - }; - - const mockAxiosError = Object.assign(new Error('Request failed with status code 400'), { - response: { - status: 400, - data: { error: 'Bad Request' }, - }, - isAxiosError: true, - }); - - (axios.request as jest.Mock).mockRejectedValue(mockAxiosError); - (axios as any).isAxiosError.mockReturnValueOnce(true); - - await expect(moocletDataService.fetchExternalMoocletsData(mockRequestParams, logger)).rejects.toThrow( - 'Mooclet server returned non-2xx status: 400' - ); - }); - - it('should handle and log errors when the request fails', async () => { - const mockRequestParams: MoocletProxyRequestParams = { - method: 'POST', - url: 'https://api.example.com/mooclet', - apiToken: 'test-token', - body: { ...mockRequest }, - }; - - const mockError = new Error('Mock Network Error'); - const mockErrorMessage = { message: `Failed to communicate with Mooclet server` }; - - (axios.request as jest.Mock).mockRejectedValue(mockError); - logger.error = jest.fn().mockReturnValue(mockErrorMessage); - await expect(moocletDataService.fetchExternalMoocletsData(mockRequestParams, logger)).rejects.toThrow( - mockErrorMessage.message - ); - }); - }); - - describe('#getRewardsForExperiment', () => { - it('should successfully fetch rewards for an experiment', async () => { - const requestBody = { - moocletId: 456, - variableName: 'outcome_var', - }; - - const mockResponse = { - count: 5, - next: null, - previous: null, - results: [ - { id: '1', version: 100, value: 1.0, variable: 'outcome_var', learner: 'user-1', mooclet: 456 }, - { id: '2', version: 100, value: 0.0, variable: 'outcome_var', learner: 'user-2', mooclet: 456 }, - { id: '3', version: 200, value: 1.0, variable: 'outcome_var', learner: 'user-3', mooclet: 456 }, - ], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const result = await moocletDataService.getRewardsForExperiment(requestBody, logger); - - expect(result).toEqual(mockResponse); - expect(moocletDataService.fetchExternalMoocletsData).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: expect.stringContaining('/value?mooclet=456&variable__name=outcome_var'), - }), - logger - ); - }); - - it('should construct correct query parameters with moocletId and variableName', async () => { - const requestBody = { - moocletId: 789, - variableName: 'test_variable', - }; - - const mockResponse = { - count: 0, - next: null, - previous: null, - results: [], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - await moocletDataService.getRewardsForExperiment(requestBody, logger); - - expect(moocletDataService.fetchExternalMoocletsData).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: expect.stringContaining('mooclet=789'), - }), - logger - ); - expect(moocletDataService.fetchExternalMoocletsData).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: expect.stringContaining('variable__name=test_variable'), - }), - logger - ); - }); - - it('should handle paginated responses with next page', async () => { - const requestBody = { - moocletId: 456, - variableName: 'outcome_var', - }; - - const mockResponse = { - count: 100, - next: 'http://api.mooclet.com/next-page', - previous: null, - results: [{ id: '1', version: 100, value: 1.0 }], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const result = await moocletDataService.getRewardsForExperiment(requestBody, logger); - - expect(result.next).toBe('http://api.mooclet.com/next-page'); - expect(result.count).toBe(100); - }); - - it('should throw error when fetchExternalMoocletsData fails', async () => { - const requestBody = { - moocletId: 456, - variableName: 'outcome_var', - }; - - const mockError = new Error('API connection failed'); - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockRejectedValue(mockError); - - await expect(moocletDataService.getRewardsForExperiment(requestBody, logger)).rejects.toThrow( - 'API connection failed' - ); - }); - - afterAll(() => { - jest.restoreAllMocks(); - }); - }); -}); diff --git a/packages/backend/test/unit/services/MoocletExperimentService.test.ts b/packages/backend/test/unit/services/MoocletExperimentService.test.ts deleted file mode 100644 index 1eb3a05de3..0000000000 --- a/packages/backend/test/unit/services/MoocletExperimentService.test.ts +++ /dev/null @@ -1,1855 +0,0 @@ -import 'reflect-metadata'; -import { ExperimentDTO } from '../../../src/api/DTO/ExperimentDTO'; -import { UserDTO } from '../../../src/api/DTO/UserDTO'; -import { MoocletExperimentRef } from '../../../src/api/models/MoocletExperimentRef'; -import { MoocletVersionConditionMap } from '../../../src/api/models/MoocletVersionConditionMap'; -import { Experiment } from '../../../src/api/models/Experiment'; -import { ExperimentCondition } from '../../../src/api/models/ExperimentCondition'; -import { ArchivedStatsRepository } from '../../../src/api/repositories/ArchivedStatsRepository'; -import { ConditionPayloadRepository } from '../../../src/api/repositories/ConditionPayloadRepository'; -import { DecisionPointRepository } from '../../../src/api/repositories/DecisionPointRepository'; -import { ExperimentAuditLogRepository } from '../../../src/api/repositories/ExperimentAuditLogRepository'; -import { SegmentRepository } from '../../../src/api/repositories/SegmentRepository'; -import { ExperimentConditionRepository } from '../../../src/api/repositories/ExperimentConditionRepository'; -import { ExperimentRepository } from '../../../src/api/repositories/ExperimentRepository'; -import { ExperimentSegmentExclusionRepository } from '../../../src/api/repositories/ExperimentSegmentExclusionRepository'; -import { ExperimentSegmentInclusionRepository } from '../../../src/api/repositories/ExperimentSegmentInclusionRepository'; -import { ExperimentUserRepository } from '../../../src/api/repositories/ExperimentUserRepository'; -import { FactorRepository } from '../../../src/api/repositories/FactorRepository'; -import { GroupExclusionRepository } from '../../../src/api/repositories/GroupExclusionRepository'; -import { IndividualExclusionRepository } from '../../../src/api/repositories/IndividualExclusionRepository'; -import { LevelCombinationElementRepository } from '../../../src/api/repositories/LevelCombinationElements'; -import { LevelRepository } from '../../../src/api/repositories/LevelRepository'; -import { MetricRepository } from '../../../src/api/repositories/MetricRepository'; -import { MonitoredDecisionPointRepository } from '../../../src/api/repositories/MonitoredDecisionPointRepository'; -import { MoocletExperimentRefRepository } from '../../../src/api/repositories/MoocletExperimentRefRepository'; -import { QueryRepository } from '../../../src/api/repositories/QueryRepository'; -import { StateTimeLogsRepository } from '../../../src/api/repositories/StateTimeLogsRepository'; -import { StratificationFactorRepository } from '../../../src/api/repositories/StratificationFactorRepository'; -import { CacheService } from '../../../src/api/services/CacheService'; -import { ErrorService } from '../../../src/api/services/ErrorService'; -import { MoocletDataService } from '../../../src/api/services/MoocletDataService'; -import { - MoocletExperimentService, - SyncCreateParams, - SyncEditParams, - SyncDeleteParams, -} from '../../../src/api/services/MoocletExperimentService'; -import { ExperimentPrecomputedSegmentService } from '../../../src/api/services/ExperimentPrecomputedSegmentService'; -import { PreviewUserService } from '../../../src/api/services/PreviewUserService'; -import { QueryService } from '../../../src/api/services/QueryService'; -import { SegmentService } from '../../../src/api/services/SegmentService'; -import { DataSource, EntityManager } from 'typeorm'; -import { - ASSIGNMENT_ALGORITHM, - ASSIGNMENT_UNIT, - CONSISTENCY_RULE, - EXPERIMENT_STATE, - EXPERIMENT_TYPE, - FILTER_MODE, - PAYLOAD_TYPE, - POST_EXPERIMENT_RULE, - SEGMENT_TYPE, -} from 'upgrade_types'; -import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; -import { MetricService } from '../../../src/api/services/MetricService'; -import { ExperimentSchedulerService } from '../../../src/api/services/ExperimentSchedulerService'; - -const mockDataSource = { - initialize: jest.fn(), - destroy: jest.fn(), - transaction: jest.fn((callback) => callback(mockDataSource.manager)), - manager: { - transaction: jest.fn(), - save: jest.fn(), - create: jest.fn(), - findOne: jest.fn(), - find: jest.fn(), - delete: jest.fn(), - }, - // Add other properties that your code might use -} as unknown as DataSource; - -jest.mock('typeorm', () => { - const originalModule = jest.requireActual('typeorm'); - return { - ...originalModule, - DataSource: jest.fn().mockImplementation(() => mockDataSource), - }; -}); - -jest.mock('../../../src/api/services/MoocletDataService'); -jest.mock('../../../src/api/repositories/ExperimentRepository'); -jest.mock('../../../src/api/repositories/ExperimentConditionRepository'); -jest.mock('../../../src/api/repositories/DecisionPointRepository'); -jest.mock('../../../src/api/repositories/ExperimentAuditLogRepository'); -jest.mock('../../../src/api/repositories/SegmentRepository'); -jest.mock('../../../src/api/repositories/IndividualExclusionRepository'); -jest.mock('../../../src/api/repositories/GroupExclusionRepository'); -jest.mock('../../../src/api/repositories/MonitoredDecisionPointRepository'); -jest.mock('../../../src/api/repositories/ExperimentUserRepository'); -jest.mock('../../../src/api/repositories/MetricRepository'); -jest.mock('../../../src/api/repositories/QueryRepository'); -jest.mock('../../../src/api/repositories/StateTimeLogsRepository'); -jest.mock('../../../src/api/repositories/ExperimentSegmentInclusionRepository'); -jest.mock('../../../src/api/repositories/ExperimentSegmentExclusionRepository'); -jest.mock('../../../src/api/repositories/ConditionPayloadRepository'); -jest.mock('../../../src/api/repositories/FactorRepository'); -jest.mock('../../../src/api/repositories/LevelRepository'); -jest.mock('../../../src/api/repositories/ArchivedStatsRepository'); -jest.mock('../../../src/api/repositories/StratificationFactorRepository'); -jest.mock('../../../src/api/repositories/MoocletExperimentRefRepository'); -jest.mock('../../../src/api/services/PreviewUserService'); -jest.mock('../../../src/api/services/SegmentService'); -jest.mock('../../../src/api/services/ScheduledJobService'); -jest.mock('../../../src/api/services/ErrorService'); -jest.mock('../../../src/api/services/CacheService'); -jest.mock('../../../src/api/services/QueryService'); -jest.mock('../../../src/api/services/MetricService'); - -jest.mock('../../../src/env', () => ({ - env: { - mooclets: { - enabled: true, - }, - aws: { - region: 'us-east-1', - accessKeyId: 'test', - secretAccessKey: 'test', - s3BucketName: 'test', - }, - app: { - version: '1.0.0-test', - }, - }, -})); - -const mockTSConfigMoocletPolicyParameters = { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - prior: { - success: 1, - failure: 1, - }, - batch_size: 1, - max_rating: 1, - min_rating: 0, - uniform_threshold: 0, - tspostdiff_thresh: 0, - outcome_variable_name: 'TS_CONFIG_TEST', -}; - -const moocletExperimentDataTSConfigurable = { - id: 'test-exp-123', - name: 'test', - description: '', - consistencyRule: CONSISTENCY_RULE.INDIVIDUAL, - assignmentUnit: ASSIGNMENT_UNIT.INDIVIDUAL, - type: EXPERIMENT_TYPE.SIMPLE, - context: ['mathstream'], - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - stratificationFactor: null, - tags: [], - conditions: [ - { - id: 'A', - conditionCode: 'question-hint-default', - assignmentWeight: 50, - description: null, - order: 1, - name: '', - }, - { - id: 'B', - conditionCode: 'question-hint-tutorbot', - assignmentWeight: 50, - description: null, - order: 2, - name: '', - }, - ], - conditionPayloads: [ - { - id: 'E', - payload: { - type: PAYLOAD_TYPE.STRING, - value: 'question-hint-default', - }, - parentCondition: 'A', - decisionPoint: 'C', - }, - { - id: 'F', - payload: { - type: PAYLOAD_TYPE.STRING, - value: 'question-hint-tutorbot', - }, - parentCondition: 'B', - decisionPoint: 'C', - }, - ], - partitions: [ - { - id: 'C', - site: 'lesson-stream', - target: 'question-hint', - description: '', - order: 1, - excludeIfReached: false, - }, - ], - experimentSegmentInclusion: [ - { - segment: { - individualForSegment: [], - groupForSegment: [ - { - type: 'All', - groupId: 'All', - }, - ], - subSegments: [], - type: SEGMENT_TYPE.PRIVATE, - }, - }, - ], - experimentSegmentExclusion: [ - { - segment: { - individualForSegment: [], - groupForSegment: [], - subSegments: [], - type: SEGMENT_TYPE.PRIVATE, - }, - }, - ], - filterMode: FILTER_MODE.EXCLUDE_ALL, - queries: [], - endOn: null, - enrollmentCompleteCondition: null, - startOn: null, - state: EXPERIMENT_STATE.ENROLLING, - postExperimentRule: POST_EXPERIMENT_RULE.CONTINUE, - revertTo: null, - moocletPolicyParameters: mockTSConfigMoocletPolicyParameters, -}; - -describe('#MoocletExperimentService', () => { - let moocletExperimentService: MoocletExperimentService; - let moocletDataService: MoocletDataService; - let experimentRepository: ExperimentRepository; - let experimentConditionRepository: ExperimentConditionRepository; - let decisionPointRepository: DecisionPointRepository; - let experimentAuditLogRepository: ExperimentAuditLogRepository; - let segmentRepository: SegmentRepository; - let individualExclusionRepository: IndividualExclusionRepository; - let groupExclusionRepository: GroupExclusionRepository; - let monitoredDecisionPointRepository: MonitoredDecisionPointRepository; - let experimentUserRepository: ExperimentUserRepository; - let metricRepository: MetricRepository; - let queryRepository: QueryRepository; - let stateTimeLogsRepository: StateTimeLogsRepository; - let experimentSegmentInclusionRepository: ExperimentSegmentInclusionRepository; - let experimentSegmentExclusionRepository: ExperimentSegmentExclusionRepository; - let conditionPayloadRepository: ConditionPayloadRepository; - let factorRepository: FactorRepository; - let levelRepository: LevelRepository; - let levelCombinationElementsRepository: LevelCombinationElementRepository; - let archivedStatsRepository: ArchivedStatsRepository; - let stratificationRepository: StratificationFactorRepository; - let moocletExperimentRefRepository: MoocletExperimentRefRepository; - let previewUserService: PreviewUserService; - let segmentService: SegmentService; - let experimentSchedulerService: ExperimentSchedulerService; - let errorService: ErrorService; - let cacheService: CacheService; - let queryService: QueryService; - let metricService: MetricService; - let experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService; - - beforeEach(() => { - moocletDataService = { - deleteMooclet: jest.fn(), - deletePolicyParameters: jest.fn(), - deleteVariable: jest.fn(), - getPolicyParameters: jest.fn(), - getVersionForNewLearner: jest.fn(), - getVariable: jest.fn(), - } as unknown as MoocletDataService; - - metricService = { - saveAllMetrics: jest.fn(), - delete: jest.fn(), - } as unknown as MetricService; - experimentPrecomputedSegmentService = { - recomputeForExperiment: jest.fn().mockResolvedValue(undefined), - scheduleRecomputeForExperiments: jest.fn(), - scheduleRecomputeForSegment: jest.fn(), - getAffectedExperimentIds: jest.fn().mockResolvedValue([]), - getPrecomputedSets: jest.fn().mockResolvedValue(new Map()), - withRecompute: jest.fn(async (_logger, _resolve, work) => work()), - } as unknown as ExperimentPrecomputedSegmentService; - - cacheService = { - delCache: jest.fn().mockResolvedValue(undefined), - } as unknown as CacheService; - - experimentRepository = { - findOneExperiment: jest.fn(), - findOne: jest.fn(), - save: jest.fn(), - } as unknown as ExperimentRepository; - - moocletExperimentRefRepository = { - findOne: jest.fn(), - delete: jest.fn().mockResolvedValue(undefined), - } as unknown as MoocletExperimentRefRepository; - - // Create service with mocked dependencies - moocletExperimentService = new MoocletExperimentService( - moocletDataService, - experimentRepository, - experimentConditionRepository, - decisionPointRepository, - experimentAuditLogRepository, - individualExclusionRepository, - groupExclusionRepository, - monitoredDecisionPointRepository, - experimentUserRepository, - metricRepository, - queryRepository, - stateTimeLogsRepository, - experimentSegmentInclusionRepository, - experimentSegmentExclusionRepository, - conditionPayloadRepository, - factorRepository, - levelRepository, - levelCombinationElementsRepository, - archivedStatsRepository, - stratificationRepository, - segmentRepository, - moocletExperimentRefRepository, - mockDataSource, - previewUserService, - segmentService, - experimentSchedulerService, - errorService, - cacheService, - queryService, - metricService, - experimentPrecomputedSegmentService - ); - }); - - const logger = { - error: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - } as unknown as UpgradeLogger; - - describe('#handleCreateMoocletTransaction', () => { - let manager: EntityManager; - let params: SyncCreateParams; - let mockExperimentResponse: ExperimentDTO; - let mockMoocletExperimentRefResponse: MoocletExperimentRef; - - beforeEach(() => { - mockExperimentResponse = { - id: 'exp-123', - moocletPolicyParameters: mockTSConfigMoocletPolicyParameters, - } as any as ExperimentDTO; - - mockMoocletExperimentRefResponse = { - id: 'moocletRef-123', - versionConditionMaps: [], - } as MoocletExperimentRef; - - manager = mockDataSource.manager as EntityManager; - params = { - // Note: experimentDTO should already be the created experiment - // since handleCreateMoocletTransaction is now called AFTER createUpgradeExperiment - experimentDTO: mockExperimentResponse, - currentUser: {} as UserDTO, - logger, - }; - - // Spy on class methods - jest - .spyOn(moocletExperimentService as any, 'orchestrateMoocletCreation') - .mockResolvedValue(mockMoocletExperimentRefResponse); - jest.spyOn(moocletExperimentService as any, 'saveMoocletExperimentRef').mockResolvedValue(undefined); - jest - .spyOn(moocletExperimentService as any, 'createAndSaveVersionConditionMapEntities') - .mockResolvedValue(undefined); - jest.spyOn(moocletExperimentService as any, 'orchestrateDeleteMoocletResources').mockResolvedValue(undefined); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should successfully create a mooclet experiment with all required resources', async () => { - const result = await moocletExperimentService['handleCreateMoocletTransaction'](manager, params); - - // Verify orchestrateMoocletCreation was called with the experiment returned from createUpgradeExperiment - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - mockExperimentResponse, - params.experimentDTO.moocletPolicyParameters, - logger - ); - - expect(moocletExperimentService['saveMoocletExperimentRef']).toHaveBeenCalledWith( - manager, - mockMoocletExperimentRefResponse - ); - - expect(moocletExperimentService['createAndSaveVersionConditionMapEntities']).toHaveBeenCalledWith( - manager, - mockMoocletExperimentRefResponse.id, - mockMoocletExperimentRefResponse.versionConditionMaps, - logger - ); - - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - - // Verify the result - expect(result).toEqual({ - ...mockExperimentResponse, - moocletPolicyParameters: params.experimentDTO.moocletPolicyParameters, - }); - - // Verify orchestrateDeleteMoocletResources was not called - expect(moocletExperimentService['orchestrateDeleteMoocletResources']).not.toHaveBeenCalled(); - }); - - it('should use experimentDTO from params (not create new)', async () => { - // Verify that handleCreateMoocletTransaction uses the experimentDTO passed in params - // (it doesn't create a new experiment - that's done in syncCreate) - const result = await moocletExperimentService['handleCreateMoocletTransaction'](manager, params); - - // orchestrateMoocletCreation should be called with params.experimentDTO - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - params.experimentDTO, - params.experimentDTO.moocletPolicyParameters, - logger - ); - - expect(result.id).toBe(params.experimentDTO.id); - }); - - it('should handle failure during orchestrateMoocletCreation and throw error', async () => { - const error = new Error('Failed to create mooclet resources'); - jest.spyOn(moocletExperimentService as any, 'orchestrateMoocletCreation').mockRejectedValue(error); - - await expect(moocletExperimentService['handleCreateMoocletTransaction'](manager, params)).rejects.toThrow(error); - - // Verify subsequent methods were not called - expect(moocletExperimentService['saveMoocletExperimentRef']).not.toHaveBeenCalled(); - expect(moocletExperimentService['createAndSaveVersionConditionMapEntities']).not.toHaveBeenCalled(); - }); - - it('should handle failure during saveMoocletExperimentRef and cleanup resources', async () => { - const error = new Error('Failed to save mooclet ref'); - jest.spyOn(moocletExperimentService as any, 'saveMoocletExperimentRef').mockRejectedValue(error); - - await expect(moocletExperimentService['handleCreateMoocletTransaction'](manager, params)).rejects.toThrow(error); - - // Verify cleanup was called - expect(moocletExperimentService['orchestrateDeleteMoocletResources']).toHaveBeenCalledWith( - mockMoocletExperimentRefResponse, - logger - ); - }); - - it('should handle failure during createAndSaveVersionConditionMapEntities and cleanup resources', async () => { - const error = new Error('Failed to save version condition maps'); - jest.spyOn(moocletExperimentService as any, 'createAndSaveVersionConditionMapEntities').mockRejectedValue(error); - - await expect(moocletExperimentService['handleCreateMoocletTransaction'](manager, params)).rejects.toThrow(error); - - // Verify cleanup was called - expect(moocletExperimentService['orchestrateDeleteMoocletResources']).toHaveBeenCalledWith( - mockMoocletExperimentRefResponse, - logger - ); - }); - - it('should auto-generate outcome variable name for ts_configurable experiments', async () => { - // Create params for ts_configurable experiment with empty outcome_variable_name - const paramsWithTSConfigurable = { - ...params, - experimentDTO: { - ...params.experimentDTO, - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: { - ...mockTSConfigMoocletPolicyParameters, - outcome_variable_name: '', // Empty outcome variable name - }, - }, - }; - - // Spy on the private generateUniqueOutcomeVariableName method - const generateNameSpy = jest - .spyOn(moocletExperimentService as any, 'generateUniqueOutcomeVariableName') - .mockReturnValue('test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE'); - - await moocletExperimentService['handleCreateMoocletTransaction'](manager, paramsWithTSConfigurable); - - // Verify that generateUniqueOutcomeVariableName was called - expect(generateNameSpy).toHaveBeenCalledWith('Test Experiment'); - - // Verify that orchestrateMoocletCreation was called with the generated name - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - paramsWithTSConfigurable.experimentDTO, - expect.objectContaining({ - outcome_variable_name: 'test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE', - }), - logger - ); - }); - - it('should always generate new outcome variable name even when one exists for ts_configurable', async () => { - // Create params for ts_configurable experiment with existing outcome_variable_name - const paramsWithExistingOutcomeName = { - ...params, - experimentDTO: { - ...params.experimentDTO, - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: { - ...mockTSConfigMoocletPolicyParameters, - outcome_variable_name: 'existing_outcome_variable', - }, - }, - }; - - // Spy on the private generateUniqueOutcomeVariableName method - const generateNameSpy = jest - .spyOn(moocletExperimentService as any, 'generateUniqueOutcomeVariableName') - .mockReturnValue('test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE'); - - await moocletExperimentService['handleCreateMoocletTransaction'](manager, paramsWithExistingOutcomeName); - - // Verify that generateUniqueOutcomeVariableName was called (should always generate) - expect(generateNameSpy).toHaveBeenCalledWith('Test Experiment'); - - // Verify that orchestrateMoocletCreation was called with the NEW generated name (not existing) - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - paramsWithExistingOutcomeName.experimentDTO, - expect.objectContaining({ - outcome_variable_name: 'test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE', - }), - logger - ); - }); - - it('should not generate outcome variable name for non-ts_configurable experiments', async () => { - // Create params for random algorithm (non-ts_configurable) - const paramsWithRandomAlgorithm = { - ...params, - experimentDTO: { - ...params.experimentDTO, - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - moocletPolicyParameters: { - // Generic policy parameters without outcome_variable_name - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }, - }, - }; - - // Spy on the private generateUniqueOutcomeVariableName method - const generateNameSpy = jest.spyOn(moocletExperimentService as any, 'generateUniqueOutcomeVariableName'); - - await moocletExperimentService['handleCreateMoocletTransaction'](manager, paramsWithRandomAlgorithm); - - // Verify that generateUniqueOutcomeVariableName was NOT called - expect(generateNameSpy).not.toHaveBeenCalled(); - - // Verify that orchestrateMoocletCreation was called with original parameters - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - paramsWithRandomAlgorithm.experimentDTO, - paramsWithRandomAlgorithm.experimentDTO.moocletPolicyParameters, - logger - ); - }); - }); - - describe('#orchestrateMoocletCreation', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should orchestrate mooclet creation successfully', async () => { - const mockMoocletPolicy = { id: 'mockMoocletPolicy123' }; - const mockMoocletResponse = { id: 'mockMoocletResponse' }; - const mockMoocletVersionsResponse = [{ id: 'mockMoocletVersionsResponse' }]; - const mockMoocletPolicyParametersResponse = { id: 'mockMoocletPolicyParametersResponse' }; - const mockMoocletVariableResponse = { id: 'mockMoocletVariableResponse' }; - - jest.spyOn(moocletExperimentService as any, 'getMoocletPolicy').mockResolvedValue(mockMoocletPolicy); - jest.spyOn(moocletExperimentService as any, 'createMooclet').mockResolvedValue(mockMoocletResponse); - jest - .spyOn(moocletExperimentService as any, 'createMoocletVersions') - .mockResolvedValue(mockMoocletVersionsResponse); - jest.spyOn(moocletExperimentService as any, 'createMoocletVersionConditionMaps').mockReturnValue([]); - jest - .spyOn(moocletExperimentService as any, 'createPolicyParameters') - .mockResolvedValue(mockMoocletPolicyParametersResponse); - jest - .spyOn(moocletExperimentService as any, 'createVariableIfNeeded') - .mockResolvedValue(mockMoocletVariableResponse); - jest.spyOn(moocletExperimentService as any, 'orchestrateDeleteMoocletResources').mockResolvedValue(undefined); - - const result = await moocletExperimentService.orchestrateMoocletCreation( - moocletExperimentDataTSConfigurable, - mockTSConfigMoocletPolicyParameters, - logger - ); - - expect(result).toBeDefined(); - expect(result?.experimentId).toBe(moocletExperimentDataTSConfigurable.id); - expect(result?.moocletId).toBe(mockMoocletResponse.id); - expect(result?.policyParametersId).toBe(mockMoocletPolicyParametersResponse.id); - expect(result?.variableId).toBe(mockMoocletVariableResponse.id); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - }); - - it('should handle errors and orchestrate deletion of mooclet resources', async () => { - const mockError = new Error('Test Error'); - - jest.spyOn(moocletExperimentService as any, 'getMoocletPolicy').mockRejectedValue(mockError); - jest.spyOn(moocletExperimentService as any, 'orchestrateDeleteMoocletResources').mockRejectedValue(mockError); - - await expect( - moocletExperimentService.orchestrateMoocletCreation( - moocletExperimentDataTSConfigurable, - mockTSConfigMoocletPolicyParameters, - logger - ) - ).rejects.toThrow(mockError); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).toHaveBeenCalled(); - }); - }); - - describe('#orchestrateDeleteMoocletResources', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.policyParametersId = 2; - mockMoocletExperimentRef.variableId = 3; - mockMoocletExperimentRef.id = 'mockMoocletExperimentRef123'; - mockMoocletExperimentRef.versionConditionMaps = []; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should delete all mooclet resources successfully', async () => { - jest.spyOn(moocletDataService, 'deleteMooclet').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deletePolicyParameters').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deleteVariable').mockResolvedValue(undefined); - - await moocletExperimentService.orchestrateDeleteMoocletResources(mockMoocletExperimentRef, logger); - - expect(moocletDataService.deleteMooclet).toHaveBeenCalledWith(mockMoocletExperimentRef.moocletId, logger); - expect(moocletDataService.deletePolicyParameters).toHaveBeenCalledWith( - mockMoocletExperimentRef.policyParametersId, - logger - ); - expect(moocletDataService.deleteVariable).toHaveBeenCalledWith(mockMoocletExperimentRef.variableId, logger); - }); - it('should handle errors, log them, and return false', async () => { - const mockError = new Error('Failed to delete mooclet'); - jest.spyOn(moocletDataService, 'deleteMooclet').mockRejectedValue(mockError); - jest.spyOn(moocletExperimentService as any, 'deleteMoocletVersions').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deletePolicyParameters').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deleteVariable').mockResolvedValue(undefined); - - const result = await moocletExperimentService.orchestrateDeleteMoocletResources(mockMoocletExperimentRef, logger); - - expect(result).toBe(false); - expect(moocletDataService.deleteMooclet).toHaveBeenCalledWith(mockMoocletExperimentRef.moocletId, logger); - expect(logger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Failed to delete Mooclet resources'), - error: mockError, - moocletExperimentRef: mockMoocletExperimentRef, - }) - ); - }); - }); - - describe('#handleDeleteMoocletTransaction', () => { - const mockExperiment = { - id: 'test-exp-123', - name: 'Test Experiment', - conditions: [], - }; - - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.policyParametersId = 2; - mockMoocletExperimentRef.variableId = 3; - mockMoocletExperimentRef.id = 'mockMoocletExperimentRef123'; - mockMoocletExperimentRef.experimentId = 'test-exp-123'; - mockMoocletExperimentRef.versionConditionMaps = []; - - const mockUser = { - id: 'user-123', - firstName: 'Test', - lastName: 'User', - email: 'test@test.com', - } as UserDTO; - - const mockDeleteParams = { - moocletExperimentRef: mockMoocletExperimentRef, - experimentId: 'test-exp-123', - currentUser: mockUser, - logger, - }; - - const mockManager = mockDataSource.manager as EntityManager; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should rollback upgrade experiment deletion when mooclet deletion fails', async () => { - // Mock the parent class delete method - const deleteSpy = jest - .spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'delete') - .mockResolvedValue(mockExperiment); - - // Mock orchestrateDeleteMoocletResources to fail (return false) - const orchestrateSpy = jest - .spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources') - .mockResolvedValue(false); - - // Call the private method directly - await expect( - (moocletExperimentService as any).handleDeleteMoocletTransaction(mockManager, mockDeleteParams) - ).rejects.toThrow(/Failed to delete Mooclet resources, aborting transaction to preserve upgrade experiment/); - - // Verify that upgrade deletion was attempted first - expect(deleteSpy).toHaveBeenCalledWith('test-exp-123', mockUser, { - existingEntityManager: mockManager, - }); - - // Verify that mooclet deletion was attempted - expect(orchestrateSpy).toHaveBeenCalledWith(mockMoocletExperimentRef, logger); - }); - - it('should successfully delete both resources when mooclet deletion succeeds', async () => { - // Mock the parent class delete method - const deleteSpy = jest - .spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'delete') - .mockResolvedValue(mockExperiment); - - // Mock orchestrateDeleteMoocletResources to succeed (return true) - const orchestrateSpy = jest - .spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources') - .mockResolvedValue(true); - - // Call the private method directly - const result = await (moocletExperimentService as any).handleDeleteMoocletTransaction( - mockManager, - mockDeleteParams - ); - - expect(result).toEqual(mockExperiment); - expect(deleteSpy).toHaveBeenCalledWith('test-exp-123', mockUser, { - existingEntityManager: mockManager, - }); - expect(orchestrateSpy).toHaveBeenCalledWith(mockMoocletExperimentRef, logger); - }); - - it('should throw error if upgrade experiment deletion fails (before mooclet deletion)', async () => { - const deleteError = new Error('Database error: cannot delete experiment'); - - // Mock the parent class delete method to fail - const deleteSpy = jest - .spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'delete') - .mockRejectedValue(deleteError); - - // Mock orchestrateDeleteMoocletResources (should not be called) - const orchestrateSpy = jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources'); - - // Call the private method directly - await expect( - (moocletExperimentService as any).handleDeleteMoocletTransaction(mockManager, mockDeleteParams) - ).rejects.toThrow(deleteError); - - // Verify upgrade deletion was attempted - expect(deleteSpy).toHaveBeenCalledWith('test-exp-123', mockUser, { - existingEntityManager: mockManager, - }); - - // Mooclet deletion should not be attempted if upgrade deletion fails - expect(orchestrateSpy).not.toHaveBeenCalled(); - }); - }); - - describe('#generateUniqueOutcomeVariableName', () => { - it('should generate a unique outcome variable name with correct format', () => { - // Mock Date to have predictable timestamp - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('Test Experiment Name'); - - expect(result).toBe('test_exper_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - // Restore original Date - jest.restoreAllMocks(); - }); - - it('should handle long experiment names by truncating to 10 characters', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName( - 'This is a very long experiment name that should be truncated' - ); - - expect(result).toBe('this_is_a__2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - expect(result.substring(0, result.indexOf('_2026'))).toHaveLength(10); - - jest.restoreAllMocks(); - }); - - it('should sanitize special characters in experiment names', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('Test@#$%Exp!'); - - expect(result).toBe('test_exp_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - jest.restoreAllMocks(); - }); - - it('should handle experiment names with leading and trailing underscores', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('___Test___'); - - expect(result).toBe('test_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - jest.restoreAllMocks(); - }); - - it('should convert uppercase names to lowercase', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('UPPERCASE'); - - expect(result).toBe('uppercase_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - jest.restoreAllMocks(); - }); - }); - - describe('#handlePotentialMoocletAssignmentAlgorithmChange', () => { - const currentUser = { - id: 'user-123', - firstName: 'Test', - lastName: 'User', - email: 'test@example.com', - } as UserDTO; - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.id = 'mooclet-ref-123'; - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.experimentId = 'test-exp-123'; - - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('Algorithm changes involving mooclets', () => { - it('should handle Mooclet A → Mooclet B (fetch, update+create, delete) in INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - const updatedExperiment = { ...experiment, name: 'Updated' }; - - // Mock checkForMoocletAssignmentAlgorithmChange - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: true, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - // Mock getMoocletExperimentRefByUpgradeExperimentId (should be called to fetch old ref) - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Mock syncUpdateWithNewMoocletResources (should be called to update + create new) - jest - .spyOn(moocletExperimentService, 'syncUpdateWithMoocletAlgorithmTransition') - .mockResolvedValue(updatedExperiment); - - // Mock orchestrateDeleteMoocletResources (should be called AFTER update) - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources').mockResolvedValue(true); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - // Verify order of operations - expect(moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId).toHaveBeenCalledWith( - experiment.id - ); - expect(moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition).toHaveBeenCalledWith({ - experimentDTO: experiment, - currentUser, - logger, - moocletRefToDelete: mockMoocletExperimentRef, - }); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).toHaveBeenCalledWith( - mockMoocletExperimentRef, - logger - ); - - // Verify order: fetch called before update, update called before delete - const fetchOrder = (moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId as jest.Mock).mock - .invocationCallOrder[0]; - const updateOrder = (moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition as jest.Mock).mock - .invocationCallOrder[0]; - const deleteOrder = (moocletExperimentService.orchestrateDeleteMoocletResources as jest.Mock).mock - .invocationCallOrder[0]; - expect(fetchOrder).toBeLessThan(updateOrder); - expect(updateOrder).toBeLessThan(deleteOrder); - - expect(result).toEqual(updatedExperiment); - }); - - it('should handle Mooclet → Non-mooclet (fetch, update, delete) in INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const updatedExperiment = { ...experiment }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Directly mock the update method (simpler than mocking all internal dependencies) - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockResolvedValue(updatedExperiment); - - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources').mockResolvedValue(true); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId).toHaveBeenCalledWith( - experiment.id - ); - expect(updateSpy).toHaveBeenCalledWith(experiment, currentUser, logger, mockDataSource.manager); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).toHaveBeenCalledWith( - mockMoocletExperimentRef, - logger - ); - - expect(result).toEqual(updatedExperiment); - - updateSpy.mockRestore(); - }); - - it('should handle Non-mooclet → Mooclet (no fetch, update+create, no delete) in INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - const updatedExperiment = { ...experiment }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: false, - isNowMooclet: true, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - jest - .spyOn(moocletExperimentService, 'syncUpdateWithMoocletAlgorithmTransition') - .mockResolvedValue(updatedExperiment); - - jest.spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId'); - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources'); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - // Verify no fetch since wasMooclet is false - expect(moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId).not.toHaveBeenCalled(); - - expect(moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition).toHaveBeenCalledWith({ - experimentDTO: experiment, - currentUser, - logger, - moocletRefToDelete: undefined, - }); - - // Verify no delete since nothing to delete - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - - expect(result).toEqual(updatedExperiment); - }); - - it('should throw error for algorithm change in non-INACTIVE state (ENROLLING)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - await expect( - moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange(experiment, currentUser, logger) - ).rejects.toThrow(/INACTIVE state/); - }); - - it('should gracefully handle deletion failure (log but not throw)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const updatedExperiment = { ...experiment }; - const deletionError = new Error('Mooclet API unavailable'); - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Directly mock the update method - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockResolvedValue(updatedExperiment); - - // Mock deletion failure - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources').mockRejectedValue(deletionError); - - // Should not throw - deletion failure should be caught and logged - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(result).toEqual(updatedExperiment); - expect(logger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Failed to delete old mooclet resources'), - }) - ); - - updateSpy.mockRestore(); - }); - - it('should not delete if update fails (transaction safety)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const updateError = new Error('Database error'); - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Mock the update method to fail - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockRejectedValue(updateError); - - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources'); - - await expect( - moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange(experiment, currentUser, logger) - ).rejects.toThrow(updateError); - - // Verify deletion was NOT called since update failed - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - - updateSpy.mockRestore(); - }); - }); - - describe('Regular mooclet updates (no algorithm change)', () => { - it('should handle regular mooclet update in ENROLLING state (allowed)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - }; - const updatedExperiment = { ...experiment, name: 'Updated name' }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: false, - wasMooclet: true, - isNowMooclet: true, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest.spyOn(moocletExperimentService, 'syncUpdate').mockResolvedValue(updatedExperiment); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(moocletExperimentService.syncUpdate).toHaveBeenCalledWith({ - experimentDTO: experiment, - currentUser, - logger, - }); - expect(result).toEqual(updatedExperiment); - }); - }); - - describe('Non-mooclet updates', () => { - it('should throw error for non-mooclet algorithm changes in non-INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING, - }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: false, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - await expect( - moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange(experiment, currentUser, logger) - ).rejects.toThrow(/INACTIVE state/); - }); - - it('should return null for non-mooclet experiments (let controller handle)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: false, - wasMooclet: false, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(result).toBeNull(); - }); - }); - }); - - describe('#syncCreate', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should create experiment and mooclet resources in a transaction', async () => { - const experimentDTO = { ...moocletExperimentDataTSConfigurable } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncCreateParams = { experimentDTO, currentUser, logger }; - - const createdExperiment = { ...experimentDTO, id: 'new-exp-id' }; - jest.spyOn(moocletExperimentService as any, 'createUpgradeExperiment').mockResolvedValue(createdExperiment); - jest - .spyOn(moocletExperimentService as any, 'handleCreateMoocletTransaction') - .mockResolvedValue(createdExperiment); - - const result = await moocletExperimentService.syncCreate(params); - - expect(result).toEqual(createdExperiment); - }); - }); - - describe('#syncUpdate', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should update experiment and mooclet resources in a transaction', async () => { - const experimentDTO = { ...moocletExperimentDataTSConfigurable } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncEditParams = { experimentDTO, currentUser, logger }; - - const updatedExperiment = { ...experimentDTO, name: 'Updated' }; - jest.spyOn(moocletExperimentService as any, 'handleEditMoocletTransaction').mockResolvedValue(updatedExperiment); - - const result = await moocletExperimentService.syncUpdate(params); - - expect(result).toEqual(updatedExperiment); - }); - }); - - describe('#syncUpdateWithMoocletAlgorithmTransition', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should update experiment and create new mooclet resources when transitioning to mooclet', async () => { - const experimentDTO = { - ...moocletExperimentDataTSConfigurable, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncEditParams = { experimentDTO, currentUser, logger }; - - const updatedExperiment = { ...experimentDTO }; - jest.spyOn(moocletExperimentService as any, 'updateUpgradeExperiment').mockResolvedValue(updatedExperiment); - jest - .spyOn(moocletExperimentService as any, 'handleCreateMoocletTransaction') - .mockResolvedValue(updatedExperiment); - jest.spyOn(moocletExperimentService, 'isMoocletExperiment').mockReturnValue(true); - - const result = await moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition(params); - - expect(result).toEqual(updatedExperiment); - expect(moocletExperimentService['updateUpgradeExperiment']).toHaveBeenCalled(); - expect(moocletExperimentService['handleCreateMoocletTransaction']).toHaveBeenCalled(); - }); - - it('should update experiment and delete old mooclet ref when transitioning away from mooclet', async () => { - const mockMoocletRef = new MoocletExperimentRef(); - mockMoocletRef.id = 'ref-123'; - - const experimentDTO = { - ...moocletExperimentDataTSConfigurable, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncEditParams = { experimentDTO, currentUser, logger, moocletRefToDelete: mockMoocletRef }; - - const updatedExperiment = { ...experimentDTO }; - jest.spyOn(moocletExperimentService as any, 'updateUpgradeExperiment').mockResolvedValue(updatedExperiment); - jest.spyOn(moocletExperimentService, 'isMoocletExperiment').mockReturnValue(false); - jest.spyOn(moocletExperimentRefRepository, 'delete').mockResolvedValue(undefined); - - const result = await moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition(params); - - expect(result).toEqual(updatedExperiment); - expect(moocletExperimentRefRepository.delete).toHaveBeenCalledWith(mockMoocletRef.id); - }); - }); - - describe('#syncDelete', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should delete experiment and mooclet resources in a transaction', async () => { - const mockMoocletRef = new MoocletExperimentRef(); - mockMoocletRef.id = 'ref-123'; - - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncDeleteParams = { - moocletExperimentRef: mockMoocletRef, - experimentId: 'exp-123', - currentUser, - logger, - }; - - const deletedExperiment = { id: 'exp-123' } as any as Experiment; - jest - .spyOn(moocletExperimentService as any, 'handleDeleteMoocletTransaction') - .mockResolvedValue(deletedExperiment); - - const result = await moocletExperimentService.syncDelete(params); - - expect(result).toEqual(deletedExperiment); - }); - }); - - describe('#isMoocletExperiment', () => { - it('should return true for mooclet algorithms', () => { - expect(moocletExperimentService.isMoocletExperiment(ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE)).toBe(true); - }); - - it('should return false for non-mooclet algorithms', () => { - expect(moocletExperimentService.isMoocletExperiment(ASSIGNMENT_ALGORITHM.RANDOM)).toBe(false); - expect(moocletExperimentService.isMoocletExperiment(ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING)).toBe(false); - }); - }); - - describe('#checkForMoocletAssignmentAlgorithmChange', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should detect algorithm change from mooclet to non-mooclet', async () => { - const oldExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - const newExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - } as ExperimentDTO; - - jest.spyOn(moocletExperimentService, 'findOne').mockResolvedValue(oldExperiment as any); - - const result = await moocletExperimentService.checkForMoocletAssignmentAlgorithmChange(newExperiment, logger); - - expect(result).toEqual({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - }); - - it('should detect no change when algorithm stays the same', async () => { - const oldExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const newExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - } as ExperimentDTO; - - jest.spyOn(moocletExperimentService, 'findOne').mockResolvedValue(oldExperiment as any); - - const result = await moocletExperimentService.checkForMoocletAssignmentAlgorithmChange(newExperiment, logger); - - expect(result).toEqual({ - hasChanged: false, - wasMooclet: false, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - }); - - it('should throw error when experiment not found', async () => { - const newExperiment = { id: 'exp-123', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as ExperimentDTO; - - jest.spyOn(moocletExperimentService, 'findOne').mockResolvedValue(null); - - await expect( - moocletExperimentService.checkForMoocletAssignmentAlgorithmChange(newExperiment, logger) - ).rejects.toThrow(/Experiment unexpectedly not found/); - }); - }); - - describe('#attachPolicyParamsToExperimentDTO', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.policyParametersId = 123; - mockMoocletExperimentRef.versionConditionMaps = [ - { - moocletVersionId: 1, - experimentCondition: { conditionCode: 'control' } as any, - } as MoocletVersionConditionMap, - { - moocletVersionId: 2, - experimentCondition: { conditionCode: 'treatment' } as any, - } as MoocletVersionConditionMap, - ]; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should attach policy parameters to experiment DTO', async () => { - const experiment = { id: 'exp-123' } as ExperimentDTO; - const mockPolicyParams = { - parameters: { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - prior: { - '1': { success: 1, failure: 1 }, - '2': { success: 1, failure: 1 }, - }, - }, - }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getPolicyParameters').mockResolvedValue(mockPolicyParams as any); - - const result = await moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, logger); - - expect(result.moocletPolicyParameters).toEqual(mockPolicyParams.parameters); - }); - - it('should transform current_posteriors from version IDs to condition codes', async () => { - const experiment = { id: 'exp-123' } as ExperimentDTO; - const mockPolicyParams = { - parameters: { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - current_posteriors: { - '1': { success: 5, failure: 2 }, - '2': { success: 3, failure: 4 }, - }, - }, - }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getPolicyParameters').mockResolvedValue(mockPolicyParams as any); - - const result = await moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, logger); - - expect(result.moocletPolicyParameters['current_posteriors']).toEqual({ - control: { success: 5, failure: 2 }, - treatment: { success: 3, failure: 4 }, - }); - }); - - it('should throw error if policy parameters cannot be fetched', async () => { - const experiment = { id: 'exp-123' } as ExperimentDTO; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getPolicyParameters').mockRejectedValue(new Error('API error')); - - await expect(moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, logger)).rejects.toThrow( - /Failed to get Mooclet policy parameters/ - ); - }); - }); - - describe('#getConditionFromMoocletProxy', () => { - const mockExperiment = { id: 'exp-123' } as Experiment; - const mockUserId = 'user-456'; - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.versionConditionMaps = [ - { - moocletVersionId: 10, - experimentCondition: { id: 'cond-1', conditionCode: 'control' } as ExperimentCondition, - } as MoocletVersionConditionMap, - ]; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should get condition from mooclet proxy', async () => { - const mockVersionResponse = { id: 10, name: 'control' }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getVersionForNewLearner').mockResolvedValue(mockVersionResponse as any); - - const result = await moocletExperimentService.getConditionFromMoocletProxy(mockExperiment, mockUserId, logger); - - expect(result).toEqual(mockMoocletExperimentRef.versionConditionMaps[0].experimentCondition); - expect(moocletDataService.getVersionForNewLearner).toHaveBeenCalledWith( - mockMoocletExperimentRef.moocletId, - mockUserId, - logger - ); - }); - - it('should throw MoocletError if moocletExperimentRef is not found', async () => { - jest.spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId').mockResolvedValue(undefined); - - await expect( - moocletExperimentService.getConditionFromMoocletProxy(mockExperiment, mockUserId, logger) - ).rejects.toThrow(`MoocletExperimentRef not found for experiment id ${mockExperiment.id}`); - }); - - it('should throw error if version not found in maps', async () => { - const mockVersionResponse = { id: 999, name: 'unknown' }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getVersionForNewLearner').mockResolvedValue(mockVersionResponse as any); - - await expect( - moocletExperimentService.getConditionFromMoocletProxy(mockExperiment, mockUserId, logger) - ).rejects.toThrow(/Version ID not found in version condition maps/); - }); - }); - - describe('#handleEnrollCondition', () => { - const mockExperimentId = 'exp-123'; - const mockConditionCode = 'control'; - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.versionConditionMaps = [ - { - experimentCondition: { id: 'cond-1', conditionCode: 'control' } as ExperimentCondition, - } as MoocletVersionConditionMap, - ]; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should return experiment condition for valid condition code', async () => { - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - const result = await moocletExperimentService.handleEnrollCondition(mockExperimentId, mockConditionCode, logger); - - expect(result).toEqual(mockMoocletExperimentRef.versionConditionMaps[0].experimentCondition); - }); - - it('should throw error if mooclet experiment ref not found', async () => { - jest.spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId').mockResolvedValue(null); - - await expect( - moocletExperimentService.handleEnrollCondition(mockExperimentId, mockConditionCode, logger) - ).rejects.toThrow(/No MoocletExperimentRef found/); - }); - - it('should throw error if condition code not found in version maps', async () => { - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - await expect( - moocletExperimentService.handleEnrollCondition(mockExperimentId, 'unknown-condition', logger) - ).rejects.toThrow(/No version found for condition/); - }); - }); - - describe('Detection methods', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.versionConditionMaps = [ - { - experimentConditionId: 'cond-1', - experimentCondition: { id: 'cond-1', conditionCode: 'control' } as any as ExperimentCondition, - } as any as MoocletVersionConditionMap, - ]; - - describe('#detectNewConditions', () => { - it('should detect new conditions', () => { - const experimentDTO = { - conditions: [ - { id: 'cond-1', conditionCode: 'control' }, - { id: 'cond-2', conditionCode: 'treatment' }, - ], - } as ExperimentDTO; - - const result = moocletExperimentService['detectNewConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toHaveLength(1); - expect(result[0].id).toBe('cond-2'); - }); - - it('should return null if no new conditions', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectNewConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toBeNull(); - }); - }); - - describe('#detectRemovedConditions', () => { - it('should detect removed conditions', () => { - const experimentDTO = { - conditions: [], - } as ExperimentDTO; - - const result = moocletExperimentService['detectRemovedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toHaveLength(1); - expect(result[0].experimentConditionId).toBe('cond-1'); - }); - - it('should return null if no conditions removed', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectRemovedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toBeNull(); - }); - }); - - describe('#detectModifiedConditions', () => { - it('should detect modified condition codes', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'modified_control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectModifiedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toHaveLength(1); - expect(result[0].experimentConditionId).toBe('cond-1'); - }); - - it('should return null if no conditions modified', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectModifiedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toBeNull(); - }); - }); - - describe('#detectExperimentDesignChanges', () => { - it('should detect multiple changes', () => { - const experimentDTO = { - conditions: [ - { id: 'cond-1', conditionCode: 'modified_control' }, - { id: 'cond-2', conditionCode: 'treatment' }, - ], - } as any as ExperimentDTO; - - const result = moocletExperimentService['detectExperimentDesignChanges']( - experimentDTO, - mockMoocletExperimentRef - ); - - expect(result.addedConditions).toHaveLength(1); - expect(result.modifiedConditions).toHaveLength(1); - }); - - it('should return null if no changes detected', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as any as ExperimentDTO; - - const result = moocletExperimentService['detectExperimentDesignChanges']( - experimentDTO, - mockMoocletExperimentRef - ); - - expect(result).toBeNull(); - }); - }); - }); - - describe('#handleEditMoocletTransaction', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.id = 'ref-123'; - mockMoocletExperimentRef.policyParametersId = 1; - mockMoocletExperimentRef.variableId = 2; - mockMoocletExperimentRef.versionConditionMaps = [ - { - moocletVersionId: 10, - experimentCondition: { conditionCode: 'control' } as any, - } as MoocletVersionConditionMap, - { - moocletVersionId: 20, - experimentCondition: { conditionCode: 'treatment' } as any, - } as MoocletVersionConditionMap, - ]; - - const mockExperiment = { - id: 'exp-123', - state: EXPERIMENT_STATE.INACTIVE, - conditions: [], - moocletPolicyParameters: mockTSConfigMoocletPolicyParameters, - } as any as ExperimentDTO; - - const mockCurrentExperiment = { - id: 'exp-123', - conditions: [], - } as any as Experiment; - - const mockCurrentUser = { id: 'user-123' } as any as UserDTO; - const manager = mockDataSource.manager as EntityManager; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should throw error for ineligible experiment state', async () => { - const ineligibleExperiment = { ...mockExperiment, state: EXPERIMENT_STATE.CANCELLED }; - const params = { experimentDTO: ineligibleExperiment, currentUser: mockCurrentUser, logger }; - - await expect((moocletExperimentService as any).handleEditMoocletTransaction(manager, params)).rejects.toThrow( - /Ineligible experiment state/ - ); - }); - - it('should update experiment without version/variable edits when no changes detected', async () => { - const params = { experimentDTO: mockExperiment, currentUser: mockCurrentUser, logger }; - const updatedExperiment = { ...mockExperiment }; - - jest.spyOn(moocletExperimentService as any, 'fetchCurrentResources').mockResolvedValue({ - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentPolicyParametersResponse: { parameters: mockExperiment.moocletPolicyParameters }, - currentExperiment: mockCurrentExperiment, - }); - - jest.spyOn(moocletExperimentService as any, 'detectExperimentDesignChanges').mockReturnValue(null); - - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockResolvedValue(updatedExperiment); - - jest.spyOn(moocletExperimentService as any, 'doRevertablePolicyParameterChange').mockResolvedValue({ - parameters: { - ...mockExperiment.moocletPolicyParameters, - prior: { - '10': { success: 1, failure: 1 }, - '20': { success: 1, failure: 1 }, - }, - }, - }); - - const result = await (moocletExperimentService as any).handleEditMoocletTransaction(manager, params); - - expect(result).toEqual(updatedExperiment); - expect(updateSpy).toHaveBeenCalled(); - - updateSpy.mockRestore(); - }); - - it('should throw error when version/variable edits attempted on enrolling experiment', async () => { - const enrollingExperiment = { - ...mockExperiment, - state: EXPERIMENT_STATE.RUNNING, - }; - const params = { experimentDTO: enrollingExperiment, currentUser: mockCurrentUser, logger }; - - jest.spyOn(moocletExperimentService as any, 'fetchCurrentResources').mockResolvedValue({ - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentPolicyParametersResponse: { parameters: mockExperiment.moocletPolicyParameters }, - currentExperiment: mockCurrentExperiment, - }); - - jest.spyOn(moocletExperimentService as any, 'detectExperimentDesignChanges').mockReturnValue({ - addedConditions: [{ id: 'new-cond', conditionCode: 'new-condition' }], - removedConditions: null, - modifiedConditions: null, - }); - - await expect((moocletExperimentService as any).handleEditMoocletTransaction(manager, params)).rejects.toThrow( - /Ineligible version edits detected for an active Mooclet experiment/ - ); - }); - - it('should rollback on error', async () => { - const params = { experimentDTO: mockExperiment, currentUser: mockCurrentUser, logger }; - const mockError = new Error('Update failed'); - - jest.spyOn(moocletExperimentService as any, 'fetchCurrentResources').mockRejectedValue(mockError); - jest.spyOn(moocletExperimentService as any, 'rollbackMoocletEdits').mockResolvedValue(undefined); - - await expect((moocletExperimentService as any).handleEditMoocletTransaction(manager, params)).rejects.toThrow( - mockError - ); - - expect(moocletExperimentService['rollbackMoocletEdits']).toHaveBeenCalled(); - }); - }); - - describe('#rollbackMoocletEdits', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.id = 'ref-123'; - - const mockCurrentExperiment = { - id: 'exp-123', - conditions: [{ id: 'cond-1', conditionCode: 'old_code' }], - } as any as Experiment; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should rollback policy parameters', async () => { - const rollbackRef = { - revertPolicyParameters: mockTSConfigMoocletPolicyParameters, - restoreVersions: null, - revertVersionModifications: null, - removeVersions: null, - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentExperiment: mockCurrentExperiment, - }; - - jest.spyOn(moocletExperimentService as any, 'handleUpdatePolicyParameters').mockResolvedValue({}); - - await (moocletExperimentService as any).rollbackMoocletEdits(rollbackRef, logger); - - expect(moocletExperimentService['handleUpdatePolicyParameters']).toHaveBeenCalledWith( - mockTSConfigMoocletPolicyParameters, - mockMoocletExperimentRef, - logger - ); - }); - - it('should throw error if rollback itself fails', async () => { - const rollbackRef = { - revertPolicyParameters: mockTSConfigMoocletPolicyParameters, - restoreVersions: null, - revertVersionModifications: null, - removeVersions: null, - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentExperiment: mockCurrentExperiment, - }; - - const rollbackError = new Error('Rollback failed'); - jest.spyOn(moocletExperimentService as any, 'handleUpdatePolicyParameters').mockRejectedValue(rollbackError); - - await expect((moocletExperimentService as any).rollbackMoocletEdits(rollbackRef, logger)).rejects.toThrow( - /Error during rollback/ - ); - }); - }); -}); diff --git a/packages/backend/test/unit/services/MoocletRewardsService.test.ts b/packages/backend/test/unit/services/MoocletRewardsService.test.ts deleted file mode 100644 index 0fb7106934..0000000000 --- a/packages/backend/test/unit/services/MoocletRewardsService.test.ts +++ /dev/null @@ -1,1345 +0,0 @@ -import { MoocletRewardsService } from '../../../src/api/services/MoocletRewardsService'; -import { MoocletExperimentRefRepository } from '../../../src/api/repositories/MoocletExperimentRefRepository'; -import { IndividualEnrollmentRepository } from '../../../src/api/repositories/IndividualEnrollmentRepository'; -import { MoocletDataService } from '../../../src/api/services/MoocletDataService'; -import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; -import { MoocletExperimentRef } from '../../../src/api/models/MoocletExperimentRef'; -import { IndividualEnrollment } from '../../../src/api/models/IndividualEnrollment'; -import { Experiment } from '../../../src/api/models/Experiment'; -import { EXPERIMENT_STATE } from 'upgrade_types'; -import { BinaryRewardAllowedValue } from 'upgrade_types'; -import { RewardValidator } from '../../../src/api/controllers/validators/RewardValidator'; -import { RequestedExperimentUser } from '../../../src/api/controllers/validators/ExperimentUserValidator'; -import { HttpError } from 'routing-controllers'; -import { configureLogger } from '../../utils/logger'; -import { MoocletExperimentService } from 'src/api/services/MoocletExperimentService'; - -describe('MoocletRewardsService', () => { - let service: MoocletRewardsService; - let mockLogger: jest.Mocked; - let mockMoocletDataService: jest.Mocked; - let mockMoocletExperimentRefRepository: jest.Mocked; - let mockIndividualEnrollmentRepository: jest.Mocked; - let mockMoocletExperimentService: jest.Mocked; - - const mockUser = { - id: 'user-123', - group: {}, - workingGroup: {}, - } as RequestedExperimentUser; - - const mockExperiment: Partial = { - id: 'experiment-123', - state: EXPERIMENT_STATE.ENROLLING, - }; - - const mockMoocletExperimentRef: Partial = { - id: 'ref-123', - experimentId: 'experiment-123', - moocletId: 456, - outcomeVariableName: 'reward_variable', - experiment: mockExperiment as Experiment, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - } as any, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - } as any, - ], - }; - - const mockEnrollment: Partial = { - id: 'enrollment-123', - userId: 'user-123', - experimentId: 'experiment-123', - conditionId: 'condition-1', - }; - - beforeAll(() => { - configureLogger(); - }); - - beforeEach(() => { - mockLogger = { - info: jest.fn(), - error: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - } as any; - - mockMoocletDataService = { - postNewReward: jest.fn(), - } as any; - - mockMoocletExperimentRefRepository = { - findOne: jest.fn(), - findActivelyEnrollingMoocletExperimentsByContextSiteTarget: jest.fn(), - } as any; - - mockIndividualEnrollmentRepository = { - findEnrollments: jest.fn(), - } as any; - - mockMoocletExperimentService = { - getMoocletExperimentRefByUpgradeExperimentId: jest.fn(), - } as any; - - service = new MoocletRewardsService( - mockMoocletExperimentRefRepository as any, - mockIndividualEnrollmentRepository as any, - mockMoocletDataService as any, - mockMoocletExperimentService as any - ); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('sendReward', () => { - describe('successful reward sending', () => { - it('should successfully send reward when all criteria are met using experimentId', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.message).toBe('Reward sent to mooclet successfully.'); - expect(result.reward).toEqual({ - variable: 'reward_variable', - value: 1, - mooclet: 456, - version: 100, - learner: 'user-123', - }); - expect(mockMoocletDataService.postNewReward).toHaveBeenCalledTimes(1); - expect(mockLogger.info).toHaveBeenCalledTimes(1); - }); - - it('should successfully send reward using decision point when experimentId not provided', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.FAILURE, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [mockMoocletExperimentRef as MoocletExperimentRef] - ); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.message).toBe('Reward sent to mooclet successfully.'); - expect(result.reward.value).toBe(0); // FAILURE maps to 0 - expect(mockMoocletDataService.postNewReward).toHaveBeenCalledTimes(1); - }); - - it('should call postNewReward without awaiting (fire-and-forget)', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await service.sendReward(mockUser, request, mockLogger as any); - - // Verify postNewReward was called but not awaited - expect(mockMoocletDataService.postNewReward).toHaveBeenCalledTimes(1); - }); - - it('should map SUCCESS reward value to 1', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.value).toBe(1); - }); - - it('should map FAILURE reward value to 0', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.FAILURE, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.value).toBe(0); - }); - }); - - describe('mooclet experiment ref validation', () => { - it('should throw 409 when no mooclet ref found by experimentId', async () => { - const request: RewardValidator = { - experimentId: 'nonexistent-experiment', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(null); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('No active mooclet experiment ref found'); - } - }); - - it('should throw 409 when no mooclet ref found by decision point', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [] - ); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('No active experiment found for decision point'); - } - }); - - it('should throw 409 when multiple mooclet refs found by decision point', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [ - mockMoocletExperimentRef as MoocletExperimentRef, - { ...mockMoocletExperimentRef, id: 'ref-456' } as MoocletExperimentRef, - ] - ); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Multiple active experiments found for decision point'); - } - }); - - it('should throw 409 when experiment is not in ENROLLING state', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const nonEnrollingRef = { - ...mockMoocletExperimentRef, - experiment: { - ...mockExperiment, - state: EXPERIMENT_STATE.ENROLLMENT_COMPLETE, - }, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(nonEnrollingRef as MoocletExperimentRef); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('not actively enrolling'); - } - }); - }); - - describe('user enrollment validation', () => { - it('should throw 409 when no enrollment found for user', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as Error).message).toContain('Could not find unique user enrollment'); - } - }); - - it('should throw 409 when multiple enrollments found for user', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - mockEnrollment as IndividualEnrollment, - { ...mockEnrollment, id: 'enrollment-456' } as IndividualEnrollment, - ]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Could not find unique user enrollment'); - } - }); - - it('should succeed when exactly one enrollment found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.message).toBe('Reward sent to mooclet successfully.'); - }); - }); - - describe('version mapping validation', () => { - it('should throw 409 when no version mapping found for enrolled condition', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const enrollmentWithUnmappedCondition = { - ...mockEnrollment, - conditionId: 'unmapped-condition', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - enrollmentWithUnmappedCondition as IndividualEnrollment, - ]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Version-condition mapping not found'); - } - }); - - it('should find correct versionId for enrolled condition', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const enrollmentCondition2 = { - ...mockEnrollment, - conditionId: 'condition-2', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - enrollmentCondition2 as IndividualEnrollment, - ]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.version).toBe(200); // version for condition-2 - }); - }); - - describe('error handling', () => { - it('should wrap unexpected errors as 409 HttpError', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockRejectedValue(new Error('Database connection failed')); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Failed to process reward request due to unexpected error'); - } - }); - - it('should re-throw HttpErrors without wrapping', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const originalError = new HttpError(404, 'Experiment not found'); - mockMoocletExperimentRefRepository.findOne.mockRejectedValue(originalError); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(originalError); - }); - - it('should log errors with appropriate context', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(null); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect(mockLogger.error).toHaveBeenCalledTimes(1); - expect(mockLogger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.any(String), - request: expect.any(Object), - }) - ); - } - }); - }); - - describe('reward payload construction', () => { - it('should construct reward with all required fields', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward).toHaveProperty('variable'); - expect(result.reward).toHaveProperty('value'); - expect(result.reward).toHaveProperty('mooclet'); - expect(result.reward).toHaveProperty('version'); - expect(result.reward).toHaveProperty('learner'); - }); - - it('should use outcomeVariableName from mooclet ref', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.variable).toBe('reward_variable'); - }); - - it('should use moocletId from mooclet ref', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.mooclet).toBe(456); - }); - - it('should use user id as learner', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.learner).toBe('user-123'); - }); - }); - }); - - describe('getRewardsSummaryForExperiment', () => { - const mockMoocletRewardsResponse = { - count: 10, - next: null, - previous: null, - results: [ - { id: '1', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-1', mooclet: 456 }, - { id: '2', version: 100, value: 0.0, variable: 'reward_variable', learner: 'user-2', mooclet: 456 }, - { id: '3', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-3', mooclet: 456 }, - { id: '4', version: 200, value: 1.0, variable: 'reward_variable', learner: 'user-4', mooclet: 456 }, - { id: '5', version: 200, value: 0.0, variable: 'reward_variable', learner: 'user-5', mooclet: 456 }, - ], - }; - - beforeEach(() => { - mockMoocletDataService.getRewardsForExperiment = jest.fn(); - }); - - it('should successfully fetch and return rewards summary', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue(mockMoocletRewardsResponse as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(result).toHaveLength(2); - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 1, - failures: 1, - successRate: '50.0%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should handle experiment with no rewards', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue({ - count: 0, - next: null, - previous: null, - results: [], - } as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 0, - failures: 0, - successRate: '0.0%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should fetch all pages and accumulate results when response has multiple pages', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const page1Response = { - count: 6, - next: 'http://mooclet-api/value?page=2', - previous: null, - results: [ - { id: '1', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-1', mooclet: 456 }, - { id: '2', version: 100, value: 0.0, variable: 'reward_variable', learner: 'user-2', mooclet: 456 }, - { id: '3', version: 200, value: 1.0, variable: 'reward_variable', learner: 'user-3', mooclet: 456 }, - ], - }; - - const page2Response = { - count: 6, - next: null, - previous: 'http://mooclet-api/value?page=1', - results: [ - { id: '4', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-4', mooclet: 456 }, - { id: '5', version: 200, value: 0.0, variable: 'reward_variable', learner: 'user-5', mooclet: 456 }, - { id: '6', version: 200, value: 1.0, variable: 'reward_variable', learner: 'user-6', mooclet: 456 }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce(page1Response as any) - .mockResolvedValueOnce(page2Response as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledTimes(2); - // Control (version 100): page1 has id1 (success) + id2 (failure), page2 has id4 (success) => 2 successes, 1 failure - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - // Treatment (version 200): page1 has id3 (success), page2 has id5 (failure) + id6 (success) => 2 successes, 1 failure - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should pass next page URL to subsequent fetches', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const nextPageUrl = 'http://mooclet-api/value?page=2'; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce({ count: 2, next: nextPageUrl, previous: null, results: [] } as any) - .mockResolvedValueOnce({ count: 2, next: null, previous: null, results: [] } as any); - - await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenNthCalledWith( - 2, - expect.anything(), - expect.anything(), - nextPageUrl - ); - }); - - it('should log info for each additional page fetched', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce({ - count: 4, - next: 'http://mooclet-api/value?page=2', - previous: null, - results: [{ id: '1', version: 100, value: 1.0 }], - } as any) - .mockResolvedValueOnce({ count: 4, next: null, previous: null, results: [] } as any); - - await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - // Once for initial fetch, once for the paginated fetch - expect(mockLogger.info).toHaveBeenCalledTimes(2); - expect(mockLogger.info).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining("But wait there's more"), - }) - ); - }); - - it('should handle three or more pages of results', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce({ - count: 3, - next: 'http://mooclet-api/value?page=2', - previous: null, - results: [{ id: '1', version: 100, value: 1.0 }], - } as any) - .mockResolvedValueOnce({ - count: 3, - next: 'http://mooclet-api/value?page=3', - previous: null, - results: [{ id: '2', version: 100, value: 0.0 }], - } as any) - .mockResolvedValueOnce({ - count: 3, - next: null, - previous: null, - results: [{ id: '3', version: 100, value: 1.0 }], - } as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledTimes(3); - expect(result[0].successes).toBe(2); - expect(result[0].failures).toBe(1); - expect(result[0].successes + result[0].failures).toBe(3); - }); - - it('should log error and re-throw when mooclet service fails', async () => { - const error = new Error('Mooclet API error'); - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockRejectedValue(error); - - await expect(service.getRewardsSummaryForExperiment('experiment-123', mockLogger)).rejects.toThrow(error); - - expect(mockLogger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Error fetching rewards summary for experiment', - experimentId: 'experiment-123', - error, - }) - ); - }); - - it('should sort results by condition order', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue(mockMoocletRewardsResponse as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(result[0].conditionCode).toBe('Control'); - expect(result[0].order).toBe(0); - expect(result[1].conditionCode).toBe('Treatment'); - expect(result[1].order).toBe(1); - }); - }); - - describe('fetchRewardsForExperiment', () => { - beforeEach(() => { - mockMoocletDataService.getRewardsForExperiment = jest.fn(); - }); - - it('should call mooclet data service with correct parameters', async () => { - const moocletRef = { - moocletId: 456, - outcomeVariableName: 'test_variable', - } as MoocletExperimentRef; - - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue({ - count: 5, - next: null, - previous: null, - results: [], - } as any); - - await service.fetchRewardsForExperiment(moocletRef, mockLogger as any); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledWith( - { - moocletId: 456, - variableName: 'test_variable', - }, - mockLogger, - undefined - ); - }); - - it('should return paginated response from mooclet data service', async () => { - const moocletRef = { - moocletId: 456, - outcomeVariableName: 'test_variable', - } as MoocletExperimentRef; - - const expectedResponse = { - count: 3, - next: 'http://next-page', - previous: null, - results: [{ id: '1', value: 1.0 }], - }; - - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue(expectedResponse as any); - - const result = await service.fetchRewardsForExperiment(moocletRef, mockLogger as any); - - expect(result).toEqual(expectedResponse); - }); - - it('should pass nextPageUrl to mooclet data service when provided', async () => { - const moocletRef = { - moocletId: 456, - outcomeVariableName: 'test_variable', - } as MoocletExperimentRef; - - const nextPageUrl = 'http://mooclet-api/value?page=2'; - - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue({ - count: 5, - next: null, - previous: 'http://mooclet-api/value?page=1', - results: [{ id: '2', value: 1.0 }], - } as any); - - await service.fetchRewardsForExperiment(moocletRef, mockLogger as any, nextPageUrl); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledWith( - { - moocletId: 456, - variableName: 'test_variable', - }, - mockLogger, - nextPageUrl - ); - }); - }); - - describe('createExperimentRewardsSummary', () => { - it('should calculate success rate correctly for each condition', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 100, value: 1.0 }, - { id: '3', version: 100, value: 0.0 }, - { id: '4', version: 200, value: 1.0 }, - { id: '5', version: 200, value: 0.0 }, - { id: '6', version: 200, value: 0.0 }, - { id: '7', version: 200, value: 0.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 1, - failures: 3, - successRate: '25.0%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should handle 100% success rate', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 100, value: 1.0 }, - { id: '3', version: 100, value: 1.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[0].successRate).toBe('100.0%'); - }); - - it('should handle 0% success rate', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 0.0 }, - { id: '2', version: 100, value: 0.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[0].successRate).toBe('0.0%'); - }); - - it('should return empty array when results is undefined', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - undefined as any, - mockLogger as any - ); - - expect(result).toEqual([]); - expect(mockLogger.warn).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'No rewards data returned from Mooclet API', - experimentId: 'exp-123', - }) - ); - }); - - it('should filter rewards by version ID correctly', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 999, value: 1.0 }, // Different version, should be ignored - { id: '3', version: 100, value: 0.0 }, - { id: '4', version: 200, value: 1.0 }, - { id: '5', version: 200, value: 0.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - // Version 999 should not be counted - expect(result[0].successes + result[0].failures).toBe(2); // Only version 100 - expect(result[1].successes + result[1].failures).toBe(2); // Only version 200 - }); - - it('should handle condition with no rewards', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 100, value: 0.0 }, - // No rewards for version 200 - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 0, - failures: 0, - successRate: '0.0%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - }); - - describe('private helper methods', () => { - describe('findMoocletExperimentRefById', () => { - it('should return mooclet ref when found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(mockMoocletExperimentRefRepository.findOne).toHaveBeenCalledTimes(1); - expect(result.message).toBe('Reward sent to mooclet successfully.'); - }); - - it('should query with correct relations', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await service.sendReward(mockUser, request, mockLogger as any); - - expect(mockMoocletExperimentRefRepository.findOne).toHaveBeenCalledWith( - expect.objectContaining({ - where: expect.objectContaining({ experimentId: 'experiment-123' }), - relations: expect.objectContaining({ - versionConditionMaps: expect.anything(), - experiment: expect.anything(), - }), - }) - ); - }); - }); - - describe('findMoocletExperimentRefByDecisionPoint', () => { - it('should use context, site, and target to find experiment', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [mockMoocletExperimentRef as MoocletExperimentRef] - ); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await service.sendReward(mockUser, request, mockLogger as any); - - expect( - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget - ).toHaveBeenCalledTimes(1); - expect( - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget - ).toHaveBeenCalledWith('home', 'site-1', 'target-1'); - }); - }); - - describe('getVersionIdByConditionId', () => { - it('should return mooclet version ID when matching condition is found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.version).toBe(100); // version for condition-1 - }); - - it('should throw 409 when no matching condition found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const unmappedEnrollment = { - ...mockEnrollment, - conditionId: 'unmapped-condition', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - unmappedEnrollment as IndividualEnrollment, - ]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - } - }); - - it('should correctly match when multiple maps exist', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const refWithMultipleMaps = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { experimentConditionId: 'condition-1', moocletVersionId: 100 }, - { experimentConditionId: 'condition-2', moocletVersionId: 200 }, - { experimentConditionId: 'condition-3', moocletVersionId: 300 }, - ], - }; - - const enrollmentCondition3 = { - ...mockEnrollment, - conditionId: 'condition-3', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(refWithMultipleMaps as any); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - enrollmentCondition3 as IndividualEnrollment, - ]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.version).toBe(300); - }); - - it('should throw 409 when versionConditionMaps is empty', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const refWithEmptyMaps = { - ...mockMoocletExperimentRef, - versionConditionMaps: [], - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(refWithEmptyMaps as any); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - } - }); - }); - }); -}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts new file mode 100644 index 0000000000..c2aaca0c8b --- /dev/null +++ b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts @@ -0,0 +1,296 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { + ThompsonSamplingService, + ConditionRewardSummary, + ThompsonSamplingConfig, + DEFAULT_PRIOR, +} from '../../../src/api/services/ThompsonSamplingService'; +import { configureLogger } from '../../utils/logger'; + +describe('ThompsonSamplingService', () => { + let service: ThompsonSamplingService; + + beforeAll(() => { + configureLogger(); + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ThompsonSamplingService], + }).compile(); + + service = module.get(ThompsonSamplingService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('selectCondition', () => { + it('throws when given an empty condition list', () => { + expect(() => service.selectCondition([], [], 0)).toThrow(); + }); + + it('returns the only condition when list has one entry', () => { + expect(service.selectCondition(['A'], [], 10)).toBe('A'); + }); + + it('always returns a valid condition code', () => { + const conditions = ['A', 'B', 'C']; + for (let i = 0; i < 30; i++) { + expect(conditions).toContain(service.selectCondition(conditions, [], 100)); + } + }); + + describe('warmup phase', () => { + it('uses uniform random during warmup — both conditions appear across 50 draws', () => { + const conditions = ['A', 'B']; + // A has an overwhelmingly dominant posterior; without warmup it would always win + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionCode: 'A', successCount: 1000, totalCount: 1000 }, + { conditionCode: 'B', successCount: 0, totalCount: 1000 }, + ]; + const config: ThompsonSamplingConfig = { warmupThreshold: 50 }; + + const results = new Set(); + for (let i = 0; i < 50; i++) { + results.add(service.selectCondition(conditions, rewardSummaries, 5, config)); + } + expect(results.has('A')).toBe(true); + expect(results.has('B')).toBe(true); + }); + + it('exits warmup once enrollment exceeds threshold', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionCode: 'A', successCount: 1000, totalCount: 1000 }, + { conditionCode: 'B', successCount: 0, totalCount: 1000 }, + ]; + const config: ThompsonSamplingConfig = { warmupThreshold: 10 }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, 100, config) === 'A') { + aCount++; + } + } + // After warmup, A's dominant posterior should win nearly every draw + expect(aCount / runs).toBeGreaterThan(0.95); + }); + }); + + describe('Thompson Sampling selection', () => { + it('selects the condition with a better reward history reliably', () => { + const conditions = ['good', 'bad']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionCode: 'good', successCount: 90, totalCount: 100 }, + { conditionCode: 'bad', successCount: 10, totalCount: 100 }, + ]; + + let goodCount = 0; + const runs = 500; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs) === 'good') { + goodCount++; + } + } + expect(goodCount / runs).toBeGreaterThan(0.9); + }); + + it('handles three or more conditions — clear winner dominates', () => { + const conditions = ['A', 'B', 'C', 'D']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionCode: 'A', successCount: 5, totalCount: 100 }, + { conditionCode: 'B', successCount: 90, totalCount: 100 }, + { conditionCode: 'C', successCount: 10, totalCount: 100 }, + { conditionCode: 'D', successCount: 5, totalCount: 100 }, + ]; + + let bCount = 0; + const runs = 500; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs) === 'B') { + bCount++; + } + } + expect(bCount / runs).toBeGreaterThan(0.8); + }); + + it('respects strong priors when no rewards have been collected', () => { + const conditions = ['A', 'B']; + const config: ThompsonSamplingConfig = { + priors: { + A: { success: 100, failure: 1 }, + B: { success: 1, failure: 100 }, + }, + }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, [], 0, config) === 'A') { + aCount++; + } + } + expect(aCount / runs).toBeGreaterThan(0.95); + }); + + it('uses DEFAULT_PRIOR for conditions missing a prior entry', () => { + const conditions = ['A', 'B']; + const config: ThompsonSamplingConfig = { + priors: { + A: { success: 100, failure: 1 }, // strong prior for A + // B intentionally absent — should use DEFAULT_PRIOR Beta(1,1) + }, + }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, [], 0, config) === 'A') { + aCount++; + } + } + expect(aCount / runs).toBeGreaterThan(0.9); + }); + + it('handles a condition with no reward summary entry (defaults to prior)', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionCode: 'A', successCount: 50, totalCount: 100 }, + // B has no entry — treated as zero rewards, uses prior only + ]; + + for (let i = 0; i < 20; i++) { + expect(conditions).toContain(service.selectCondition(conditions, rewardSummaries, 200)); + } + }); + }); + + describe('minimumDrawDifference', () => { + it('falls back to uniform when threshold is larger than any possible draw difference', () => { + const conditions = ['A', 'B']; + // Threshold of 2 always triggers since Beta draws are in [0,1] + const config: ThompsonSamplingConfig = { minimumDrawDifference: 2 }; + + const results = new Set(); + for (let i = 0; i < 50; i++) { + results.add(service.selectCondition(conditions, [], 100, config)); + } + expect(results.has('A')).toBe(true); + expect(results.has('B')).toBe(true); + }); + + it('does not interfere when threshold is zero', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionCode: 'A', successCount: 90, totalCount: 100 }, + { conditionCode: 'B', successCount: 10, totalCount: 100 }, + ]; + const config: ThompsonSamplingConfig = { minimumDrawDifference: 0 }; + + let aCount = 0; + const runs = 200; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs, config) === 'A') { + aCount++; + } + } + expect(aCount / runs).toBeGreaterThan(0.85); + }); + }); + }); + + describe('estimateConditionWeights', () => { + it('returns empty object for empty conditions', () => { + expect(service.estimateConditionWeights([])).toEqual({}); + }); + + it('returns 100 for a single condition', () => { + expect(service.estimateConditionWeights([{ code: 'A', alpha: 1, beta: 1 }])).toEqual({ A: 100 }); + }); + + it('weights always sum to exactly 100', () => { + const conditions = [ + { code: 'A', alpha: 5, beta: 3 }, + { code: 'B', alpha: 2, beta: 8 }, + { code: 'C', alpha: 10, beta: 1 }, + ]; + const weights = service.estimateConditionWeights(conditions); + const total = Object.values(weights).reduce((sum, w) => sum + w, 0); + expect(total).toBe(100); + }); + + it('all weights are non-negative integers', () => { + const conditions = [ + { code: 'A', alpha: 3, beta: 3 }, + { code: 'B', alpha: 3, beta: 3 }, + { code: 'C', alpha: 3, beta: 3 }, + { code: 'D', alpha: 3, beta: 3 }, + ]; + const weights = service.estimateConditionWeights(conditions); + for (const w of Object.values(weights)) { + expect(w).toBeGreaterThanOrEqual(0); + expect(Number.isInteger(w)).toBe(true); + } + }); + + it('two equal Beta(1,1) arms split near 50/50', () => { + const conditions = [ + { code: 'A', alpha: 1, beta: 1 }, + { code: 'B', alpha: 1, beta: 1 }, + ]; + const weights = service.estimateConditionWeights(conditions); + // With 10k draws and σ≈0.5%, ±10% is ~20σ — essentially never flaky + expect(weights['A']).toBeGreaterThanOrEqual(40); + expect(weights['A']).toBeLessThanOrEqual(60); + expect(weights['B']).toBeGreaterThanOrEqual(40); + expect(weights['B']).toBeLessThanOrEqual(60); + }); + + it('four equal arms each get roughly 25%', () => { + const conditions = ['A', 'B', 'C', 'D'].map((code) => ({ code, alpha: 2, beta: 2 })); + const weights = service.estimateConditionWeights(conditions); + for (const w of Object.values(weights)) { + expect(w).toBeGreaterThanOrEqual(15); + expect(w).toBeLessThanOrEqual(35); + } + }); + + it('dominant condition (90 successes / 10 failures) captures most weight', () => { + const conditions = [ + { code: 'winner', alpha: 1 + 90, beta: 1 + 10 }, + { code: 'loser', alpha: 1 + 10, beta: 1 + 90 }, + ]; + const weights = service.estimateConditionWeights(conditions); + expect(weights['winner']).toBeGreaterThanOrEqual(90); + expect(weights['loser']).toBeLessThanOrEqual(10); + }); + + it('strong prior with no reward data drives weight toward the favored arm', () => { + const conditions = [ + { code: 'favored', alpha: 100, beta: 1 }, + { code: 'weak', alpha: 1, beta: 100 }, + ]; + const weights = service.estimateConditionWeights(conditions); + expect(weights['favored']).toBeGreaterThanOrEqual(90); + }); + + it('respects numDraws parameter — custom draw count still sums to 100', () => { + const conditions = [ + { code: 'A', alpha: 3, beta: 2 }, + { code: 'B', alpha: 2, beta: 3 }, + ]; + const weights = service.estimateConditionWeights(conditions, 500); + const total = Object.values(weights).reduce((sum, w) => sum + w, 0); + expect(total).toBe(100); + }); + }); + + describe('DEFAULT_PRIOR', () => { + it('is Beta(1,1) — the uninformative prior', () => { + expect(DEFAULT_PRIOR).toEqual({ success: 1, failure: 1 }); + }); + }); +}); diff --git a/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts b/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts index 06113fe0d0..ce9a4efd48 100644 --- a/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts +++ b/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts @@ -69,6 +69,6 @@ export const API_ENDPOINTS: APIEndpoints = { exportAllExperimentIncludeLists: '/experiments/export/includeLists', exportAllExperimentExcludeLists: '/experiments/export/excludeLists', importExperimentList: '/experiments/lists/import', - getMoocletRewardsData: '/experiments/mooclet-rewards', featureFlagGraphInfo: '/flags/date', + experimentsRewardsSummary: '/experiments/rewards', } as const; diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts index 52be0ad3b0..dccc313e53 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts @@ -126,6 +126,11 @@ export class ExperimentDataService { return this.http.post(url, params); } + fetchRewardsDataForExperiment(experimentId: string): Observable { + const url = `${API_ENDPOINTS.experimentsRewardsSummary}/${experimentId}`; + return this.http.get(url); + } + addInclusionList(list: ExperimentSegmentListRequest): Observable { const url = API_ENDPOINTS.addExperimentInclusionList; return this.http.post(url, list); @@ -218,9 +223,4 @@ export class ExperimentDataService { }; return this.updateExperiment(updatedExperiment); } - - fetchMoocletRewardsDataForExperiment(experimentId: string): Observable { - const url = `${API_ENDPOINTS.getMoocletRewardsData}/${experimentId}`; - return this.http.get(url); - } } diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts index d1c4f559de..004aa2d578 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts @@ -62,7 +62,7 @@ import { map, take, tap } from 'rxjs/operators'; import { LocalStorageService } from '../local-storage/local-storage.service'; import { ExperimentSegmentListRequest } from '../segments/store/segments.model'; import { ConditionWeightUpdate } from '../../features/dashboard/experiments/modals/edit-condition-weights-modal/edit-condition-weights-modal.component'; -import { MoocletTSConfigurablePolicyParametersDTO, Prior } from 'upgrade_types'; +import { Prior } from 'upgrade_types'; import { selectCurrentUserEmail } from '../auth/store/auth.selectors'; @Injectable() @@ -301,10 +301,10 @@ export class ExperimentService { updateExperimentConditionPrior(experiment: ExperimentVM, prior: Record): void { const updatedExperiment: ExperimentVM = { ...experiment, - moocletPolicyParameters: { - ...experiment.moocletPolicyParameters, - prior, - } as MoocletTSConfigurablePolicyParametersDTO, + thompsonSamplingConfig: { + ...experiment.thompsonSamplingConfig, + priors: prior, + }, }; this.store$.dispatch( experimentAction.actionUpsertExperiment({ diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.spec.ts deleted file mode 100644 index dacc5c746c..0000000000 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.spec.ts +++ /dev/null @@ -1,483 +0,0 @@ -import { Validators } from '@angular/forms'; -import { - MoocletExperimentHelperService, - EditableTSConfigurablePolicyParameters, - formatTSConfigurablePolicyParamDetails, -} from './mooclet-helper.service'; -import { ASSIGNMENT_ALGORITHM, MoocletTSConfigurablePolicyParametersDTO } from 'upgrade_types'; -import { ExperimentVM } from './store/experiments.model'; -import { environment } from '../../../environments/environment'; - -describe('MoocletAlgorithmHelperService', () => { - let service: MoocletExperimentHelperService; - - beforeEach(() => { - service = new MoocletExperimentHelperService(); - }); - - // ============================================================================ - // Environment and Feature Flag Methods - // ============================================================================ - - describe('isMoocletEnabled', () => { - it('should return true when mooclet toggle is enabled', () => { - environment.moocletToggle = true; - expect(service.isMoocletEnabled()).toBe(true); - }); - - it('should return false when mooclet toggle is disabled', () => { - environment.moocletToggle = false; - expect(service.isMoocletEnabled()).toBe(false); - }); - }); - - describe('isMoocletAlgorithm', () => { - it('should return true for TS_CONFIGURABLE algorithm', () => { - expect(service.isMoocletAlgorithm(ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE)).toBe(true); - }); - - it('should return false for RANDOM algorithm', () => { - expect(service.isMoocletAlgorithm(ASSIGNMENT_ALGORITHM.RANDOM)).toBe(false); - }); - - it('should return false for STRATIFIED_RANDOM_SAMPLING algorithm', () => { - expect(service.isMoocletAlgorithm(ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING)).toBe(false); - }); - }); - - // ============================================================================ - // TS Configurable Default Parameters - // ============================================================================ - - describe('getTSConfigurableDefaults', () => { - it('should return a MoocletTSConfigurablePolicyParametersDTO instance', () => { - const result = service.getTSConfigurableDefaults(); - expect(result).toBeInstanceOf(MoocletTSConfigurablePolicyParametersDTO); - }); - - it('should return default values matching DTO defaults', () => { - const result = service.getTSConfigurableDefaults(); - const expected = new MoocletTSConfigurablePolicyParametersDTO(); - - expect(result.batch_size).toBe(expected.batch_size); - expect(result.uniform_threshold).toBe(expected.uniform_threshold); - expect(result.tspostdiff_thresh).toBe(expected.tspostdiff_thresh); - expect(result.max_rating).toBe(expected.max_rating); - expect(result.min_rating).toBe(expected.min_rating); - }); - }); - - // ============================================================================ - // Derive Editable Parameters - // ============================================================================ - - describe('deriveEditableParametersForTSConfigurable', () => { - it('should return default values when no existing params provided', () => { - const result = service.deriveEditableParametersForTSConfigurable(); - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - - expect(result).toEqual({ - batch_size: defaults.batch_size, - uniform_threshold: defaults.uniform_threshold, - tspostdiff_thresh: defaults.tspostdiff_thresh, - }); - }); - - it('should return default values when existing params is undefined', () => { - const result = service.deriveEditableParametersForTSConfigurable(undefined); - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - - expect(result.batch_size).toBe(defaults.batch_size); - }); - - it('should extract editable fields from existing params', () => { - const existingParams = createMockTSConfigurableParams({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }); - - const result = service.deriveEditableParametersForTSConfigurable(existingParams); - - expect(result).toEqual({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }); - }); - - it('should not include non-editable fields in result', () => { - const existingParams = createMockTSConfigurableParams({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - outcome_variable_name: 'test_outcome', - max_rating: 5, - min_rating: 1, - }); - - const result = service.deriveEditableParametersForTSConfigurable(existingParams); - - expect((result as any).assignmentAlgorithm).toBeUndefined(); - expect((result as any).outcome_variable_name).toBeUndefined(); - expect((result as any).max_rating).toBeUndefined(); - expect((result as any).min_rating).toBeUndefined(); - }); - }); - - // ============================================================================ - // Build Complete DTO - // ============================================================================ - - describe('buildTSConfigurablePolicyParametersDTO', () => { - beforeEach(() => { - jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-01-15T10:30:00.000Z'); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should build complete DTO from editable parameters', () => { - const editableParams: EditableTSConfigurablePolicyParameters = { - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }; - - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.batch_size).toBe(50); - expect(result.uniform_threshold).toBe(100); - expect(result.tspostdiff_thresh).toBe(5); - }); - - it('should set assignmentAlgorithm to MOOCLET_TS_CONFIGURABLE', () => { - const editableParams = createMockEditableParams(); - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.assignmentAlgorithm).toBe(ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE); - }); - - it('should not set outcome_variable_name since it is generated server-side', () => { - const editableParams = createMockEditableParams(); - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.outcome_variable_name).toBeUndefined(); - }); - - it('should set max_rating and min_rating from defaults', () => { - const editableParams = createMockEditableParams(); - const defaults = service.getTSConfigurableDefaults(); - - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.max_rating).toBe(defaults.max_rating); - expect(result.min_rating).toBe(defaults.min_rating); - }); - - it('should not include outcome_variable_name for any experiment name', () => { - const editableParams = createMockEditableParams(); - - const result1 = service.buildTSConfigurablePolicyParametersDTO(editableParams); - const result2 = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result1.outcome_variable_name).toBeUndefined(); - expect(result2.outcome_variable_name).toBeUndefined(); - }); - }); - - // ============================================================================ - // Field Validators - // ============================================================================ - - describe('getTSConfigurableFieldValidators', () => { - it('should return validators for all editable fields', () => { - const validators = service.getTSConfigurableFieldValidators(); - - expect(validators.batch_size).toBeDefined(); - expect(validators.uniform_threshold).toBeDefined(); - expect(validators.tspostdiff_thresh).toBeDefined(); - }); - - it('should include required validator for all fields', () => { - const validators = service.getTSConfigurableFieldValidators(); - - Object.values(validators).forEach((fieldValidators) => { - expect(fieldValidators).toContain(Validators.required); - }); - }); - - it('should include min validators based on defaults', () => { - const validators = service.getTSConfigurableFieldValidators(); - const defaults = service.getTSConfigurableDefaults(); - - expect(validators.batch_size.length).toBe(4); - expect(validators.uniform_threshold.length).toBe(4); - expect(validators.tspostdiff_thresh.length).toBe(3); - - // Test that min validator works correctly for batch_size - const batchSizeMinValidator = validators.batch_size[1]; - // Use a value below the minimum - should fail (return error object) - expect(batchSizeMinValidator({ value: defaults.batch_size - 1 } as any)).toBeTruthy(); - // Use the minimum value - should pass (return null) - expect(batchSizeMinValidator({ value: defaults.batch_size } as any)).toBeNull(); - // Use a value above the minimum - should pass (return null) - expect(batchSizeMinValidator({ value: defaults.batch_size + 1 } as any)).toBeNull(); - }); - - it('should return exactly 3 validators per field (required + min)', () => { - const validators = service.getTSConfigurableFieldValidators(); - - Object.values(validators).forEach((fieldValidators) => { - if (fieldValidators === validators.tspostdiff_thresh) { - expect(fieldValidators.length).toBe(3); - return; - } - expect(fieldValidators.length).toBe(4); - }); - }); - }); - - // ============================================================================ - // Supported Mooclet Algorithms - // ============================================================================ - - describe('getSupportedMoocletAlgorithmOptions', () => { - it('should return empty array when mooclet is disabled', () => { - environment.moocletToggle = false; - const result = service.getSupportedMoocletAlgorithmOptions(); - - expect(result).toEqual([]); - }); - - it('should return algorithm options when mooclet is enabled', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - expect(result.length).toBeGreaterThan(0); - expect(Array.isArray(result)).toBe(true); - }); - - it('should return options with value and description properties', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - result.forEach((option) => { - expect(option).toHaveProperty('value'); - expect(option).toHaveProperty('description'); - expect(typeof option.value).toBe('string'); - expect(typeof option.description).toBe('string'); - }); - }); - - it('should include MOOCLET_TS_CONFIGURABLE in options', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - const tsConfigOption = result.find((opt) => opt.value === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE); - expect(tsConfigOption).toBeDefined(); - expect(tsConfigOption.description).toContain('Adaptive Experiment Algorithm'); - }); - - it('should format description correctly', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - result.forEach((option) => { - expect(option.description).toMatch(/^Adaptive Experiment Algorithm: /); - expect(option.description).toContain(option.value); - }); - }); - }); - - // ============================================================================ - // Validation - // ============================================================================ - - describe('validateTSConfigurablePolicyParameters', () => { - it('should return observable of validation errors', (done) => { - const params = createMockTSConfigurableParams(); - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(Array.isArray(errors)).toBe(true); - done(); - }); - }); - - it('should validate valid parameters successfully', (done) => { - const validParams = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - outcome_variable_name: 'test_outcome', - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(validParams).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - - it('should add assignmentAlgorithm automatically', (done) => { - const params = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - outcome_variable_name: 'test', - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - - it('should validate successfully even with empty outcome_variable_name', (done) => { - const params = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - outcome_variable_name: '', // Empty string should not cause validation errors - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - - it('should validate successfully without outcome_variable_name property', (done) => { - const params = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - // outcome_variable_name property completely missing - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - }); -}); - -// ============================================================================ -// Pure Function Tests -// ============================================================================ - -describe('formatTSConfigurablePolicyParamDetails (pure function)', () => { - it('should return undefined when mooclet toggle is disabled', () => { - environment.moocletToggle = false; - const experiment = createMockExperimentVM(); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeUndefined(); - }); - - it('should return undefined when algorithm is not TS_CONFIGURABLE', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeUndefined(); - }); - - it('should return undefined when moocletPolicyParameters is missing', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: undefined, - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeUndefined(); - }); - - it('should format parameters correctly', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: createMockTSConfigurableParams({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }), - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeDefined(); - expect(Array.isArray(result)).toBe(true); - expect(result.length).toBe(3); - }); - - it('should include all configurable parameters', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: createMockTSConfigurableParams({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }), - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - const values = result.map((param) => param.value); - - expect(values).toContain(50); - expect(values).toContain(100); - expect(values).toContain(5); - }); -}); - -// ============================================================================ -// Helper Functions -// ============================================================================ - -function createMockEditableParams( - overrides?: Partial -): EditableTSConfigurablePolicyParameters { - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - return { - batch_size: defaults.batch_size, - uniform_threshold: defaults.uniform_threshold, - tspostdiff_thresh: defaults.tspostdiff_thresh, - ...overrides, - }; -} - -function createMockTSConfigurableParams( - overrides?: Partial -): MoocletTSConfigurablePolicyParametersDTO { - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - return { - ...defaults, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - outcome_variable_name: 'test_outcome', - ...overrides, - } as MoocletTSConfigurablePolicyParametersDTO; -} - -function createMockExperimentVM(overrides?: Partial): ExperimentVM { - return { - id: 'exp-1', - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - moocletPolicyParameters: undefined, - ...overrides, - } as ExperimentVM; -} diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.ts deleted file mode 100644 index 1769318e94..0000000000 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { Injectable } from '@angular/core'; -import { Observable, from } from 'rxjs'; -import { validate, ValidationError } from 'class-validator'; -import { plainToInstance } from 'class-transformer'; -import { ValidatorFn, Validators } from '@angular/forms'; -import { - ASSIGNMENT_ALGORITHM, - MOOCLET_POLICY_SCHEMA_MAP, - MoocletTSConfigurablePolicyParametersDTO, - SUPPORTED_MOOCLET_ALGORITHMS, -} from 'upgrade_types'; -import { ExperimentVM, TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS } from './store/experiments.model'; -import { environment } from '../../../environments/environment'; -import { BullettedListKeyValueFormat } from '@shared-component-lib/common-section-card-overview-details/common-section-card-overview-details.component'; -import { CommonFormHelpersService } from '../../shared/services/common-form-helpers.service'; - -// ============================================================================ -// Pure Functions (exported for use in selectors and other pure contexts) -// ============================================================================ - -const DEFAULT_MAX_NUMBER_INPUT = 1000000; - -/** - * Format TS Configurable policy parameters for display in the overview section. - * Returns an array of objects with translation keys and values for proper i18n support. - */ -export function formatTSConfigurablePolicyParamDetails( - experiment: ExperimentVM -): BullettedListKeyValueFormat[] | undefined { - if ( - !isMoocletEnabled() || - !isMoocletExperiment(experiment) || - experiment?.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE - ) { - console.warn('mooclet is disabled or experiment is not using TS Configurable algorithm'); - return undefined; - } - - const params = experiment.moocletPolicyParameters; - const formattedParams: BullettedListKeyValueFormat[] = []; - - formattedParams.push({ - labelKey: TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS.BATCH_SIZE, - value: params.batch_size, - }); - formattedParams.push({ - labelKey: TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS.UNIFORM_THRESHOLD, - value: params.uniform_threshold, - }); - formattedParams.push({ - labelKey: TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS.TSPOSTDIFF_THRESH, - value: params.tspostdiff_thresh, - }); - return formattedParams; -} - -export function isMoocletEnabled(): boolean { - return environment.moocletToggle; -} - -export function isMoocletAlgorithm(algorithm: ASSIGNMENT_ALGORITHM): boolean { - return !!MOOCLET_POLICY_SCHEMA_MAP[algorithm]; -} - -export function isMoocletExperiment(experiment: ExperimentVM): boolean { - return isMoocletAlgorithm(experiment.assignmentAlgorithm) && !!experiment.moocletPolicyParameters; -} - -// ============================================================================ -// Type Definitions -// ============================================================================ - -export interface EditableTSConfigurablePolicyParameters { - batch_size: number; - uniform_threshold: number; - tspostdiff_thresh: number; -} - -/** - * Service providing helper methods for mooclet adaptive algorithms - */ -@Injectable({ - providedIn: 'root', -}) -export class MoocletExperimentHelperService { - // ============================================================================ - // General Mooclet Algorithm Helpers - // ============================================================================ - - isMoocletEnabled(): boolean { - return isMoocletEnabled(); - } - - isMoocletAlgorithm(algorithm: ASSIGNMENT_ALGORITHM): boolean { - return isMoocletAlgorithm(algorithm); - } - - /** - * Get supported mooclet algorithm options for UI display. - * Returns algorithm options with descriptions, or empty array if mooclet is disabled. - */ - getSupportedMoocletAlgorithmOptions(): Array<{ - value: ASSIGNMENT_ALGORITHM; - description: string; - }> { - if (!this.isMoocletEnabled()) { - console.warn('Mooclet API is disabled in the environment configuration.'); - return []; - } - - const supportedMoocletAlgorithms = SUPPORTED_MOOCLET_ALGORITHMS as ASSIGNMENT_ALGORITHM[]; - return supportedMoocletAlgorithms.map((algorithmName) => ({ - value: algorithmName, - description: `Adaptive Experiment Algorithm: ${algorithmName}`, - })); - } - - // ============================================================================ - // ts_configurable Policy Parameters Helpers - // ============================================================================ - - isTSConfigurable(algorithm: ASSIGNMENT_ALGORITHM): boolean { - return algorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE; - } - - /** - * Get default TS Configurable parameters instance. - * Provides a single source of truth for default parameter values. - */ - getTSConfigurableDefaults(): MoocletTSConfigurablePolicyParametersDTO { - return new MoocletTSConfigurablePolicyParametersDTO(); - } - - /** - * Validate mooclet policy parameters against the TS Configurable schema. - * Returns an Observable of validation errors (empty array if valid). - */ - validateTSConfigurablePolicyParameters(jsonValue: any): Observable { - const ValidatorClass = MOOCLET_POLICY_SCHEMA_MAP[ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE]; - - const plainDTO = { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - ...jsonValue, - }; - const DTOInstance = plainToInstance(ValidatorClass, plainDTO); - // allowing unknown values until we can use nested validation properly - return from(validate(DTOInstance, { forbidUnknownValues: false })); - } - - /** - * Derive editable form values from existing or default parameters. - * Extracts only the user-configurable fields for form initialization. - */ - deriveEditableParametersForTSConfigurable( - existingParams?: MoocletTSConfigurablePolicyParametersDTO - ): EditableTSConfigurablePolicyParameters { - const defaults = this.getTSConfigurableDefaults(); - const source = existingParams || defaults; - - return { - batch_size: source.batch_size, - uniform_threshold: source.uniform_threshold, - tspostdiff_thresh: source.tspostdiff_thresh, - }; - } - - /** - * Build complete DTO from editable parameters. - * Combines user-provided values with system-generated and default values. - * Note: outcome_variable_name is now generated server-side during experiment creation. - */ - buildTSConfigurablePolicyParametersDTO( - editableParams: EditableTSConfigurablePolicyParameters - ): MoocletTSConfigurablePolicyParametersDTO { - const defaults = this.getTSConfigurableDefaults(); - return { - // User-configurable fields - batch_size: editableParams.batch_size, - uniform_threshold: editableParams.uniform_threshold, - tspostdiff_thresh: editableParams.tspostdiff_thresh, - // System-managed fields - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - max_rating: defaults.max_rating, - min_rating: defaults.min_rating, - } as MoocletTSConfigurablePolicyParametersDTO; - } - - /** - * Get field validators configuration for TS Configurable parameters. - * Provides validation rules based on default minimum values. - */ - getTSConfigurableFieldValidators(): Record { - const defaults = this.getTSConfigurableDefaults(); - - return { - batch_size: [ - Validators.required, - Validators.min(defaults.batch_size), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - uniform_threshold: [ - Validators.required, - Validators.min(defaults.uniform_threshold), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - tspostdiff_thresh: [Validators.required, Validators.min(defaults.tspostdiff_thresh), Validators.max(1.0)], - }; - } - - /** - * Get field validators for per-condition prior success/failure inputs used in the prior editor. - */ - getPriorFieldValidators(): Record { - const priorDefault = 1; - return { - successes: [ - Validators.required, - Validators.min(priorDefault), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - failures: [ - Validators.required, - Validators.min(priorDefault), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - }; - } -} diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts index 804d9ab623..3b6354aacb 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts @@ -48,9 +48,6 @@ import { actionExportExperimentInfoFailure, actionExportExperimentDesign, actionExportExperimentDesignSuccess, - actionFetchRewardsDataForExperiment, - actionFetchRewardsDataForExperimentSuccess, - actionFetchRewardsDataForExperimentFailure, } from './experiments.actions'; import { ExperimentEffects } from './experiments.effects'; import { @@ -1375,75 +1372,4 @@ describe('ExperimentEffects', () => { tick(0); })); }); - - describe('fetchRewardsDataForExperiment$', () => { - it('should dispatch actionFetchRewardsDataForExperimentSuccess on successful fetch', fakeAsync(() => { - const experimentId = 'test-experiment-123'; - const mockRewardsSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; - - experimentDataService.fetchMoocletRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); - - const expectedAction = actionFetchRewardsDataForExperimentSuccess({ - experimentId, - rewardsSummary: mockRewardsSummary, - }); - - service.fetchRewardsDataForExperiment$.subscribe((resultingAction) => { - expect(resultingAction).toEqual(expectedAction); - }); - - actions$.next(actionFetchRewardsDataForExperiment({ experimentId })); - - tick(0); - })); - - it('should dispatch actionFetchRewardsDataForExperimentFailure on fetch error', fakeAsync(() => { - const experimentId = 'test-experiment-123'; - const error = new Error('API error'); - - experimentDataService.fetchMoocletRewardsDataForExperiment = jest.fn().mockReturnValue(throwError(error)); - - const expectedAction = actionFetchRewardsDataForExperimentFailure({ error }); - - service.fetchRewardsDataForExperiment$.subscribe((resultingAction) => { - expect(resultingAction).toEqual(expectedAction); - }); - - actions$.next(actionFetchRewardsDataForExperiment({ experimentId })); - - tick(0); - })); - - it('should call experimentDataService with correct experimentId', fakeAsync(() => { - const experimentId = 'test-experiment-456'; - const mockRewardsSummary = []; - - experimentDataService.fetchMoocletRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); - - service.fetchRewardsDataForExperiment$.subscribe(() => { - expect(experimentDataService.fetchMoocletRewardsDataForExperiment).toHaveBeenCalledWith(experimentId); - }); - - actions$.next(actionFetchRewardsDataForExperiment({ experimentId })); - - tick(0); - })); - }); }); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts index d2c8f23454..950cbe56bb 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts @@ -713,14 +713,10 @@ export class ExperimentEffects { fetchRewardsDataForExperiment$ = createEffect(() => this.actions$.pipe( ofType(experimentAction.actionFetchRewardsDataForExperiment), - map((action) => action.experimentId), - switchMap((experimentId) => - this.experimentDataService.fetchMoocletRewardsDataForExperiment(experimentId).pipe( + switchMap(({ experimentId }) => + this.experimentDataService.fetchRewardsDataForExperiment(experimentId).pipe( map((rewardsSummary) => - experimentAction.actionFetchRewardsDataForExperimentSuccess({ - experimentId, - rewardsSummary, - }) + experimentAction.actionFetchRewardsDataForExperimentSuccess({ experimentId, rewardsSummary }) ), catchError((error) => of(experimentAction.actionFetchRewardsDataForExperimentFailure({ error }))) ) diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts index 212f7095d6..498386f2df 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts @@ -17,8 +17,6 @@ import { PAYLOAD_TYPE, CONDITION_ORDER, ASSIGNMENT_ALGORITHM, - MoocletTSConfigurablePolicyParametersDTO, - MoocletPolicyParametersDTO, REPEATED_MEASURE, SEGMENT_TYPE, IEnrollmentCompleteCondition, @@ -28,6 +26,13 @@ import { ExperimentRewardsSummary, } from 'upgrade_types'; import { Segment } from '../../segments/store/segments.model'; + +export interface ThompsonSamplingConfigDTO { + warmupThreshold?: number; + minimumDrawDifference?: number; + batchSize?: number; + priors?: Record; +} export { CONSISTENCY_RULE, ASSIGNMENT_UNIT, @@ -284,7 +289,7 @@ export interface Experiment { experimentSegmentExclusion: SegmentNew[]; groupSatisfied?: number; backendVersion: string; - moocletPolicyParameters?: MoocletTSConfigurablePolicyParametersDTO; + thompsonSamplingConfig?: ThompsonSamplingConfigDTO; } export interface ParticipantsMember { @@ -510,7 +515,7 @@ export interface DraftExperimentRequest { endOn?: string; revertTo?: string; backendVersion?: string; - moocletPolicyParameters?: MoocletPolicyParametersDTO; + thompsonSamplingConfig?: ThompsonSamplingConfigDTO; rewardMetricKey?: string; // Arrays that can be empty for drafts @@ -593,10 +598,10 @@ export const EXPERIMENT_OVERVIEW_LABELS = { TAGS: 'Tags', } as const; -export const TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS = { +export const THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS = { BATCH_SIZE: 'home.new-experiment.design.ts-configurable-policy.batch-size.label.text', - UNIFORM_THRESHOLD: 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text', - TSPOSTDIFF_THRESH: 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text', + WARMUP_THRESHOLD: 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text', + MINIMUM_DRAW_DIFFERENCE: 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text', }; export const EXPERIMENT_ROOT_DISPLAYED_COLUMNS = Object.values(EXPERIMENT_ROOT_COLUMN_NAMES); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts index 102e8bcf39..d39e0724a5 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts @@ -26,7 +26,7 @@ import { ExperimentRewardsSummary, } from 'upgrade_types'; import { determineWeightingMethod, isWeightSumValid } from '../condition-helper.service'; -import { formatTSConfigurablePolicyParamDetails } from '../mooclet-helper.service'; +import { formatThompsonSamplingConfigDetails } from '../thompson-sampling-helper.service'; import { KeyValueFormat } from '@shared-component-lib/common-section-card-overview-details/common-section-card-overview-details.component'; export const selectExperimentState = createFeatureSelector('experiments'); @@ -197,9 +197,8 @@ export const selectExperimentOverviewDetails = createSelector(selectSelectedExpe }; // Add policy parameters if they exist - if (experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { - details[EXPERIMENT_OVERVIEW_LABELS.ADAPTIVE_ALGORITHM_PARAMETERS] = - formatTSConfigurablePolicyParamDetails(experiment); + if (experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + details[EXPERIMENT_OVERVIEW_LABELS.ADAPTIVE_ALGORITHM_PARAMETERS] = formatThompsonSamplingConfigDetails(experiment); } // Always add tags at the end @@ -453,7 +452,7 @@ export const selectDisabledExperimentFields = createSelector(selectSelectedExper } if ([EXPERIMENT_STATE.COMPLETED, EXPERIMENT_STATE.ARCHIVED].includes(state)) { - return [...baseRestrictedFields, 'moocletPolicyParameters']; + return [...baseRestrictedFields, 'thompsonSamplingConfig']; } return baseRestrictedFields; diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts new file mode 100644 index 0000000000..debd885c7e --- /dev/null +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts @@ -0,0 +1,110 @@ +import { Injectable } from '@angular/core'; +import { ValidatorFn, Validators } from '@angular/forms'; +import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; +import { + ExperimentVM, + THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS, + ThompsonSamplingConfigDTO, +} from './store/experiments.model'; +import { BullettedListKeyValueFormat } from '@shared-component-lib/common-section-card-overview-details/common-section-card-overview-details.component'; +import { CommonFormHelpersService } from '../../shared/services/common-form-helpers.service'; + +const DEFAULT_MAX_NUMBER_INPUT = 1000000; + +export interface EditableThompsonSamplingConfig { + batchSize: number; + warmupThreshold: number; + minimumDrawDifference: number; +} + +export function isThompsonSamplingExperiment(experiment: ExperimentVM): boolean { + return experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; +} + +export function formatThompsonSamplingConfigDetails( + experiment: ExperimentVM +): BullettedListKeyValueFormat[] | undefined { + if (experiment?.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return undefined; + } + + const config = experiment.thompsonSamplingConfig; + return [ + { labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.BATCH_SIZE, value: config?.batchSize }, + { labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.WARMUP_THRESHOLD, value: config?.warmupThreshold }, + { + labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.MINIMUM_DRAW_DIFFERENCE, + value: config?.minimumDrawDifference, + }, + ]; +} + +@Injectable({ + providedIn: 'root', +}) +export class ThompsonSamplingHelperService { + isThompsonSamplingAlgorithm(algorithm: ASSIGNMENT_ALGORITHM): boolean { + return algorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + } + + isThompsonSamplingExperiment(experiment: ExperimentVM): boolean { + return isThompsonSamplingExperiment(experiment); + } + + getDefaults(): EditableThompsonSamplingConfig { + return { batchSize: 1, warmupThreshold: 0, minimumDrawDifference: 0 }; + } + + deriveEditableParameters(existing?: ThompsonSamplingConfigDTO): EditableThompsonSamplingConfig { + const defaults = this.getDefaults(); + return { + batchSize: existing?.batchSize ?? defaults.batchSize, + warmupThreshold: existing?.warmupThreshold ?? defaults.warmupThreshold, + minimumDrawDifference: existing?.minimumDrawDifference ?? defaults.minimumDrawDifference, + }; + } + + buildConfig(editableParams: EditableThompsonSamplingConfig): ThompsonSamplingConfigDTO { + return { + batchSize: editableParams.batchSize, + warmupThreshold: editableParams.warmupThreshold, + minimumDrawDifference: editableParams.minimumDrawDifference, + }; + } + + getFieldValidators(): Record { + return { + batchSize: [ + Validators.required, + Validators.min(1), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + warmupThreshold: [ + Validators.required, + Validators.min(0), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + minimumDrawDifference: [Validators.required, Validators.min(0), Validators.max(1.0)], + }; + } + + getPriorFieldValidators(): Record { + const minValue = 1; + return { + successes: [ + Validators.required, + Validators.min(minValue), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + failures: [ + Validators.required, + Validators.min(minValue), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + }; + } +} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts index 1e7a22dc5c..f9fe434a9a 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts @@ -11,7 +11,7 @@ import { Observable, combineLatest, map, startWith } from 'rxjs'; import { CommonLearnMoreLinkComponent, CommonModalComponent } from '@shared-component-lib'; import { CommonFormHelpersService } from '../../../../../shared/services/common-form-helpers.service'; import { CommonModalConfig } from '@shared-component-lib/common-modal/common-modal.types'; -import { MoocletExperimentHelperService } from '../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../core/experiments/thompson-sampling-helper.service'; import { Prior } from 'upgrade_types'; import { SharedModule } from '../../../../../shared/shared.module'; @@ -51,7 +51,7 @@ export class EditConditionPriorModalComponent implements OnInit { public config: CommonModalConfig<{ conditions: ConditionPriorUpdate[] }>, public dialogRef: MatDialogRef, private readonly formBuilder: FormBuilder, - private readonly moocletHelperService: MoocletExperimentHelperService + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService ) {} ngOnInit(): void { @@ -60,7 +60,7 @@ export class EditConditionPriorModalComponent implements OnInit { } createPriorForm(): void { - const validators = this.moocletHelperService.getPriorFieldValidators(); + const validators = this.thompsonSamplingHelperService.getPriorFieldValidators(); const conditionsFormArray = this.formBuilder.array( this.conditions.map((condition) => diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html index 85acb7ff5a..c18d95e2d1 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html @@ -14,19 +14,19 @@ {{ 'home.new-experiment.design.ts-configurable-policy.batch-size.label.text' | translate }} - + {{ 'home.new-experiment.design.ts-configurable-policy.batch-size.hint.text' | translate }} - @if (policyForm.get('batch_size')?.hasError('min')) { + @if (policyForm.get('batchSize')?.hasError('min')) { {{ 'home.new-experiment.design.ts-configurable-policy.batch-size.min-error.text' | translate }} - } @if (policyForm.get('batch_size')?.hasError('max')) { + } @if (policyForm.get('batchSize')?.hasError('max')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.max-error.text' | translate }} - } @if (policyForm.get('batch_size')?.hasError('integer')) { + } @if (policyForm.get('batchSize')?.hasError('integer')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.integer-error.text' | translate }} @@ -37,19 +37,19 @@ {{ 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text' | translate }} - + {{ 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text' | translate }} - @if (policyForm.get('uniform_threshold')?.hasError('min')) { + @if (policyForm.get('warmupThreshold')?.hasError('min')) { {{ 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.min-error.text' | translate }} - } @if (policyForm.get('uniform_threshold')?.hasError('max')) { + } @if (policyForm.get('warmupThreshold')?.hasError('max')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.max-error.text' | translate }} - } @if (policyForm.get('uniform_threshold')?.hasError('integer')) { + } @if (policyForm.get('warmupThreshold')?.hasError('integer')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.integer-error.text' | translate }} @@ -63,7 +63,7 @@ {{ 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.hint.text' | translate }} - @if (policyForm.get('tspostdiff_thresh')?.hasError('min')) { + @if (policyForm.get('minimumDrawDifference')?.hasError('min')) { {{ 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.min-error.text' | translate }} - } @if (policyForm.get('tspostdiff_thresh')?.hasError('max')) { + } @if (policyForm.get('minimumDrawDifference')?.hasError('max')) { {{ 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.max-error.text' | translate }} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts index cc28d0b6c7..a45a5175fa 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts @@ -1,26 +1,14 @@ import { Component, EventEmitter, inject, Input, OnDestroy, OnInit, Output } from '@angular/core'; import { FormBuilder, FormGroup, ReactiveFormsModule } from '@angular/forms'; - import { MatFormFieldModule } from '@angular/material/form-field'; import { MatInputModule } from '@angular/material/input'; import { TranslateModule } from '@ngx-translate/core'; +import { BehaviorSubject, map, Observable, of, startWith, Subject, Subscription } from 'rxjs'; +import { ThompsonSamplingConfigDTO } from '../../../../../../core/experiments/store/experiments.model'; import { - BehaviorSubject, - debounceTime, - from, - map, - Observable, - startWith, - Subject, - Subscription, - switchMap, -} from 'rxjs'; -import { ValidationError } from 'class-validator'; -import { MoocletTSConfigurablePolicyParametersDTO } from 'upgrade_types'; -import { - EditableTSConfigurablePolicyParameters, - MoocletExperimentHelperService, -} from '../../../../../../core/experiments/mooclet-helper.service'; + EditableThompsonSamplingConfig, + ThompsonSamplingHelperService, +} from '../../../../../../core/experiments/thompson-sampling-helper.service'; import isEqual from 'lodash.isequal'; @Component({ @@ -31,27 +19,26 @@ import isEqual from 'lodash.isequal'; styleUrl: './ts-configurable-policy-parameters-form.component.scss', }) export class TsConfigurablePolicyParametersFormComponent implements OnInit, OnDestroy { - @Input() existingPolicyParams?: MoocletTSConfigurablePolicyParametersDTO; - @Input() disabled = false; // Disable all form fields when true - @Output() parametersChange = new EventEmitter(); + @Input() existingPolicyParams?: ThompsonSamplingConfigDTO; + @Input() disabled = false; + @Output() parametersChange = new EventEmitter(); @Output() validationChange = new EventEmitter(); @Output() formChanged = new EventEmitter(); private readonly formBuilder = inject(FormBuilder); - private readonly moocletExperimentHelperService = inject(MoocletExperimentHelperService); + private readonly thompsonSamplingHelperService = inject(ThompsonSamplingHelperService); policyForm: FormGroup; - validationErrors$ = new BehaviorSubject([]); + validationErrors$ = new BehaviorSubject([]); isInitialFormValueChanged$: Observable; - initialFormValue: EditableTSConfigurablePolicyParameters; - formValueChanges$ = new Subject(); + initialFormValue: EditableThompsonSamplingConfig; + formValueChanges$ = new Subject(); subscriptions = new Subscription(); ngOnInit(): void { this.initializeFormValues(); this.createForm(); - // Disable form if disabled input is true if (this.disabled) { this.policyForm.disable(); } @@ -66,73 +53,48 @@ export class TsConfigurablePolicyParametersFormComponent implements OnInit, OnDe } private initializeFormValues(): void { - // Delegate to service to derive initial form values from existing or default parameters - this.initialFormValue = this.moocletExperimentHelperService.deriveEditableParametersForTSConfigurable( - this.existingPolicyParams - ); + this.initialFormValue = this.thompsonSamplingHelperService.deriveEditableParameters(this.existingPolicyParams); } private createForm(): void { const params = this.initialFormValue; - const validators = this.moocletExperimentHelperService.getTSConfigurableFieldValidators(); + const validators = this.thompsonSamplingHelperService.getFieldValidators(); this.policyForm = this.formBuilder.group({ - batch_size: [params.batch_size, validators.batch_size], - uniform_threshold: [params.uniform_threshold, validators.uniform_threshold], - tspostdiff_thresh: [params.tspostdiff_thresh, validators.tspostdiff_thresh], + batchSize: [params.batchSize, validators.batchSize], + warmupThreshold: [params.warmupThreshold, validators.warmupThreshold], + minimumDrawDifference: [params.minimumDrawDifference, validators.minimumDrawDifference], }); } private setupValidation(): void { - // Set up validation pipeline with debounce - this.subscriptions.add( - this.formValueChanges$ - .pipe( - debounceTime(300), - switchMap((formValue) => this.validateParameters(formValue)) - ) - .subscribe((errors) => { - this.validationErrors$.next(errors); - this.emitValidationState(errors); - }) - ); - - // Emit validation state immediately when Angular form validity changes this.subscriptions.add( this.policyForm.statusChanges.subscribe(() => { - this.emitValidationState(this.validationErrors$.value); + this.emitValidationState(); }) ); } - private emitValidationState(backendErrors: ValidationError[]): void { - // Treat disabled form as valid; otherwise require Angular + backend validation to pass - const formDisabled = this.policyForm.disabled; - const isValid = formDisabled || (this.policyForm.valid && backendErrors.length === 0); + private emitValidationState(): void { + const isValid = this.policyForm.disabled || this.policyForm.valid; this.validationChange.emit(isValid); } private listenToFormChanges(): void { this.subscriptions.add( - this.policyForm.valueChanges.subscribe((formValue: EditableTSConfigurablePolicyParameters) => { + this.policyForm.valueChanges.subscribe((formValue: EditableThompsonSamplingConfig) => { this.emitFormValueChanges(formValue); }) ); - // Trigger initial validation and emit initial form state this.emitFormValueChanges(this.policyForm.value); - // Emit initial validation state considering Angular form validity - this.emitValidationState(this.validationErrors$.value); + this.emitValidationState(); } private listenForIsInitialFormValueChanged() { this.isInitialFormValueChanged$ = this.policyForm.valueChanges.pipe( startWith(this.policyForm.value), - map(() => { - // Compare form values with initial parameters - const currentValues = this.policyForm.value; - return !isEqual(currentValues, this.initialFormValue); - }) + map(() => !isEqual(this.policyForm.value, this.initialFormValue)) ); this.subscriptions.add( this.isInitialFormValueChanged$.subscribe((hasChanged) => { @@ -141,20 +103,8 @@ export class TsConfigurablePolicyParametersFormComponent implements OnInit, OnDe ); } - private validateParameters(formValue: EditableTSConfigurablePolicyParameters): Observable { - const completeParams = this.buildCompletePolicyParametersDTO(formValue); - return from(this.moocletExperimentHelperService.validateTSConfigurablePolicyParameters(completeParams)); - } - - private emitFormValueChanges(formValue: EditableTSConfigurablePolicyParameters): void { + private emitFormValueChanges(formValue: EditableThompsonSamplingConfig): void { this.formValueChanges$.next(formValue); - this.parametersChange.emit(this.buildCompletePolicyParametersDTO(formValue)); - } - - private buildCompletePolicyParametersDTO( - formValue: EditableTSConfigurablePolicyParameters - ): MoocletTSConfigurablePolicyParametersDTO { - // Delegate DTO assembly to service - return this.moocletExperimentHelperService.buildTSConfigurablePolicyParametersDTO(formValue); + this.parametersChange.emit(this.thompsonSamplingHelperService.buildConfig(formValue)); } } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html index 30c92b84f4..22946fa96d 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html @@ -159,16 +159,13 @@ {{ 'experiments.upsert-experiment-modal.stratification-factor-hint.text' | translate }} - } - - - @if (assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { + } @if (assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts index eefc94f756..7f2853d5d6 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts @@ -36,14 +36,14 @@ import { POST_EXPERIMENT_RULE, ASSIGNMENT_ALGORITHM_DISPLAY_MAP, EXPERIMENT_TYPE, - MoocletPolicyParametersDTO, } from 'upgrade_types'; import { CommonModalConfig } from '@shared-component-lib/common-modal/common-modal.types'; import { StratificationFactorsService } from '../../../../../core/stratification-factors/stratification-factors.service'; import { ENV, Environment } from '../../../../../../environments/environment-types'; import { SharedModule } from '../../../../../shared/shared.module'; import { TsConfigurablePolicyParametersFormComponent } from './ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component'; -import { MoocletExperimentHelperService } from '../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../core/experiments/thompson-sampling-helper.service'; +import { ThompsonSamplingConfigDTO } from '../../../../../core/experiments/store/experiments.model'; @Component({ selector: 'upsert-experiment-modal', @@ -94,20 +94,20 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { CommonTagInputType = CommonTagInputType; /** - * Mooclet policy parameters state management + * Thompson Sampling config state management * - * This property holds the current mooclet policy parameters and is updated in several scenarios: - * 1. EDIT mode initialization: Set from sourceExperiment.moocletPolicyParameters if present - * 2. Algorithm changes: Set to defaults when switching TO a mooclet algorithm (if not already set) - * 3. Algorithm changes: Cleared when switching FROM mooclet to non-mooclet algorithm + * This property holds the current Thompson Sampling config and is updated in several scenarios: + * 1. EDIT mode initialization: Set from sourceExperiment.thompsonSamplingConfig if present + * 2. Algorithm changes: Set to defaults when switching TO Thompson Sampling (if not already set) + * 3. Algorithm changes: Cleared when switching FROM Thompson Sampling to another algorithm * 4. Child form events: Updated when user modifies parameters in the child form * * The child form receives existing params via async pipe but emits updates through event handlers. */ - moocletPolicyParametersFormValue: MoocletPolicyParametersDTO; - isMoocletFormValid$ = new BehaviorSubject(true); - isMoocletFormChanged$ = new BehaviorSubject(false); - isMoocletFormDisabled = false; // Set to true when experiment is in CANCELLED state + thompsonSamplingConfigFormValue: ThompsonSamplingConfigDTO; + isTSFormValid$ = new BehaviorSubject(true); + isTSFormChanged$ = new BehaviorSubject(false); + isTSFormDisabled = false; // Set to true when experiment is in CANCELLED state // Enum references for template UPSERT_EXPERIMENT_ACTION = UPSERT_EXPERIMENT_ACTION; @@ -189,7 +189,7 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { private readonly formBuilder: FormBuilder, private readonly experimentService: ExperimentService, private readonly stratificationFactorsService: StratificationFactorsService, - private readonly moocletExperimentHelperService: MoocletExperimentHelperService, + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService, public dialogRef: MatDialogRef, @Inject(ENV) private readonly environment: Environment ) { @@ -199,9 +199,10 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { description: 'Condition will be assigned within subjects (e.g., participant sees multiple conditions).', }); } - // Delegate to service to get supported mooclet algorithm options - const moocletAlgorithmOptions = this.moocletExperimentHelperService.getSupportedMoocletAlgorithmOptions(); - this.assignmentAlgorithms.push(...moocletAlgorithmOptions); + this.assignmentAlgorithms.push({ + value: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + description: ASSIGNMENT_ALGORITHM_DISPLAY_MAP[ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING], + }); } ngOnInit(): void { @@ -235,9 +236,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { this.experimentForm.get(fieldName)?.disable(); }); - // Handle moocletPolicyParameters separately (it's a child form component) - if (fieldsToDisable.includes('moocletPolicyParameters')) { - this.isMoocletFormDisabled = true; + if (fieldsToDisable.includes('thompsonSamplingConfig')) { + this.isTSFormDisabled = true; } }) ); @@ -268,9 +268,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { // Initialize consistency rules state based on initial unit of assignment this.initializeConsistencyRules(initialValues.unitOfAssignment); - // Initialize moocletPolicyParameters in EDIT mode to preserve existing values - if (action === UPSERT_EXPERIMENT_ACTION.EDIT && sourceExperiment?.moocletPolicyParameters) { - this.moocletPolicyParametersFormValue = sourceExperiment.moocletPolicyParameters; + if (action === UPSERT_EXPERIMENT_ACTION.EDIT && sourceExperiment?.thompsonSamplingConfig) { + this.thompsonSamplingConfigFormValue = sourceExperiment.thompsonSamplingConfig; } this.initialFormValues$.next(this.experimentForm.getRawValue() as ExperimentFormData); @@ -365,15 +364,15 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { // Button is disabled if: // 1. Currently loading, OR - // 2. Changes are required and none detected in primary or mooclet form, OR - // 3. Either primary or mooclet form is invalid + // 2. Changes are required and none detected in primary or Thompson Sampling form, OR + // 3. Either primary or Thompson Sampling form is invalid listenForPrimaryButtonDisabled() { this.isPrimaryButtonDisabled$ = this.isLoadingUpsertExperiment$.pipe( - combineLatestWith(this.isInitialFormValueChanged$, this.isMoocletFormValid$, this.isMoocletFormChanged$), - map(([isLoading, isInitialFormValueChanged, isMoocletFormValid, isMoocletFormChanged]) => { + combineLatestWith(this.isInitialFormValueChanged$, this.isTSFormValid$, this.isTSFormChanged$), + map(([isLoading, isInitialFormValueChanged, isTSFormValid, isTSFormChanged]) => { const changesRequiredToAllowSubmit = this.config.params.action !== UPSERT_EXPERIMENT_ACTION.DUPLICATE; - const hasChanges = isInitialFormValueChanged || isMoocletFormChanged; - const allFormsValid = this.experimentForm.valid && isMoocletFormValid; + const hasChanges = isInitialFormValueChanged || isTSFormChanged; + const allFormsValid = this.experimentForm.valid && isTSFormValid; return isLoading || (changesRequiredToAllowSubmit && !hasChanges) || !allFormsValid; }) @@ -398,8 +397,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { this.subscriptions.add( this.experimentForm.get('assignmentAlgorithm')?.valueChanges.subscribe((algorithm) => { this.validateStratificationFactorSelection(algorithm); - // Always check for mooclet algorithm changes to handle both TO and FROM mooclet transitions - this.checkForMoocletAlgorithmChange(); + // Check for Thompson Sampling changes to handle both TO and FROM transitions + this.checkForAlgorithmChange(); }) ); } @@ -485,20 +484,17 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { ); } - checkForMoocletAlgorithmChange(): void { + checkForAlgorithmChange(): void { const algorithm = this.assignmentAlgorithmValue; - if (!this.moocletExperimentHelperService.isMoocletAlgorithm(algorithm)) { - // Clear params when switching to non-mooclet algorithm - this.moocletPolicyParametersFormValue = undefined; - } else if (this.moocletExperimentHelperService.isTSConfigurable(algorithm)) { - // Only set defaults if we don't already have params - // This preserves existing values in EDIT mode or after user has made changes - if (!this.moocletPolicyParametersFormValue) { - this.moocletPolicyParametersFormValue = this.moocletExperimentHelperService.getTSConfigurableDefaults(); + if (algorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + if (!this.thompsonSamplingConfigFormValue) { + this.thompsonSamplingConfigFormValue = this.thompsonSamplingHelperService.buildConfig( + this.thompsonSamplingHelperService.getDefaults() + ); } } else { - throw new Error(`Unsupported mooclet algorithm selected: ${algorithm}`); + this.thompsonSamplingConfigFormValue = undefined; } } @@ -537,23 +533,23 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { } /** - * Event handlers for mooclet policy parameters child form + * Event handlers for Thompson Sampling config child form * * These handlers receive events from the child form and update parent state: - * - onMoocletParametersChange: Updates moocletPolicyParameters with user changes - * - onMoocletFormValidityChange: Updates validation state for button disable logic - * - onMoocletFormChanged: Tracks if user has modified the form (for save button enable) + * - onTSConfigChange: Updates thompsonSamplingConfig with user changes + * - onTSFormValidityChange: Updates validation state for button disable logic + * - onTSFormChanged: Tracks if user has modified the form (for save button enable) */ - onMoocletParametersChange(params: MoocletPolicyParametersDTO): void { - this.moocletPolicyParametersFormValue = { ...params, assignmentAlgorithm: this.assignmentAlgorithmValue }; + onTSConfigChange(params: ThompsonSamplingConfigDTO): void { + this.thompsonSamplingConfigFormValue = params; } - onMoocletFormValidityChange(isValid: boolean): void { - this.isMoocletFormValid$.next(isValid); + onTSFormValidityChange(isValid: boolean): void { + this.isTSFormValid$.next(isValid); } - onMoocletFormChanged(hasChanged: boolean): void { - this.isMoocletFormChanged$.next(hasChanged); + onTSFormChanged(hasChanged: boolean): void { + this.isTSFormChanged$.next(hasChanged); } // --------------------------------------------------------------------------- @@ -638,8 +634,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { backendVersion: undefined, // @IsOptional - can be undefined }; - if (this.moocletPolicyParametersFormValue) { - experimentRequest.moocletPolicyParameters = this.moocletPolicyParametersFormValue; + if (this.thompsonSamplingConfigFormValue) { + experimentRequest.thompsonSamplingConfig = this.thompsonSamplingConfigFormValue; } this.experimentService.createNewExperiment(experimentRequest); @@ -690,8 +686,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { backendVersion: sourceExperiment.backendVersion, }; - if (this.moocletPolicyParametersFormValue) { - experimentRequest.moocletPolicyParameters = this.moocletPolicyParametersFormValue; + if (this.thompsonSamplingConfigFormValue) { + experimentRequest.thompsonSamplingConfig = this.thompsonSamplingConfigFormValue; } this.experimentService.createNewExperiment(experimentRequest); @@ -757,10 +753,10 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { revertTo: sourceExperiment.revertTo, }; - if (this.moocletExperimentHelperService.isMoocletAlgorithm(assignmentAlgorithm)) { - experimentRequest.moocletPolicyParameters = this.moocletPolicyParametersFormValue; + if (this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(assignmentAlgorithm)) { + experimentRequest.thompsonSamplingConfig = this.thompsonSamplingConfigFormValue; } else { - experimentRequest.moocletPolicyParameters = undefined; + experimentRequest.thompsonSamplingConfig = undefined; } this.experimentService.updateExperiment(experimentRequest as unknown as ExperimentVM); diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html index 0bdadd2140..4ce16bf521 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html @@ -34,8 +34,8 @@ [showActions]="canShowActions" [actionsDisabled]="vm.restriction.isDisabled" [actionsTooltip]="restrictionTooltip" - [isMoocletExperiment]="isMoocletExperiment(vm.experiment)" - [prior]="vm.experiment.moocletPolicyParameters?.prior" + [isThompsonSamplingExperiment]="isThompsonSamplingExperiment(vm.experiment)" + [prior]="vm.experiment.thompsonSamplingConfig?.priors" (rowAction)="onRowAction($event, vm.experiment.id, appContext)" (editWeights)="onEditWeights($event, vm.experiment)" (editPrior)="onEditPrior($event, vm.experiment)" diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts index edd563220b..5ed935466a 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts @@ -26,7 +26,7 @@ import { Prior } from 'upgrade_types'; import { ConditionHelperService } from '../../../../../../../core/experiments/condition-helper.service'; import { selectConditionWeightsValid } from '../../../../../../../core/experiments/store/experiments.selectors'; import { Store } from '@ngrx/store'; -import { MoocletExperimentHelperService } from '../../../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../../../core/experiments/thompson-sampling-helper.service'; @Component({ selector: 'app-experiment-conditions-section-card', @@ -62,7 +62,7 @@ export class ExperimentConditionsSectionCardComponent implements OnInit { private readonly dialogService: DialogService, private readonly conditionHelperService: ConditionHelperService, private readonly store: Store, - private readonly moocletHelperService: MoocletExperimentHelperService + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService ) {} ngOnInit() { @@ -79,8 +79,8 @@ export class ExperimentConditionsSectionCardComponent implements OnInit { ); } - isMoocletExperiment(experiment: Experiment) { - return this.moocletHelperService.isMoocletAlgorithm(experiment?.assignmentAlgorithm); + isThompsonSamplingExperiment(experiment: Experiment) { + return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment?.assignmentAlgorithm); } onAddConditionClick(appContext: string, experimentId: string): void { @@ -135,7 +135,7 @@ export class ExperimentConditionsSectionCardComponent implements OnInit { } onEditPrior(conditions: ExperimentCondition[], experiment: ExperimentVM): void { - const existingPrior = experiment.moocletPolicyParameters?.prior; + const existingPrior = experiment.thompsonSamplingConfig?.priors; this.dialogService .openEditConditionPriorModal(conditions, existingPrior) .subscribe((result: Record | undefined) => { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html index a65b9e8c3c..bd3ddf97e3 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html @@ -25,7 +25,7 @@ *matHeaderCellDef class="weight-column ft-14-600" [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" - [matTooltipDisabled]="!isMoocletExperiment" + [matTooltipDisabled]="!isThompsonSamplingExperiment" matTooltipPosition="above" > {{ CONDITION_TRANSLATION_KEYS.WEIGHT | translate }} @@ -35,17 +35,17 @@ *matCellDef="let condition" class="weight-column ft-14-400" [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" - [matTooltipDisabled]="!isMoocletExperiment" + [matTooltipDisabled]="!isThompsonSamplingExperiment" matTooltipPosition="above" > - {{ isMoocletExperiment ? 'N/A' : condition.assignmentWeight }} + {{ isThompsonSamplingExperiment ? 'N/A' : condition.assignmentWeight }} - @if (showActions && !isMoocletExperiment) { + @if (showActions && !isThompsonSamplingExperiment) {
- @if (showActions && isMoocletExperiment) { + @if (showActions && isThompsonSamplingExperiment) {
; @Output() rowAction = new EventEmitter(); @Output() editWeights = new EventEmitter(); @Output() editPrior = new EventEmitter(); get displayedColumns(): string[] { - if (this.isMoocletExperiment) { + if (this.isThompsonSamplingExperiment) { return ['condition', 'priorSuccesses', 'priorFailures', 'priorEdit', 'description', 'actions']; } return ['condition', 'weight', 'weightEdit', 'description', 'actions']; diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts index f255c4cf81..dc62bf93ac 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts @@ -19,7 +19,7 @@ import { Observable, Subscription, combineLatest } from 'rxjs'; import { map, filter, startWith } from 'rxjs/operators'; import { Experiment } from '../../../../../../core/experiments/store/experiments.model'; import { SegmentsService } from '../../../../../../core/segments/segments.service'; -import { MoocletExperimentHelperService } from '../../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../../core/experiments/thompson-sampling-helper.service'; import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; @Component({ @@ -56,7 +56,7 @@ export class ExperimentDetailsPageContentComponent implements OnInit, OnDestroy private readonly router: Router, private readonly route: ActivatedRoute, private readonly segmentService: SegmentsService, - private readonly moocletHelperService: MoocletExperimentHelperService + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService ) {} ngOnInit() { @@ -91,10 +91,7 @@ export class ExperimentDetailsPageContentComponent implements OnInit, OnDestroy if (!experiment) { return false; } - const isMoocletEnabled = this.moocletHelperService.isMoocletEnabled(); - const hasMoocletPolicyParameters = !!experiment.moocletPolicyParameters; - const isTSConfigurable = experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE; - return isMoocletEnabled && hasMoocletPolicyParameters && isTSConfigurable; + return experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; }) ); } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html index d09cc9b7c0..d03e78e68a 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html @@ -11,25 +11,29 @@ class="ft-14-600" *matHeaderCellDef [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" - [matTooltipDisabled]="!isMoocletExperiment(experiment)" + [matTooltipDisabled]="!isThompsonSamplingExperiment(experiment)" matTooltipPosition="above" > - {{ key.includes('Icon') || (key === 'weight' && isMoocletExperiment(experiment)) ? '' : columnHeaders[key] }} + {{ + key.includes('Icon') || (key === 'weight' && isThompsonSamplingExperiment(experiment)) + ? '' + : columnHeaders[key] + }} - - @if (key === 'weight' && isMoocletExperiment(experiment)) { + + @if (key === 'weight' && isThompsonSamplingExperiment(experiment)) { N/A } @else { @if (!key.includes('Icon')) { {{ element.data[key] }} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts index a6a5f36345..0868978fe0 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts @@ -6,7 +6,7 @@ import { MatIconModule } from '@angular/material/icon'; import { MatTableModule } from '@angular/material/table'; import { CommonModule } from '@angular/common'; import { EnrollmentPointPartitionTableComponent } from '../enrollment-point-partition-table/enrollment-point-partition-table.component'; -import { MoocletExperimentHelperService } from '../../../../../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../../../../../core/experiments/thompson-sampling-helper.service'; import { MatTooltipModule } from '@angular/material/tooltip'; @Component({ selector: 'app-enrollment-condition-expandable-row', @@ -32,7 +32,10 @@ export class EnrollmentConditionExpandableRowComponent implements OnDestroy { columnHeaders = {}; translateSub: Subscription; - constructor(private translate: TranslateService, private moocletHelperService: MoocletExperimentHelperService) { + constructor( + private translate: TranslateService, + private thompsonSamplingHelperService: ThompsonSamplingHelperService + ) { this.translateSub = this.translate .get([ 'global.condition.text', @@ -52,8 +55,8 @@ export class EnrollmentConditionExpandableRowComponent implements OnDestroy { }); } - isMoocletExperiment(experiment: ExperimentVM): boolean { - return this.moocletHelperService.isMoocletAlgorithm(experiment?.assignmentAlgorithm); + isThompsonSamplingExperiment(experiment: ExperimentVM): boolean { + return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment?.assignmentAlgorithm); } toggleExpandableSymbol(id: string): void { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html index 814e5b3633..ef201791d0 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html @@ -82,6 +82,22 @@ + + + + {{ 'experiments.details.posteriors.estimated-weight.text' | translate }} + + + {{ row.estimatedWeight != null ? '≈' + row.estimatedWeight + '%' : '—' }} + + + diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss index 2de24b6719..bd3540bf4c 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss @@ -96,6 +96,10 @@ width: 8%; } + .estimated-weight-column { + width: 10%; + } + .data-column { text-align: center; } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts index 00edc1d6a8..fe198430cd 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts @@ -2,13 +2,14 @@ import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; import { CommonModule } from '@angular/common'; import { MatTableModule } from '@angular/material/table'; import { MatProgressBarModule } from '@angular/material/progress-bar'; +import { MatTooltipModule } from '@angular/material/tooltip'; import { TranslateModule } from '@ngx-translate/core'; import { ExperimentRewardsSummary } from 'upgrade_types'; @Component({ selector: 'app-ts-configurable-reward-count-table', standalone: true, - imports: [CommonModule, MatTableModule, MatProgressBarModule, TranslateModule], + imports: [CommonModule, MatTableModule, MatProgressBarModule, MatTooltipModule, TranslateModule], templateUrl: './ts-configurable-reward-count-table.component.html', styleUrl: './ts-configurable-reward-count-table.component.scss', changeDetection: ChangeDetectionStrategy.OnPush, @@ -26,5 +27,6 @@ export class TSConfigurableRewardCountTableComponent { 'failures', 'failurePrior', 'failurePosterior', + 'estimatedWeight', ]; } diff --git a/packages/frontend/projects/upgrade/src/assets/i18n/en.json b/packages/frontend/projects/upgrade/src/assets/i18n/en.json index 4b015c586d..077bc9fb1a 100644 --- a/packages/frontend/projects/upgrade/src/assets/i18n/en.json +++ b/packages/frontend/projects/upgrade/src/assets/i18n/en.json @@ -183,7 +183,6 @@ "home.new-experiment.design.condition.column-header.alias.text": "ALIAS", "home.new-experiment.design.condition.column-header.weight.text": "Weight(%)", "home.new-experiment.design.condition.column-header.include.text": "INCLUDE", - "home.new-experiment.design.mooclet-policy-parameters.header.text": "Mooclet Policy Parameters", "home.new-experiment.design.ts-configurable-policy.section-label.text": "Thompson Sampling (Configurable) Policy Parameters", "home.new-experiment.design.ts-configurable-policy.prior-success.label.text": "Prior Success", "home.new-experiment.design.ts-configurable-policy.prior-success.hint.text": "Prior success count for Beta distribution (default: 1)", @@ -195,13 +194,13 @@ "home.new-experiment.design.ts-configurable-policy.batch-size.label.text": "Batch Size", "home.new-experiment.design.ts-configurable-policy.batch-size.hint.text": "Number of rewards to collect before updating the model (default: 1)", "home.new-experiment.design.ts-configurable-policy.batch-size.min-error.text": "Batch size must be at least 1", - "home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text": "Uniform Threshold", - "home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text": "Start experiment with uniform random distribution until this number of rewards collected (default: 0)", - "home.new-experiment.design.ts-configurable-policy.uniform-threshold.min-error.text": "Uniform threshold must be 0 or higher", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text": "TS Post Diff Threshold", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.hint.text": "Thompson Sampling posterior difference threshold", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.min-error.text": "TS Post Diff threshold must be 0.0 or higher.", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.max-error.text": "TS Post Diff threshold must be 1.0 or lower.", + "home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text": "Warmup Threshold", + "home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text": "Use uniform random selection until total enrollments exceed this count (default: 0)", + "home.new-experiment.design.ts-configurable-policy.uniform-threshold.min-error.text": "Warmup threshold must be 0 or higher", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text": "Minimum Draw Difference", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.hint.text": "Fall back to uniform selection when the top two sampled draws differ by less than this value (0.0–1.0, default: 0)", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.min-error.text": "Minimum draw difference must be 0.0 or higher.", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.max-error.text": "Minimum draw difference must be 1.0 or lower.", "home.new-experiment.design.ts-configurable-policy.general.max-error.text": "Please enter a value less than 1,000,000.", "home.new-experiment.design.ts-configurable-policy.general.integer-error.text": "Please enter a whole number (integer) value.", "home.new-experiment.metrics.text": "Metrics", @@ -273,7 +272,6 @@ "home.view-experiment.experiment-payloads.column-label.target": "TARGET", "home.view-experiment.experiment-payloads.column-label.condition": "CONDITION", "home.view-experiment.experiment-payloads.column-label.payload": "PAYLOAD", - "home.view-experiment.experiment-mooclet-policy-parameters-title.text": "Mooclet Policy Parameters", "home.view-experiment.metrics-analysis-unavailable.text": "Experiment metrics analysis has been temporarily disabled in this view. This information is still available when exporting the experiment data.", "home.view-experiment.graph-type.text": "Type", "home.view-experiment.graph-conditions.text": "Conditions", @@ -583,6 +581,8 @@ "experiments.details.posteriors.posterior.text": "Posterior", "experiments.details.posteriors.failures.text": "Failures", "experiments.details.posteriors.percentOfTotalRewards.text": "% of All Rewards Received", + "experiments.details.posteriors.estimated-weight.text": "Est. Weight", + "experiments.details.posteriors.estimated-weight-tooltip.text": "Estimated % of Thompson Sampling draws this condition would win based on current posteriors (10,000 simulated draws).", "experiments.details.posteriors.no-data-row.text": "Experiment has not begun to collect feedback.", "experiments.upsert-include-list-modal.name-hint.text": "The name for this include list.", "experiments.upsert-exclude-list-modal.name-hint.text": "The name for this exclude list.", diff --git a/packages/frontend/projects/upgrade/src/environments/environment-types.ts b/packages/frontend/projects/upgrade/src/environments/environment-types.ts index f4f5964f5a..75ab1ecf33 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment-types.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment-types.ts @@ -67,8 +67,8 @@ export interface APIEndpoints { addSegmentList: string; getGroupAssignmentStatus: string; stratification: string; - getMoocletRewardsData: string; featureFlagGraphInfo: string; + experimentsRewardsSummary: string; } export interface Environment { @@ -84,7 +84,6 @@ export interface Environment { errorLogsToggle: boolean; withinSubjectExperimentSupportToggle: boolean; metricAnalyticsExperimentDisplayToggle: boolean; - moocletToggle: boolean; // these have been removed but optional to prevent annoyance switching between branches pollingEnabled?: boolean; pollingInterval?: number; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts b/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts index 1f3da7bdcf..fbfed33a10 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: false, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts b/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts index b5c82d145f..2db41ea2c5 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: false, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts b/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts index f36fe056a5..b0f70ccc87 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts @@ -18,5 +18,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: false, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.prod.ts b/packages/frontend/projects/upgrade/src/environments/environment.prod.ts index 7aedb5d89b..68c48e299b 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.prod.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.prod.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.qa.ts b/packages/frontend/projects/upgrade/src/environments/environment.qa.ts index d8af43b3bd..0b92b5eb13 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment.qa.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.qa.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: true, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.staging.ts b/packages/frontend/projects/upgrade/src/environments/environment.staging.ts index a46f5d2f20..f2a2e31eb0 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment.staging.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.staging.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: true, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.ts b/packages/frontend/projects/upgrade/src/environments/environment.ts index dccf995fff..06ca6e8225 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.ts @@ -18,5 +18,4 @@ export const environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/types/CLAUDE.md b/packages/types/CLAUDE.md index 7cc12cf753..aafff82dba 100644 --- a/packages/types/CLAUDE.md +++ b/packages/types/CLAUDE.md @@ -19,7 +19,6 @@ src/ Experiment/ enums.ts # all enums + display map constants interfaces.ts # all interfaces - Mooclet/ # Mooclet-specific types and policy schemas index.ts # single re-export point for everything ``` diff --git a/packages/types/src/Experiment/enums.ts b/packages/types/src/Experiment/enums.ts index 6675a08996..3980bb971e 100644 --- a/packages/types/src/Experiment/enums.ts +++ b/packages/types/src/Experiment/enums.ts @@ -37,13 +37,13 @@ export const CONDITION_ORDER_DISPLAY_MAP = { export enum ASSIGNMENT_ALGORITHM { RANDOM = 'random', STRATIFIED_RANDOM_SAMPLING = 'stratified random sampling', - MOOCLET_TS_CONFIGURABLE = 'ts_configurable', + THOMPSON_SAMPLING = 'thompson_sampling', } export const ASSIGNMENT_ALGORITHM_DISPLAY_MAP = { [ASSIGNMENT_ALGORITHM.RANDOM]: 'Random', [ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING]: 'Stratified Random Sampling', - [ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE]: 'Thompson Sampling (Configurable)', + [ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING]: 'Thompson Sampling', }; export enum POST_EXPERIMENT_RULE { @@ -116,8 +116,6 @@ export enum SERVER_ERROR { MISSING_HEADER_USER_ID = 'Missing `User-Id` header', SEGMENT_DUPLICATE_NAME = 'Segment with same name already exists for this app-context.', INVALID_APP_CONTEXT = 'Invalid app context', - MOOCLET_REWARD_ERROR = 'Error processing Mooclet reward', - MOOCLET_ERROR = 'Mooclet synchronization error', } export enum MARKED_DECISION_POINT_STATUS { diff --git a/packages/types/src/Experiment/interfaces.ts b/packages/types/src/Experiment/interfaces.ts index 288d7571a0..a1446e3dea 100644 --- a/packages/types/src/Experiment/interfaces.ts +++ b/packages/types/src/Experiment/interfaces.ts @@ -333,3 +333,32 @@ export interface DuplicateSegmentNameError { context: string; httpCode: 400; } + +export interface Prior { + success: number; + failure: number; +} + +export enum BinaryRewardAllowedValue { + SUCCESS = 'SUCCESS', + FAILURE = 'FAILURE', +} + +export const BinaryRewardValueMap = { + [BinaryRewardAllowedValue.SUCCESS]: 1, + [BinaryRewardAllowedValue.FAILURE]: 0, +}; + +export interface ExperimentRewardsByCondition { + conditionCode: string; + successes: number; + failures: number; + successRate: string; + order: number; + priorSuccess?: number; + priorFailure?: number; + /** Estimated probability of winning a Thompson Sampling draw, as an integer percentage (0–100). */ + estimatedWeight?: number; +} + +export type ExperimentRewardsSummary = Array; diff --git a/packages/types/src/Mooclet/MoocletPolicyParametersDTO.ts b/packages/types/src/Mooclet/MoocletPolicyParametersDTO.ts deleted file mode 100644 index bddc7c8d7f..0000000000 --- a/packages/types/src/Mooclet/MoocletPolicyParametersDTO.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { IsDefined, IsIn } from 'class-validator'; -import { ASSIGNMENT_ALGORITHM } from '../Experiment/enums'; - -// this will be a Union type of all possible policy parameters once more are introduced -export abstract class MoocletPolicyParametersDTO { - @IsDefined() - @IsIn(Object.values(ASSIGNMENT_ALGORITHM)) - assignmentAlgorithm: ASSIGNMENT_ALGORITHM; -} diff --git a/packages/types/src/Mooclet/MoocletTSConfigurablePolicyParametersDTO.ts b/packages/types/src/Mooclet/MoocletTSConfigurablePolicyParametersDTO.ts deleted file mode 100644 index 2673c589cb..0000000000 --- a/packages/types/src/Mooclet/MoocletTSConfigurablePolicyParametersDTO.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { IsNumber, IsString, ValidateNested, IsOptional, IsObject, IsDefined } from 'class-validator'; -import { Type } from 'class-transformer'; -import { MoocletPolicyParametersDTO } from './MoocletPolicyParametersDTO'; - -export class Prior { - @IsDefined() - @IsNumber() - @Type(() => Number) - failure = 1; - - @IsDefined() - @IsNumber() - @Type(() => Number) - success = 1; -} - -export class CurrentPosteriors { - @IsNumber() - @Type(() => Number) - failures = 0; - - @IsNumber() - @Type(() => Number) - successes = 0; -} - -export class MoocletTSConfigurablePolicyParametersDTO extends MoocletPolicyParametersDTO { - @IsOptional() - @IsObject() - prior?: Record; - - @IsOptional() - @IsObject() - @ValidateNested({ each: true }) - @Type(() => CurrentPosteriors) - current_posteriors?: Record; - - @IsNumber() - batch_size = 1; - - @IsNumber() - max_rating = 1; - - @IsNumber() - min_rating = 0; - - @IsNumber() - uniform_threshold = 0; - - @IsNumber() - tspostdiff_thresh = 0; - - @IsOptional() - @IsString() - outcome_variable_name?: string; -} diff --git a/packages/types/src/Mooclet/index.ts b/packages/types/src/Mooclet/index.ts deleted file mode 100644 index c535e3ad48..0000000000 --- a/packages/types/src/Mooclet/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import 'reflect-metadata'; -import { ASSIGNMENT_ALGORITHM } from '../Experiment/enums'; -import { - CurrentPosteriors, - MoocletTSConfigurablePolicyParametersDTO, - Prior, -} from './MoocletTSConfigurablePolicyParametersDTO'; -import { MoocletPolicyParametersDTO } from './MoocletPolicyParametersDTO'; - -const MOOCLET_POLICY_SCHEMA_MAP = { - [ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE]: MoocletTSConfigurablePolicyParametersDTO, -}; - -const SUPPORTED_MOOCLET_ALGORITHMS = Object.keys(MOOCLET_POLICY_SCHEMA_MAP); - -enum BinaryRewardAllowedValue { - SUCCESS = 'SUCCESS', - FAILURE = 'FAILURE', -} - -const BinaryRewardValueMap = { - [BinaryRewardAllowedValue.SUCCESS]: 1, - [BinaryRewardAllowedValue.FAILURE]: 0, -}; - -interface ExperimentRewardsByCondition { - conditionCode: string; - successes: number; - failures: number; - successRate: string; - order: number; - priorSuccess?: number; - priorFailure?: number; -} - -type ExperimentRewardsSummary = Array; - -export { - MOOCLET_POLICY_SCHEMA_MAP, - SUPPORTED_MOOCLET_ALGORITHMS, - Prior, - CurrentPosteriors, - MoocletPolicyParametersDTO, - MoocletTSConfigurablePolicyParametersDTO, - BinaryRewardAllowedValue, - BinaryRewardValueMap, - ExperimentRewardsSummary, - ExperimentRewardsByCondition, -}; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 4133ce13e3..9720688124 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -96,13 +96,8 @@ export { export { SYSTEM_USER_EMAIL, DEV_USER_EMAIL, FAKE_DEV_CREDENTIAL } from './User'; export { Prior, - CurrentPosteriors, - MoocletPolicyParametersDTO, - MoocletTSConfigurablePolicyParametersDTO, - MOOCLET_POLICY_SCHEMA_MAP, - SUPPORTED_MOOCLET_ALGORITHMS, BinaryRewardAllowedValue, BinaryRewardValueMap, ExperimentRewardsByCondition, ExperimentRewardsSummary, -} from './Mooclet'; +} from './Experiment/interfaces'; diff --git a/postman/ClientAPI.postman_collection.json b/postman/ClientAPI.postman_collection.json index a4e59e1621..cfb6fdd040 100644 --- a/postman/ClientAPI.postman_collection.json +++ b/postman/ClientAPI.postman_collection.json @@ -305,7 +305,7 @@ "host": ["{{baseUrl}}"], "path": ["v6", "reward"] }, - "description": "Send reward signal for adaptive experiments (Mooclets) using direct experiment ID lookup. Reward values: 'SUCCESS' or 'FAILURE'." + "description": "Send reward signal for adaptive (Thompson Sampling) experiments using direct experiment ID lookup. Reward values: 'SUCCESS' or 'FAILURE'." } }, { From 9e0be5225f55e530e50d2014a2e2d19f9eaed7fa Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:23:23 -0400 Subject: [PATCH 02/28] ensure priors won't be silently dropped on update Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../ThompsonSamplingExperimentCrudService.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index ef4a4db03f..9a01c4ce8a 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -65,6 +65,27 @@ export class ThompsonSamplingExperimentCrudService { batchSize: params.batchSize ?? null, } ); + + if (!params.priors) { + return; + } + + const config = await this.configRepository.findByExperimentId(experimentId); + if (!config) { + return; + } + + await Promise.all( + Object.entries(params.priors).map(([conditionId, prior]) => + this.posteriorStateRepository.update( + { configId: config.id, conditionId }, + { + priorSuccess: prior?.success ?? 1, + priorFailure: prior?.failure ?? 1, + } + ) + ) + ); } /** From 7ac9488b77e98adaefca8e065ef4b8d2a671e07e Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:24:06 -0400 Subject: [PATCH 03/28] removed unused import Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../ts-configurable-policy-parameters-form.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts index a45a5175fa..0217b52ee8 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts @@ -3,7 +3,7 @@ import { FormBuilder, FormGroup, ReactiveFormsModule } from '@angular/forms'; import { MatFormFieldModule } from '@angular/material/form-field'; import { MatInputModule } from '@angular/material/input'; import { TranslateModule } from '@ngx-translate/core'; -import { BehaviorSubject, map, Observable, of, startWith, Subject, Subscription } from 'rxjs'; +import { BehaviorSubject, map, Observable, startWith, Subject, Subscription } from 'rxjs'; import { ThompsonSamplingConfigDTO } from '../../../../../../core/experiments/store/experiments.model'; import { EditableThompsonSamplingConfig, From 5c50be055416dd00426b2e7e3780030c055013e6 Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:25:41 -0400 Subject: [PATCH 04/28] use utility that already exists Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../experiment-details-page-content.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts index dc62bf93ac..8643e0a06f 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts @@ -91,7 +91,7 @@ export class ExperimentDetailsPageContentComponent implements OnInit, OnDestroy if (!experiment) { return false; } - return experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment.assignmentAlgorithm); }) ); } From 8de23e895db2d3fab8ce1bb5a9a7254d0bf3a60a Mon Sep 17 00:00:00 2001 From: doswalt Date: Wed, 2 Sep 2026 13:11:20 -0400 Subject: [PATCH 05/28] refine how reward counts are used, actually wire up batch-size --- CLAUDE.md | 12 +- packages/backend/src/api/DTO/ExperimentDTO.ts | 79 +++++- .../api/controllers/ExperimentController.ts | 6 + .../src/api/models/ConditionPosteriorState.ts | 18 ++ .../ThompsonSamplingExperimentConfig.ts | 2 +- .../services/ExperimentAssignmentService.ts | 5 +- .../ThompsonSamplingExperimentCrudService.ts | 4 +- .../services/ThompsonSamplingRewardService.ts | 63 ++++- .../api/services/ThompsonSamplingService.ts | 29 ++- .../1781308800000-cleanupMoocletEntities.ts | 46 ---- ...200000-bootstrapThompsonSamplingConfigs.ts | 44 ---- ...> 1788362726319-nativeThompsonSampling.ts} | 69 ++++- .../ThompsonSamplingRewardService.test.ts | 238 ++++++++++++++++++ .../services/ThompsonSamplingService.test.ts | 28 +-- .../store/experiments.effects.spec.ts | 74 ++++++ 15 files changed, 573 insertions(+), 144 deletions(-) delete mode 100644 packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts delete mode 100644 packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts rename packages/backend/src/database/migrations/{1781222400000-thompsonSamplingEntities.ts => 1788362726319-nativeThompsonSampling.ts} (63%) create mode 100644 packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 4989900d09..61862c81e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,7 +97,7 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - [x] New entity: `ThompsonSamplingReward` (raw reward events — audit trail + recalculation) - [x] Repositories: `ThompsonSamplingExperimentConfigRepository`, `ConditionPosteriorStateRepository` - [x] Add `ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING = 'thompson_sampling'` to `upgrade_types` -- [x] DB migration: `1781222400000-thompsonSamplingEntities` +- [x] DB migration: consolidated into `1788362726319-nativeThompsonSampling` (see Phase 6 note) **Phase 3 — Assignment integration** - [x] Wire `ThompsonSamplingService` into `ExperimentAssignmentService` — `assignThompsonSampling()` method handles THOMPSON_SAMPLING algorithm @@ -121,7 +121,7 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - [x] Deleted: `mooclet-helper.service.ts`, `mooclet-helper.service.spec.ts`; rewards effect tests removed **Phase 6 — Cleanup** -- [x] DB migration `1781308800000-cleanupMoocletEntities`: data-migrates `ts_configurable` → `thompson_sampling`, drops mooclet tables, removes `ts_configurable` from enum +- [x] DB migration `1788362726319-nativeThompsonSampling`: data-migrates `ts_configurable` → `thompson_sampling`, drops mooclet tables, removes `ts_configurable` from enum, creates the three Thompson Sampling tables (including `pendingSuccessCount`/`pendingFailureCount`/`pendingTotalCount` for batched posterior updates) and bootstraps config/posterior rows. This single migration replaces what were originally four separate ones (`thompsonSamplingEntities`, `cleanupMoocletEntities`, `bootstrapThompsonSamplingConfigs`, and a later `addPendingRewardCountsToConditionPosteriorState`) — none had shipped to dev or been applied to any real database, so they were squashed rather than kept as sequential steps. Its timestamp is deliberately ordered after `1783627365221-experimentPrecomputedSegment`, the newest migration actually merged to dev as of this branch's last rebase — check `git log origin/dev -- packages/backend/src/database/migrations` before merging in case dev has moved further; this migration's timestamp must stay the newest of the two. - [x] Removed `MOOCLETS_*` env vars from `env.ts` and `.env.example` - [x] `upgrade_types`: removed `Mooclet/` directory, `MOOCLET_TS_CONFIGURABLE` enum value, `SUPPORTED_MOOCLET_ALGORITHMS`; `Prior`, `BinaryRewardAllowedValue`, `ExperimentRewardsSummary` moved to `Experiment/interfaces.ts` - [x] Deleted: mooclet models, repository, error class, types file @@ -135,6 +135,14 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - **Form/overview labels still showing old terms**: Updated `packages/frontend/projects/upgrade/src/assets/i18n/en.json` — `"uniform-threshold.label.text"` → "Warmup Threshold", `"tspostdiff-thresh.label.text"` → "Minimum Draw Difference" (with updated hints). The translation keys themselves are unchanged; only the values were updated. +- **`thompsonSamplingConfig` request validation was missing**: `ExperimentDTO.thompsonSamplingConfig` was an unvalidated inline object type, so negative/NaN priors or thresholds could reach `ThompsonSamplingService`'s Beta/Gamma sampling math. Replaced with a `ThompsonSamplingConfigValidator` class (`@ValidateNested()` + `@Type()`), bounds mirrored from `ThompsonSamplingHelperService.getFieldValidators()`/`getPriorFieldValidators()` on the frontend: `warmupThreshold`/`batchSize` are integers in `1`–`1,000,000` (`warmupThreshold` allows `0`), `minimumDrawDifference` is `0`–`1`, and `priors` (a `Record`) is validated via a custom `IsThompsonSamplingPriorsRecord` decorator requiring integer `success`/`failure` in `1`–`1,000,000` — strictly positive, since they feed `alpha`/`beta` into `sampleGamma()`. + +- **`warmupThreshold` was gated on the wrong count**: `assignThompsonSampling()` computed `totalEnrollments` as `sum(ConditionPosteriorState.totalCount)`, but `totalCount` only increments on reward (`ThompsonSamplingRewardService`), never on assignment — so the variable name lied about what it measured. Renamed to `totalRewardCount` throughout (`ThompsonSamplingService.ts`, `ExperimentAssignmentService.ts`, `ThompsonSamplingExperimentConfig.ts`). This is also the semantically correct thing to gate on: the posteriors (`alpha`/`beta`) only move when rewards arrive, so warmup should measure how much reward evidence has accumulated, not how many users were assigned — gating on assignment count would exit warmup while posteriors are still sitting at the prior, or (in the low-latency-reward case) too early relative to actual evidence. The MoocLet reference engine gated on assignment count (`variable__name="version"` in `policies.py`), but that behavior was never a requirement here — CLAUDE.md already says not to copy logic from that reference — and gating on rewards is the better design regardless. + +- **`batchSize` wired in**: was stored on `ThompsonSamplingExperimentConfig` and exposed in the UI form but never consumed anywhere. `ConditionPosteriorState` gained `pendingSuccessCount`/`pendingFailureCount`/`pendingTotalCount` columns; `ThompsonSamplingRewardService.applyOrBufferReward()` now buffers rewards there and only folds them into `successCount`/`failureCount`/`totalCount` (the values that drive the posteriors used for sampling) once `batchSize` observations have accumulated. `batchSize` unset or `≤1` applies immediately (matches pre-batching behavior). The raw `ThompsonSamplingReward` audit row is always written regardless of batching — nothing is lost, batching only delays when a reward affects assignment. + +- **`failureCount`/`pendingFailureCount` are stored, not derived**: `ConditionPosteriorState` now stores both counts explicitly rather than computing failures as `totalCount - successCount` (and `pendingFailureCount` as `pendingTotalCount - pendingSuccessCount`) at every call site. All three prior read sites were updated to read the column directly: `ThompsonSamplingExperimentCrudService.getRewardsSummary()` (`failures`/`beta`), `ThompsonSamplingService.selectCondition()` (`beta`, via a new `failureCount` field on `ConditionRewardSummary`), and `ExperimentAssignmentService.assignThompsonSampling()` (populates `ConditionRewardSummary.failureCount` from `state.failureCount` when building reward summaries for assignment). + ### Architecture notes - **conditionId as algorithm key**: `ThompsonSamplingService.selectCondition()` uses condition UUIDs (not `conditionCode`) as identifiers, since `conditionCode` is nullable. `ConditionPosteriorState` rows are keyed by `conditionId`. The `priors` field in `ThompsonSamplingConfigDTO` is therefore also keyed by conditionId. diff --git a/packages/backend/src/api/DTO/ExperimentDTO.ts b/packages/backend/src/api/DTO/ExperimentDTO.ts index 8120c75cea..0dfd2a7773 100644 --- a/packages/backend/src/api/DTO/ExperimentDTO.ts +++ b/packages/backend/src/api/DTO/ExperimentDTO.ts @@ -11,6 +11,8 @@ import { IsOptional, IsString, IsUUID, + Max, + Min, ValidateIf, ValidateNested, ValidationArguments, @@ -495,6 +497,74 @@ function IsAssignmentUnitGroupConsistent(validationOptions?: ValidationOptions) }; } +const MAX_NUMBER_INPUT = 1_000_000; +const MIN_PRIOR_VALUE = 1; + +function IsThompsonSamplingPriorsRecord(validationOptions?: ValidationOptions) { + return function (object: any, propertyName: string) { + registerDecorator({ + name: 'isThompsonSamplingPriorsRecord', + target: object.constructor, + propertyName: propertyName, + options: validationOptions, + validator: { + validate(value: any) { + if (value === undefined || value === null) { + return true; + } + if (typeof value !== 'object' || Array.isArray(value)) { + return false; + } + return Object.values(value).every((prior: any) => { + if (typeof prior !== 'object' || prior === null) { + return false; + } + const { success, failure } = prior; + return ( + Number.isInteger(success) && + success >= MIN_PRIOR_VALUE && + success <= MAX_NUMBER_INPUT && + Number.isInteger(failure) && + failure >= MIN_PRIOR_VALUE && + failure <= MAX_NUMBER_INPUT + ); + }); + }, + defaultMessage() { + return ( + 'Each entry in priors must have integer success/failure values between ' + + `${MIN_PRIOR_VALUE} and ${MAX_NUMBER_INPUT}.` + ); + }, + }, + }); + }; +} + +class ThompsonSamplingConfigValidator { + @IsOptional() + @IsInt() + @Min(0) + @Max(MAX_NUMBER_INPUT) + public warmupThreshold?: number; + + @IsOptional() + @IsNumber() + @Min(0) + @Max(1) + public minimumDrawDifference?: number; + + @IsOptional() + @IsInt() + @Min(1) + @Max(MAX_NUMBER_INPUT) + public batchSize?: number; + + @IsOptional() + @IsThompsonSamplingPriorsRecord() + public priors?: Record; +} + export class ExperimentDTO extends BaseExperimentWithoutPayload { @IsOptional() @IsArray() @@ -503,12 +573,9 @@ export class ExperimentDTO extends BaseExperimentWithoutPayload { public conditionPayloads?: ConditionPayloadValidator[]; @IsOptional() - public thompsonSamplingConfig?: { - warmupThreshold?: number; - minimumDrawDifference?: number; - batchSize?: number; - priors?: Record; - }; + @ValidateNested() + @Type(() => ThompsonSamplingConfigValidator) + public thompsonSamplingConfig?: ThompsonSamplingConfigValidator; } export class OldExperimentDTO extends BaseExperimentWithoutPayload { diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index a839fac5a2..a1300b1677 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -1972,10 +1972,16 @@ export class ExperimentController { if (experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { const config = await this.thompsonSamplingCrudService.getConfigForExperiment(experiment.id); if (config) { + const priors: Record = {}; + (config.conditionPosteriorStates ?? []).forEach((state) => { + priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; + }); + experiment.thompsonSamplingConfig = { warmupThreshold: config.warmupThreshold, minimumDrawDifference: config.minimumDrawDifference, batchSize: config.batchSize, + priors, }; } } diff --git a/packages/backend/src/api/models/ConditionPosteriorState.ts b/packages/backend/src/api/models/ConditionPosteriorState.ts index c4e5689f79..491e9d480a 100644 --- a/packages/backend/src/api/models/ConditionPosteriorState.ts +++ b/packages/backend/src/api/models/ConditionPosteriorState.ts @@ -37,7 +37,25 @@ export class ConditionPosteriorState extends BaseModel { @Column({ type: 'int', default: 0 }) successCount: number; + /** Accumulated failures since the experiment started. Always equal to totalCount - successCount; + * stored (not derived) so callers can read it directly rather than recomputing it everywhere. */ + @Column({ type: 'int', default: 0 }) + failureCount: number; + /** Total rewards received for this condition (successes + failures). */ @Column({ type: 'int', default: 0 }) totalCount: number; + + /** Successes recorded since the last batch flush; folded into successCount once batchSize is reached. */ + @Column({ type: 'int', default: 0 }) + pendingSuccessCount: number; + + /** Failures recorded since the last batch flush; folded into failureCount once batchSize is reached. + * Always equal to pendingTotalCount - pendingSuccessCount; stored for the same reason as failureCount. */ + @Column({ type: 'int', default: 0 }) + pendingFailureCount: number; + + /** Rewards recorded since the last batch flush; folded into totalCount once batchSize is reached. */ + @Column({ type: 'int', default: 0 }) + pendingTotalCount: number; } diff --git a/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts index 629994ec88..80bda352d6 100644 --- a/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts +++ b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts @@ -15,7 +15,7 @@ export class ThompsonSamplingExperimentConfig extends BaseModel { @Column({ nullable: true }) experimentId?: string; - /** Use uniform random selection until total enrollments exceed this count. */ + /** Use uniform random selection until total reward observations exceed this count. */ @Column({ nullable: true }) warmupThreshold?: number; diff --git a/packages/backend/src/api/services/ExperimentAssignmentService.ts b/packages/backend/src/api/services/ExperimentAssignmentService.ts index 105f99a915..204e1867cd 100644 --- a/packages/backend/src/api/services/ExperimentAssignmentService.ts +++ b/packages/backend/src/api/services/ExperimentAssignmentService.ts @@ -2094,6 +2094,7 @@ export class ExperimentAssignmentService { const rewardSummaries: ConditionRewardSummary[] = config.conditionPosteriorStates.map((state) => ({ conditionCode: state.conditionId, successCount: state.successCount, + failureCount: state.failureCount, totalCount: state.totalCount, })); @@ -2102,7 +2103,7 @@ export class ExperimentAssignmentService { priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; }); - const totalEnrollments = config.conditionPosteriorStates.reduce((sum, s) => sum + s.totalCount, 0); + const totalRewardCount = config.conditionPosteriorStates.reduce((sum, s) => sum + s.totalCount, 0); const tsConfig: ThompsonSamplingConfig = { priors, @@ -2113,7 +2114,7 @@ export class ExperimentAssignmentService { const selectedConditionId = this.thompsonSamplingService.selectCondition( conditionIds, rewardSummaries, - totalEnrollments, + totalRewardCount, tsConfig ); diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 9a01c4ce8a..243ee24430 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -104,10 +104,10 @@ export class ThompsonSamplingExperimentCrudService { const rows = config.conditionPosteriorStates.map((state) => { const successes = state.successCount; - const failures = state.totalCount - state.successCount; + const failures = state.failureCount; const successRate = state.totalCount > 0 ? ((successes / state.totalCount) * 100).toFixed(1) + '%' : '0.0%'; const alpha = state.priorSuccess + state.successCount; - const beta = state.priorFailure + (state.totalCount - state.successCount); + const beta = state.priorFailure + state.failureCount; return { code: state.condition?.conditionCode ?? state.conditionId, alpha, diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index 80582b9b6c..a7e49237a3 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -79,11 +79,7 @@ export class ThompsonSamplingRewardService { ); } - // Increment counts atomically; successCount only increments on success - await this.posteriorStateRepository.increment({ id: state.id }, 'totalCount', 1); - if (success) { - await this.posteriorStateRepository.increment({ id: state.id }, 'successCount', 1); - } + await this.applyOrBufferReward(state.id, success, config.batchSize); logger.info({ message: 'Thompson Sampling reward recorded', @@ -104,6 +100,63 @@ export class ThompsonSamplingRewardService { } } + /** + * Fold a reward into the posterior (successCount/totalCount), or buffer it as pending until + * batchSize reward observations have accumulated for this condition. The raw event is always + * persisted to ThompsonSamplingReward regardless of batching — batching only delays when a + * reward affects which condition gets sampled next, it never drops data. + */ + private async applyOrBufferReward(stateId: string, success: boolean, batchSize?: number): Promise { + const effectiveBatchSize = batchSize && batchSize > 1 ? batchSize : 1; + + if (effectiveBatchSize <= 1) { + await this.posteriorStateRepository.increment({ id: stateId }, 'totalCount', 1); + if (success) { + await this.posteriorStateRepository.increment({ id: stateId }, 'successCount', 1); + } else { + await this.posteriorStateRepository.increment({ id: stateId }, 'failureCount', 1); + } + return; + } + + await this.posteriorStateRepository.increment({ id: stateId }, 'pendingTotalCount', 1); + if (success) { + await this.posteriorStateRepository.increment({ id: stateId }, 'pendingSuccessCount', 1); + } else { + await this.posteriorStateRepository.increment({ id: stateId }, 'pendingFailureCount', 1); + } + + const refreshedState = await this.posteriorStateRepository.findOne({ where: { id: stateId } }); + + if (refreshedState.pendingTotalCount >= effectiveBatchSize) { + await this.flushPendingRewards( + stateId, + refreshedState.pendingSuccessCount, + refreshedState.pendingFailureCount, + refreshedState.pendingTotalCount + ); + } + } + + private async flushPendingRewards( + stateId: string, + pendingSuccessCount: number, + pendingFailureCount: number, + pendingTotalCount: number + ): Promise { + await this.posteriorStateRepository.increment({ id: stateId }, 'totalCount', pendingTotalCount); + if (pendingSuccessCount > 0) { + await this.posteriorStateRepository.increment({ id: stateId }, 'successCount', pendingSuccessCount); + } + if (pendingFailureCount > 0) { + await this.posteriorStateRepository.increment({ id: stateId }, 'failureCount', pendingFailureCount); + } + await this.posteriorStateRepository.update( + { id: stateId }, + { pendingSuccessCount: 0, pendingFailureCount: 0, pendingTotalCount: 0 } + ); + } + private async findConfigById( experimentId: string, request: RewardValidator, diff --git a/packages/backend/src/api/services/ThompsonSamplingService.ts b/packages/backend/src/api/services/ThompsonSamplingService.ts index 1d81aea769..73c2e88fa6 100644 --- a/packages/backend/src/api/services/ThompsonSamplingService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingService.ts @@ -8,13 +8,14 @@ export interface ConditionPrior { export interface ConditionRewardSummary { conditionCode: string; successCount: number; + failureCount: number; totalCount: number; } export interface ThompsonSamplingConfig { /** Per-condition Beta distribution priors. Conditions without an entry use DEFAULT_PRIOR. */ priors?: Record; - /** Use uniform random selection until total enrollments exceed this count. */ + /** Use uniform random selection until total reward observations exceed this count. */ warmupThreshold?: number; /** Fall back to uniform when the top two sampled draws differ by less than this value. */ minimumDrawDifference?: number; @@ -24,14 +25,6 @@ export const DEFAULT_PRIOR: ConditionPrior = { success: 1, failure: 1 }; @Service() export class ThompsonSamplingService { - /** - * Select a condition using Thompson Sampling. - * - * @param conditionCodes - All eligible condition codes for this experiment - * @param rewardSummaries - Accumulated reward counts per condition - * @param totalEnrollments - Total number of enrollments across all conditions - * @param config - Optional algorithm parameters (priors, warmup, thresholds) - */ /** * Estimate how often each condition would "win" a Thompson Sampling draw given current posteriors. * @@ -81,10 +74,18 @@ export class ThompsonSamplingService { return result; } + /** + * Select a condition using Thompson Sampling. + * + * @param conditionCodes - All eligible condition codes for this experiment + * @param rewardSummaries - Accumulated reward counts per condition + * @param totalRewardCount - Total number of reward observations across all conditions + * @param config - Optional algorithm parameters (priors, warmup, thresholds) + */ selectCondition( conditionCodes: string[], rewardSummaries: ConditionRewardSummary[], - totalEnrollments: number, + totalRewardCount: number, config: ThompsonSamplingConfig = {} ): string { if (conditionCodes.length === 0) { @@ -94,8 +95,10 @@ export class ThompsonSamplingService { return conditionCodes[0]; } - // Warmup phase: use uniform random until sufficient data has been collected - if (config.warmupThreshold !== undefined && totalEnrollments <= config.warmupThreshold) { + // Warmup phase: use uniform random until sufficient reward evidence has been collected. + // Gated on reward observations (not assignments) — the posteriors only move when rewards + // arrive, so that's the right measure of "how much evidence do we actually have." + if (config.warmupThreshold !== undefined && totalRewardCount <= config.warmupThreshold) { return this.uniformRandom(conditionCodes); } @@ -105,7 +108,7 @@ export class ThompsonSamplingService { const summary = summaryMap.get(code); const prior = config.priors?.[code] ?? DEFAULT_PRIOR; const alpha = prior.success + (summary?.successCount ?? 0); - const beta = prior.failure + (summary ? summary.totalCount - summary.successCount : 0); + const beta = prior.failure + (summary?.failureCount ?? 0); return { code, draw: this.sampleBeta(alpha, beta) }; }); diff --git a/packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts b/packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts deleted file mode 100644 index 5a84039807..0000000000 --- a/packages/backend/src/database/migrations/1781308800000-cleanupMoocletEntities.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class CleanupMoocletEntities1781308800000 implements MigrationInterface { - name = 'CleanupMoocletEntities1781308800000'; - - public async up(queryRunner: QueryRunner): Promise { - // Migrate any existing ts_configurable experiments to thompson_sampling - await queryRunner.query( - `UPDATE "experiment" SET "assignmentAlgorithm" = 'thompson_sampling' WHERE "assignmentAlgorithm" = 'ts_configurable'` - ); - - // Drop mooclet tables (cascade handles FK references) - await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_version_condition_map"`); - await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_experiment_ref"`); - - // Remove ts_configurable from the assignment algorithm enum - await queryRunner.query( - `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` - ); - await queryRunner.query( - `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'thompson_sampling')` - ); - await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); - await queryRunner.query( - `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` - ); - await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); - await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); - } - - public async down(queryRunner: QueryRunner): Promise { - // Restore ts_configurable enum value - await queryRunner.query( - `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` - ); - await queryRunner.query( - `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'ts_configurable', 'thompson_sampling')` - ); - await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); - await queryRunner.query( - `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` - ); - await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); - await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); - } -} diff --git a/packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts b/packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts deleted file mode 100644 index a305917084..0000000000 --- a/packages/backend/src/database/migrations/1781395200000-bootstrapThompsonSamplingConfigs.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class BootstrapThompsonSamplingConfigs1781395200000 implements MigrationInterface { - name = 'BootstrapThompsonSamplingConfigs1781395200000'; - - public async up(queryRunner: QueryRunner): Promise { - // Create ThompsonSamplingExperimentConfig rows for any thompson_sampling experiments - // that don't have one yet (e.g. experiments converted from ts_configurable by the cleanup migration). - await queryRunner.query(` - INSERT INTO "thompson_sampling_experiment_config" ("experimentId", "versionNumber") - SELECT e.id, 1 - FROM "experiment" e - WHERE e."assignmentAlgorithm" = 'thompson_sampling' - AND NOT EXISTS ( - SELECT 1 FROM "thompson_sampling_experiment_config" c WHERE c."experimentId" = e.id - ) - `); - - // Create ConditionPosteriorState rows for each condition of those experiments. - await queryRunner.query(` - INSERT INTO "condition_posterior_state" ("configId", "conditionId", "priorSuccess", "priorFailure", "successCount", "totalCount", "versionNumber") - SELECT c.id, ec.id, 1, 1, 0, 0, 1 - FROM "thompson_sampling_experiment_config" c - JOIN "experiment_condition" ec ON ec."experimentId" = c."experimentId" - WHERE NOT EXISTS ( - SELECT 1 FROM "condition_posterior_state" ps - WHERE ps."configId" = c.id AND ps."conditionId" = ec.id - ) - `); - } - - public async down(queryRunner: QueryRunner): Promise { - // Remove posterior states and configs that were created by this migration. - // We identify "bootstrapped" rows as those with no warmupThreshold/minimumDrawDifference/batchSize - // (all nullable, all NULL means they came from this migration with defaults only). - // This is a best-effort rollback — if configs were subsequently edited, those edits are lost. - await queryRunner.query(` - DELETE FROM "thompson_sampling_experiment_config" - WHERE "warmupThreshold" IS NULL - AND "minimumDrawDifference" IS NULL - AND "batchSize" IS NULL - `); - } -} diff --git a/packages/backend/src/database/migrations/1781222400000-thompsonSamplingEntities.ts b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts similarity index 63% rename from packages/backend/src/database/migrations/1781222400000-thompsonSamplingEntities.ts rename to packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts index 8a35988a53..bff994353f 100644 --- a/packages/backend/src/database/migrations/1781222400000-thompsonSamplingEntities.ts +++ b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts @@ -1,23 +1,44 @@ import { MigrationInterface, QueryRunner } from 'typeorm'; -export class ThompsonSamplingEntities1781222400000 implements MigrationInterface { - name = 'ThompsonSamplingEntities1781222400000'; +/** + * Consolidates what was originally three (then four) separate migrations for the native Thompson + * Sampling feature (thompsonSamplingEntities, cleanupMoocletEntities, bootstrapThompsonSamplingConfigs, + * addPendingRewardCountsToConditionPosteriorState) into one. None of those had shipped to dev or been + * applied to any real database — they only ever existed on this unmerged branch — so there was no + * reason to keep them as separate steps; consolidating avoids interleaving with migrations that *have* + * since been merged to dev (remove-twoCharacterId, featureFlagPrecomputedSegment, + * experimentPrecomputedSegment), which this migration's timestamp is intentionally ordered after. + */ +export class NativeThompsonSampling1788362726319 implements MigrationInterface { + name = 'NativeThompsonSampling1788362726319'; public async up(queryRunner: QueryRunner): Promise { - // Add thompson_sampling to the assignment algorithm enum + // Swap 'ts_configurable' for 'thompson_sampling' in the assignment algorithm enum. Existing rows + // are remapped in the same USING expression that performs the type conversion, so there's no + // intermediate enum state where both values coexist. await queryRunner.query( `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` ); await queryRunner.query( - `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'ts_configurable', 'thompson_sampling')` + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'thompson_sampling')` ); await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); - await queryRunner.query( - `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` - ); + await queryRunner.query(` + ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" + USING ( + CASE "assignmentAlgorithm"::text + WHEN 'ts_configurable' THEN 'thompson_sampling' + ELSE "assignmentAlgorithm"::text + END + )::"public"."experiment_assignmentalgorithm_enum" + `); await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); + // Drop mooclet tables (cascade handles FK references) + await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_version_condition_map"`); + await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_experiment_ref"`); + // thompson_sampling_experiment_config: one-to-one with experiment await queryRunner.query( `CREATE TABLE "thompson_sampling_experiment_config" ( @@ -34,7 +55,9 @@ export class ThompsonSamplingEntities1781222400000 implements MigrationInterface )` ); - // condition_posterior_state: per-condition Beta distribution state + // condition_posterior_state: per-condition Beta distribution state. pendingSuccessCount/ + // pendingTotalCount buffer rewards between batch flushes (see ThompsonSamplingRewardService) — + // included from the start since nothing has been applied anywhere yet. await queryRunner.query( `CREATE TABLE "condition_posterior_state" ( "id" uuid NOT NULL DEFAULT uuid_generate_v4(), @@ -43,7 +66,11 @@ export class ThompsonSamplingEntities1781222400000 implements MigrationInterface "priorSuccess" double precision NOT NULL DEFAULT 1, "priorFailure" double precision NOT NULL DEFAULT 1, "successCount" integer NOT NULL DEFAULT 0, + "failureCount" integer NOT NULL DEFAULT 0, "totalCount" integer NOT NULL DEFAULT 0, + "pendingSuccessCount" integer NOT NULL DEFAULT 0, + "pendingFailureCount" integer NOT NULL DEFAULT 0, + "pendingTotalCount" integer NOT NULL DEFAULT 0, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), "versionNumber" integer NOT NULL, @@ -85,6 +112,24 @@ export class ThompsonSamplingEntities1781222400000 implements MigrationInterface await queryRunner.query( `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_condition" FOREIGN KEY ("conditionId") REFERENCES "experiment_condition"("id") ON DELETE CASCADE ON UPDATE NO ACTION` ); + + // Bootstrap config + posterior state rows for any experiment already flagged thompson_sampling + // (including ones just remapped from ts_configurable above). Both tables are brand new in this + // migration, so every such experiment necessarily lacks rows — no NOT EXISTS guard needed. + await queryRunner.query(` + INSERT INTO "thompson_sampling_experiment_config" ("experimentId", "versionNumber") + SELECT e.id, 1 + FROM "experiment" e + WHERE e."assignmentAlgorithm" = 'thompson_sampling' + `); + + await queryRunner.query(` + INSERT INTO "condition_posterior_state" + ("configId", "conditionId", "priorSuccess", "priorFailure", "successCount", "failureCount", "totalCount", "pendingSuccessCount", "pendingFailureCount", "pendingTotalCount", "versionNumber") + SELECT c.id, ec.id, 1, 1, 0, 0, 0, 0, 0, 0, 1 + FROM "thompson_sampling_experiment_config" c + JOIN "experiment_condition" ec ON ec."experimentId" = c."experimentId" + `); } public async down(queryRunner: QueryRunner): Promise { @@ -101,7 +146,13 @@ export class ThompsonSamplingEntities1781222400000 implements MigrationInterface await queryRunner.query(`DROP TABLE "condition_posterior_state"`); await queryRunner.query(`DROP TABLE "thompson_sampling_experiment_config"`); - // Remove thompson_sampling from the enum + // Best-effort: the pre-migration enum has no 'thompson_sampling' value, so any experiment left in + // that state would fail the column cast below. Fall back to 'random' — this is a rollback of a + // feature that was never live, not a data-preserving downgrade. + await queryRunner.query( + `UPDATE "experiment" SET "assignmentAlgorithm" = 'random' WHERE "assignmentAlgorithm" = 'thompson_sampling'` + ); + await queryRunner.query( `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` ); diff --git a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts new file mode 100644 index 0000000000..49fc34727f --- /dev/null +++ b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts @@ -0,0 +1,238 @@ +import { ThompsonSamplingRewardService } from '../../../src/api/services/ThompsonSamplingRewardService'; +import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; +import { BinaryRewardAllowedValue, EXPERIMENT_STATE } from 'upgrade_types'; +import { RewardValidator } from '../../../src/api/controllers/validators/RewardValidator'; +import { RequestedExperimentUser } from '../../../src/api/controllers/validators/ExperimentUserValidator'; +import { configureLogger } from '../../utils/logger'; + +const logger = new UpgradeLogger(); + +const EXPERIMENT_ID = 'experiment-1'; +const CONDITION_ID = 'condition-1'; +const USER_ID = 'user-1'; + +function makeUser(): RequestedExperimentUser { + return { id: USER_ID, requestedUserId: USER_ID } as RequestedExperimentUser; +} + +function makeRequest(rewardValue: BinaryRewardAllowedValue = BinaryRewardAllowedValue.SUCCESS): RewardValidator { + return { experimentId: EXPERIMENT_ID, rewardValue } as RewardValidator; +} + +describe('ThompsonSamplingRewardService', () => { + beforeAll(() => { + configureLogger(); + }); + + let tsRewardRepository: any; + let posteriorStateRepository: any; + let tsConfigRepository: any; + let individualEnrollmentRepository: any; + let service: ThompsonSamplingRewardService; + + // In-memory posterior state row, mutated by the mocked increment/update calls so + // assertions can inspect the final counts after one or more recordReward() calls. + let state: { + id: string; + successCount: number; + failureCount: number; + totalCount: number; + pendingSuccessCount: number; + pendingFailureCount: number; + pendingTotalCount: number; + }; + + function makeConfig(batchSize?: number) { + return { + experimentId: EXPERIMENT_ID, + batchSize, + experiment: { state: EXPERIMENT_STATE.ENROLLING }, + }; + } + + beforeEach(() => { + state = { + id: 'state-1', + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingSuccessCount: 0, + pendingFailureCount: 0, + pendingTotalCount: 0, + }; + + tsRewardRepository = { save: jest.fn().mockResolvedValue(undefined) }; + + posteriorStateRepository = { + findByConditionId: jest.fn().mockResolvedValue(state), + increment: jest.fn((criteria: { id: string }, column: keyof typeof state, amount: number) => { + (state as any)[column] += amount; + return Promise.resolve(undefined); + }), + findOne: jest.fn().mockImplementation(() => Promise.resolve({ ...state })), + update: jest.fn((criteria: { id: string }, partial: Partial) => { + Object.assign(state, partial); + return Promise.resolve(undefined); + }), + }; + + tsConfigRepository = { + findOne: jest.fn().mockResolvedValue(makeConfig()), + findByDecisionPoint: jest.fn().mockResolvedValue([]), + }; + + individualEnrollmentRepository = { + findEnrollments: jest.fn().mockResolvedValue([{ conditionId: CONDITION_ID }]), + }; + + service = new ThompsonSamplingRewardService( + tsRewardRepository, + posteriorStateRepository, + tsConfigRepository, + individualEnrollmentRepository + ); + }); + + describe('warmup threshold (reward count)', () => { + it('always persists the raw reward event regardless of batching', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(5)); + + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + expect(tsRewardRepository.save).toHaveBeenCalledWith({ + experimentId: EXPERIMENT_ID, + conditionId: CONDITION_ID, + userId: USER_ID, + success: true, + }); + }); + + it('increments totalCount immediately when batchSize is unset (default behavior)', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(undefined)); + + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + expect(state.totalCount).toBe(1); + expect(state.successCount).toBe(1); + expect(state.pendingTotalCount).toBe(0); + }); + + it('increments totalCount immediately when batchSize is 1', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(1)); + + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + + expect(state.totalCount).toBe(1); + expect(state.successCount).toBe(0); + expect(state.failureCount).toBe(1); + expect(state.pendingTotalCount).toBe(0); + }); + }); + + describe('batchSize', () => { + it('buffers rewards as pending until batchSize is reached', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(3)); + + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + expect(state.pendingTotalCount).toBe(1); + expect(state.pendingSuccessCount).toBe(1); + expect(state.pendingFailureCount).toBe(0); + expect(state.totalCount).toBe(0); + expect(state.successCount).toBe(0); + + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + expect(state.pendingTotalCount).toBe(2); + expect(state.pendingSuccessCount).toBe(1); + expect(state.pendingFailureCount).toBe(1); + expect(state.totalCount).toBe(0); + expect(state.successCount).toBe(0); + }); + + it('flushes pending counts into successCount/totalCount once batchSize is reached', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(3)); + + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + expect(state.totalCount).toBe(3); + expect(state.successCount).toBe(2); + expect(state.failureCount).toBe(1); + expect(state.pendingTotalCount).toBe(0); + expect(state.pendingSuccessCount).toBe(0); + expect(state.pendingFailureCount).toBe(0); + }); + + it('resets the pending buffer after a flush so the next batch starts fresh', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(2)); + + // First batch of 2 flushes... + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + expect(state.totalCount).toBe(2); + expect(state.successCount).toBe(2); + + // ...a single reward into the next batch should only be pending, not yet applied. + await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + expect(state.totalCount).toBe(2); + expect(state.successCount).toBe(2); + expect(state.pendingTotalCount).toBe(1); + expect(state.pendingSuccessCount).toBe(0); + expect(state.pendingFailureCount).toBe(1); + }); + + it('keeps pendingFailureCount readable without deriving it — always equals pendingTotalCount - pendingSuccessCount', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(10)); + + const values = [ + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.SUCCESS, + ]; + for (const value of values) { + await service.recordReward(makeUser(), makeRequest(value), logger); + } + + expect(state.pendingSuccessCount).toBe(2); + expect(state.pendingFailureCount).toBe(2); + expect(state.pendingFailureCount).toBe(state.pendingTotalCount - state.pendingSuccessCount); + }); + + it('keeps failureCount readable without deriving it — always equals totalCount - successCount', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(4)); + + const values = [ + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.SUCCESS, + ]; + for (const value of values) { + await service.recordReward(makeUser(), makeRequest(value), logger); + } + + expect(state.successCount).toBe(2); + expect(state.failureCount).toBe(2); + expect(state.failureCount).toBe(state.totalCount - state.successCount); + }); + + it('never drops rewards — total applied + pending always equals rewards recorded', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(4)); + + const values = [ + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + ]; + for (const value of values) { + await service.recordReward(makeUser(), makeRequest(value), logger); + } + + expect(state.totalCount + state.pendingTotalCount).toBe(values.length); + expect(tsRewardRepository.save).toHaveBeenCalledTimes(values.length); + }); + }); +}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts index c2aaca0c8b..41270a7020 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts @@ -47,8 +47,8 @@ describe('ThompsonSamplingService', () => { const conditions = ['A', 'B']; // A has an overwhelmingly dominant posterior; without warmup it would always win const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 1000, totalCount: 1000 }, - { conditionCode: 'B', successCount: 0, totalCount: 1000 }, + { conditionCode: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionCode: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, ]; const config: ThompsonSamplingConfig = { warmupThreshold: 50 }; @@ -60,11 +60,11 @@ describe('ThompsonSamplingService', () => { expect(results.has('B')).toBe(true); }); - it('exits warmup once enrollment exceeds threshold', () => { + it('exits warmup once reward count exceeds threshold', () => { const conditions = ['A', 'B']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 1000, totalCount: 1000 }, - { conditionCode: 'B', successCount: 0, totalCount: 1000 }, + { conditionCode: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionCode: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, ]; const config: ThompsonSamplingConfig = { warmupThreshold: 10 }; @@ -84,8 +84,8 @@ describe('ThompsonSamplingService', () => { it('selects the condition with a better reward history reliably', () => { const conditions = ['good', 'bad']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'good', successCount: 90, totalCount: 100 }, - { conditionCode: 'bad', successCount: 10, totalCount: 100 }, + { conditionCode: 'good', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionCode: 'bad', successCount: 10, failureCount: 90, totalCount: 100 }, ]; let goodCount = 0; @@ -101,10 +101,10 @@ describe('ThompsonSamplingService', () => { it('handles three or more conditions — clear winner dominates', () => { const conditions = ['A', 'B', 'C', 'D']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 5, totalCount: 100 }, - { conditionCode: 'B', successCount: 90, totalCount: 100 }, - { conditionCode: 'C', successCount: 10, totalCount: 100 }, - { conditionCode: 'D', successCount: 5, totalCount: 100 }, + { conditionCode: 'A', successCount: 5, failureCount: 95, totalCount: 100 }, + { conditionCode: 'B', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionCode: 'C', successCount: 10, failureCount: 90, totalCount: 100 }, + { conditionCode: 'D', successCount: 5, failureCount: 95, totalCount: 100 }, ]; let bCount = 0; @@ -158,7 +158,7 @@ describe('ThompsonSamplingService', () => { it('handles a condition with no reward summary entry (defaults to prior)', () => { const conditions = ['A', 'B']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 50, totalCount: 100 }, + { conditionCode: 'A', successCount: 50, failureCount: 50, totalCount: 100 }, // B has no entry — treated as zero rewards, uses prior only ]; @@ -185,8 +185,8 @@ describe('ThompsonSamplingService', () => { it('does not interfere when threshold is zero', () => { const conditions = ['A', 'B']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 90, totalCount: 100 }, - { conditionCode: 'B', successCount: 10, totalCount: 100 }, + { conditionCode: 'A', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionCode: 'B', successCount: 10, failureCount: 90, totalCount: 100 }, ]; const config: ThompsonSamplingConfig = { minimumDrawDifference: 0 }; diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts index 3b6354aacb..9636a92320 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts @@ -48,6 +48,9 @@ import { actionExportExperimentInfoFailure, actionExportExperimentDesign, actionExportExperimentDesignSuccess, + actionFetchRewardsDataForExperiment, + actionFetchRewardsDataForExperimentSuccess, + actionFetchRewardsDataForExperimentFailure, } from './experiments.actions'; import { ExperimentEffects } from './experiments.effects'; import { @@ -1372,4 +1375,75 @@ describe('ExperimentEffects', () => { tick(0); })); }); + + describe('fetchRewardsDataForExperiment$', () => { + it('should dispatch actionFetchRewardsDataForExperimentSuccess on successful fetch', fakeAsync(() => { + const experimentId = 'test-experiment-123'; + const mockRewardsSummary = [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + { + conditionCode: 'Treatment', + successes: 8, + failures: 7, + total: 15, + successRate: '53.3%', + order: 1, + }, + ]; + + experimentDataService.fetchRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); + + const expectedAction = actionFetchRewardsDataForExperimentSuccess({ + experimentId, + rewardsSummary: mockRewardsSummary, + }); + + service.fetchRewardsDataForExperiment$.subscribe((resultingAction) => { + expect(resultingAction).toEqual(expectedAction); + }); + + actions$.next(actionFetchRewardsDataForExperiment({ experimentId })); + + tick(0); + })); + + it('should dispatch actionFetchRewardsDataForExperimentFailure on fetch error', fakeAsync(() => { + const experimentId = 'test-experiment-123'; + const error = new Error('API error'); + + experimentDataService.fetchRewardsDataForExperiment = jest.fn().mockReturnValue(throwError(error)); + + const expectedAction = actionFetchRewardsDataForExperimentFailure({ error }); + + service.fetchRewardsDataForExperiment$.subscribe((resultingAction) => { + expect(resultingAction).toEqual(expectedAction); + }); + + actions$.next(actionFetchRewardsDataForExperiment({ experimentId })); + + tick(0); + })); + + it('should call experimentDataService with correct experimentId', fakeAsync(() => { + const experimentId = 'test-experiment-456'; + const mockRewardsSummary = []; + + experimentDataService.fetchRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); + + service.fetchRewardsDataForExperiment$.subscribe(() => { + expect(experimentDataService.fetchRewardsDataForExperiment).toHaveBeenCalledWith(experimentId); + }); + + actions$.next(actionFetchRewardsDataForExperiment({ experimentId })); + + tick(0); + })); + }); }); From e64d6756f18bb0920fa5419e4bf31052171afecc Mon Sep 17 00:00:00 2001 From: doswalt Date: Thu, 3 Sep 2026 17:05:40 -0400 Subject: [PATCH 06/28] use experiments cache in reward path, send immediate receipt response instead of waiting, update the wording of some things --- CLAUDE.md | 6 + .../ExperimentClientController.v6.ts | 19 +- .../backend/src/api/services/CacheService.ts | 1 + .../services/ExperimentAssignmentService.ts | 8 +- .../ThompsonSamplingExperimentCrudService.ts | 22 +- .../services/ThompsonSamplingRewardService.ts | 219 +++++++----- .../api/services/ThompsonSamplingService.ts | 42 ++- .../test/unit/services/CacheService.test.ts | 2 + .../ExperimentAssignmentService.test.ts | 51 ++- ...mpsonSamplingExperimentCrudService.test.ts | 59 ++++ .../ThompsonSamplingRewardService.test.ts | 326 +++++++++++++++--- .../services/ThompsonSamplingService.test.ts | 32 +- .../upsert-experiment-modal.component.ts | 8 +- .../projects/upgrade/src/assets/i18n/en.json | 3 +- packages/types/src/Experiment/enums.ts | 1 + 15 files changed, 637 insertions(+), 162 deletions(-) create mode 100644 packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 61862c81e1..cde0996f08 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -143,6 +143,12 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - **`failureCount`/`pendingFailureCount` are stored, not derived**: `ConditionPosteriorState` now stores both counts explicitly rather than computing failures as `totalCount - successCount` (and `pendingFailureCount` as `pendingTotalCount - pendingSuccessCount`) at every call site. All three prior read sites were updated to read the column directly: `ThompsonSamplingExperimentCrudService.getRewardsSummary()` (`failures`/`beta`), `ThompsonSamplingService.selectCondition()` (`beta`, via a new `failureCount` field on `ConditionRewardSummary`), and `ExperimentAssignmentService.assignThompsonSampling()` (populates `ConditionRewardSummary.failureCount` from `state.failureCount` when building reward summaries for assignment). +- **`batchSize` was gated per-condition instead of per-experiment**: `applyOrBufferReward()` originally checked `pendingTotalCount` on only the one `ConditionPosteriorState` row the incoming reward belonged to, so `batchSize` behaved as a per-condition threshold — e.g. with `batchSize=5`, 4 rewards on condition A and 1 on condition B would never flush, since neither condition individually reached 5. `batchSize` is meant to pace how often posteriors move for the experiment as a whole, and a reward for any condition is evidence toward that same shared cadence. Fixed by summing `pendingTotalCount` across every `ConditionPosteriorState` row for the experiment (`ConditionPosteriorStateRepository.findByConfigId(state.configId)`) after buffering the incoming reward, and — once that sum reaches `batchSize` — flushing every condition's pending buffer (not just the one that tipped it over), so a low-volume condition still gets its pending counts folded in as soon as the shared batch closes. + +- **`warmupThreshold` undercounted rewards sitting in an unflushed batch**: `ExperimentAssignmentService.assignThompsonSampling()` computed `totalRewardCount` as `sum(ConditionPosteriorState.totalCount)` across conditions, which is correct only when `batchSize` is unset/≤1 (immediate apply). Once batching is active, a reward is "collected" (persisted to `ThompsonSamplingReward`, buffered in `pendingTotalCount`) before it's folded into `totalCount` — so with a large `batchSize`, warmup could stay active far longer than the actual evidence collected would justify, since pending rewards across all conditions weren't counted at all. Fixed by summing `state.totalCount + state.pendingTotalCount` across conditions, consistent with the reward-evidence semantics the earlier `warmupThreshold` fix (above) already established: warmup should track how much reward evidence has actually arrived, not how much of it has been flushed into the posteriors yet. + +- **`POST /v6/reward` is now fire-and-forget, with the config lookup cached**: this endpoint sat on the client's response path doing 8-10 sequential, uncached DB round trips (config lookup, enrollment lookup, audit insert, posterior increments, and — when a batch closes — a flush per condition) even though nothing in the client SDK waits on the result to make a UI decision. `ThompsonSamplingRewardService.recordReward()` was split into `acceptReward()` (public, synchronous — returns a "received and is being processed" receipt immediately) and `processReward()` (private, does the actual work in the background via `acceptReward().catch(...)`, never awaited by the controller). A failure that used to become a `409`/`500` HTTP response is now only logged: `logAndAbort()` (renamed from `throwConflictError()`) logs and throws an internal `RewardProcessingAborted` sentinel purely for control flow, which `acceptReward()`'s catch recognizes and does not re-log (an error of any other type gets one generic "Unexpected error processing..." log line). The `/v6/reward` swagger doc's `200` response and description were updated to describe the receipt/async-logging contract, and its `409`/`500` response entries were removed since nothing throws to the HTTP layer here anymore. Separately, `findConfigById`/`findConfigByDecisionPoint` now go through `CacheService.wrap()` under a new `CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX` (bucketed under the existing `experiments` TTL/refresh-threshold config in `CacheService`'s `PREFIX_CATEGORY` map) — the same `wrap()`-based pattern `ExperimentService.getCachedValidExperiments()` already uses, since `warmupThreshold`/`minimumDrawDifference`/`batchSize` and the joined `experiment.state` only change on an admin edit, far less often than rewards arrive. `ThompsonSamplingExperimentCrudService.createConfig()`/`updateConfig()` call a new `invalidateConfigCache()` (`cacheService.resetPrefixCache(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX)`) after writing — a full-prefix reset rather than a single-key delete, because the decision-point-keyed cache entries embed the same config fields and there's no cheap way to know which `context:site:target` keys reference a given experiment (same blunt-but-safe approach `ExperimentService.updateList()` already takes with `EXPERIMENT_KEY_PREFIX` for the analogous problem). `syncConditions()` does not invalidate, since it only touches `conditionPosteriorStates`, which these cached config objects don't include. One accepted gap: `ExperimentService.updateState()` (experiment start/stop) does not invalidate this cache, so a just-stopped experiment can still accept a reward for up to the `experiments` bucket's TTL — the same TTL-bounded staleness `getCachedValidExperiments()` already tolerates for the assignment path. + ### Architecture notes - **conditionId as algorithm key**: `ThompsonSamplingService.selectCondition()` uses condition UUIDs (not `conditionCode`) as identifiers, since `conditionCode` is nullable. `ConditionPosteriorState` rows are keyed by `conditionId`. The `priors` field in `ThompsonSamplingConfigDTO` is therefore also keyed by conditionId. diff --git a/packages/backend/src/api/controllers/ExperimentClientController.v6.ts b/packages/backend/src/api/controllers/ExperimentClientController.v6.ts index 7132519602..9722730b79 100644 --- a/packages/backend/src/api/controllers/ExperimentClientController.v6.ts +++ b/packages/backend/src/api/controllers/ExperimentClientController.v6.ts @@ -843,6 +843,11 @@ export class ExperimentClientController { * 2. **Decision Point Lookup** - Provide `context` and `decisionPoint` (site and target) to look up the experiment * * At least one of these methods must be provided. + * + * **Asynchronous processing:** This endpoint acknowledges receipt immediately and records the reward + * (config/enrollment lookup, posterior update) in the background — the response does not wait on it. + * A problem with the reward itself (unknown experiment, no matching enrollment, experiment no longer + * enrolling, etc.) is therefore not returned to the caller; it is only visible in server-side logs. * consumes: * - application/json * parameters: @@ -909,14 +914,16 @@ export class ExperimentClientController { * - application/json * responses: * '200': - * description: Reward successfully sent to the adaptive experiment + * description: | + * Reward received and queued for processing. This does not guarantee the reward was recorded - + * see "Asynchronous processing" above. * schema: * type: object * properties: * message: * type: string - * example: Reward sent successfully - * description: Success message + * example: Reward received and is being processed. + * description: Receipt message * request: * type: object * description: Echo of the original request data @@ -939,10 +946,6 @@ export class ExperimentClientController { * description: BadRequestError - Invalid parameters (e.g., missing required fields, invalid rewardValue) * '401': * description: AuthorizationRequiredError - * '409': - * description: Conflict - Data conflict (e.g., site or target not found, enrollment data not found, etc) - * '500': - * description: Internal Server Error */ @Post('reward') public async sendReward( @@ -952,7 +955,7 @@ export class ExperimentClientController { rewardData: RewardValidator ): Promise { request.logger.info({ message: 'Starting the sendReward call for user' }); - return this.thompsonSamplingRewardService.recordReward(request.userDoc, rewardData, request.logger); + return this.thompsonSamplingRewardService.acceptReward(request.userDoc, rewardData, request.logger); } /** diff --git a/packages/backend/src/api/services/CacheService.ts b/packages/backend/src/api/services/CacheService.ts index d73c0af493..cf88873dc4 100644 --- a/packages/backend/src/api/services/CacheService.ts +++ b/packages/backend/src/api/services/CacheService.ts @@ -14,6 +14,7 @@ const PREFIX_CATEGORY: Record = { [CACHE_PREFIX.FEATURE_FLAG_PRECOMPUTED_SEGMENT_KEY_PREFIX]: 'featureFlags', [CACHE_PREFIX.EXPERIMENT_PRECOMPUTED_SEGMENT_KEY_PREFIX]: 'experiments', [CACHE_PREFIX.SETTING_KEY_PREFIX]: 'settings', + [CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX]: 'experiments', }; // this module will get swapped in if caching is enabled but the cache manager fails to initialize as a dummy default deliverer diff --git a/packages/backend/src/api/services/ExperimentAssignmentService.ts b/packages/backend/src/api/services/ExperimentAssignmentService.ts index 204e1867cd..da8a582d0d 100644 --- a/packages/backend/src/api/services/ExperimentAssignmentService.ts +++ b/packages/backend/src/api/services/ExperimentAssignmentService.ts @@ -2103,7 +2103,13 @@ export class ExperimentAssignmentService { priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; }); - const totalRewardCount = config.conditionPosteriorStates.reduce((sum, s) => sum + s.totalCount, 0); + // Include pendingTotalCount so a batchSize buffer awaiting flush still counts toward + // warmup evidence — rewards are "collected" as soon as recordReward() persists them, + // even if batching hasn't folded them into totalCount/successCount/failureCount yet. + const totalRewardCount = config.conditionPosteriorStates.reduce( + (sum, s) => sum + s.totalCount + s.pendingTotalCount, + 0 + ); const tsConfig: ThompsonSamplingConfig = { priors, diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 243ee24430..9ee31dc7d2 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -4,7 +4,8 @@ import { ThompsonSamplingExperimentConfigRepository } from '../repositories/Thom import { ConditionPosteriorStateRepository } from '../repositories/ConditionPosteriorStateRepository'; import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; import { ThompsonSamplingService } from './ThompsonSamplingService'; -import { ExperimentRewardsSummary } from 'upgrade_types'; +import { CacheService } from './CacheService'; +import { CACHE_PREFIX, ExperimentRewardsSummary } from 'upgrade_types'; type ConditionRef = { id: string }; @@ -21,7 +22,8 @@ export class ThompsonSamplingExperimentCrudService { constructor( @InjectRepository() private configRepository: ThompsonSamplingExperimentConfigRepository, @InjectRepository() private posteriorStateRepository: ConditionPosteriorStateRepository, - private thompsonSamplingService: ThompsonSamplingService + private thompsonSamplingService: ThompsonSamplingService, + private cacheService: CacheService ) {} public async getConfigForExperiment(experimentId: string): Promise { @@ -53,6 +55,8 @@ export class ThompsonSamplingExperimentCrudService { ) ); + await this.invalidateConfigCache(); + return config; } @@ -66,6 +70,8 @@ export class ThompsonSamplingExperimentCrudService { } ); + await this.invalidateConfigCache(); + if (!params.priors) { return; } @@ -160,4 +166,16 @@ export class ThompsonSamplingExperimentCrudService { await this.posteriorStateRepository.remove(toRemove); } } + + /** + * Clears every cached config lookup ThompsonSamplingRewardService may have made — both the + * by-experimentId and by-decision-point keys share this prefix. A targeted delete of just the + * affected experimentId key isn't enough on its own: the decision-point-keyed entries embed the + * same config fields and there's no cheap way to know which context/site/target keys reference + * this experiment, so the whole prefix is reset instead (same approach ExperimentService.updateList + * uses for validExperiments- when a similar can't-cheaply-target-one-key situation comes up). + */ + private async invalidateConfigCache(): Promise { + await this.cacheService.resetPrefixCache(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + } } diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index a7e49237a3..9ac163afb2 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -1,13 +1,14 @@ import { Service } from 'typedi'; -import { HttpError } from 'routing-controllers'; import { InjectRepository } from '../../typeorm-typedi-extensions'; import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; -import { BinaryRewardAllowedValue, EXPERIMENT_STATE, SERVER_ERROR } from 'upgrade_types'; +import { BinaryRewardAllowedValue, CACHE_PREFIX, EXPERIMENT_STATE } from 'upgrade_types'; import { ThompsonSamplingRewardRepository } from '../repositories/ThompsonSamplingRewardRepository'; import { ConditionPosteriorStateRepository } from '../repositories/ConditionPosteriorStateRepository'; import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; import { IndividualEnrollmentRepository } from '../repositories/IndividualEnrollmentRepository'; import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; +import { ConditionPosteriorState } from '../models/ConditionPosteriorState'; +import { CacheService } from './CacheService'; import { RewardValidator } from '../controllers/validators/RewardValidator'; import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; @@ -16,6 +17,13 @@ export interface IThompsonSamplingRewardResponse { request: RewardValidator; } +/** + * Thrown internally to unwind out of processReward() once a failure has already been logged via + * logAndAbort() — nothing downstream is waiting on this rejection (see acceptReward()), so it + * exists purely for control flow, not to be reported anywhere else. + */ +class RewardProcessingAborted extends Error {} + @Service() export class ThompsonSamplingRewardService { constructor( @@ -26,114 +34,139 @@ export class ThompsonSamplingRewardService { @InjectRepository() private tsConfigRepository: ThompsonSamplingExperimentConfigRepository, @InjectRepository() - private individualEnrollmentRepository: IndividualEnrollmentRepository + private individualEnrollmentRepository: IndividualEnrollmentRepository, + private cacheService: CacheService ) {} - public async recordReward( + /** + * Acknowledges the reward immediately and does the actual work (config/enrollment lookups, the + * audit write, posterior updates) in the background. Nothing on the client side is waiting on + * this to make a UI decision, so there's no reason to hold the connection — and make the caller + * pay for however many DB round trips recording and batching take — before responding. A failure + * only surfaces in the server logs; see processReward()/logAndAbort(). + */ + public acceptReward( user: RequestedExperimentUser, request: RewardValidator, logger: UpgradeLogger - ): Promise { + ): IThompsonSamplingRewardResponse { + this.processReward(user, request, logger).catch((error) => { + if (!(error instanceof RewardProcessingAborted)) { + logger.error({ + message: `Unexpected error processing Thompson Sampling reward (userId: ${user.id}, experimentId: ${ + request.experimentId ?? 'not provided' + }).`, + error, + request, + }); + } + }); + + return { message: 'Reward received and is being processed.', request }; + } + + private async processReward( + user: RequestedExperimentUser, + request: RewardValidator, + logger: UpgradeLogger + ): Promise { const { experimentId, context, decisionPoint, rewardValue } = request; const success = rewardValue === BinaryRewardAllowedValue.SUCCESS; - try { - const config = experimentId - ? await this.findConfigById(experimentId, request, logger) - : await this.findConfigByDecisionPoint(context, decisionPoint, request, logger); + const config = experimentId + ? await this.findConfigById(experimentId, request, logger) + : await this.findConfigByDecisionPoint(context, decisionPoint, request, logger); - if (config.experiment.state !== EXPERIMENT_STATE.ENROLLING) { - this.throwConflictError( - `Experiment ${config.experimentId} is not actively enrolling (state: ${config.experiment.state}), reward not recorded.`, - request, - logger - ); - } - - const enrollments = await this.individualEnrollmentRepository.findEnrollments(user.id, [config.experimentId]); + if (config.experiment.state !== EXPERIMENT_STATE.ENROLLING) { + this.logAndAbort( + `Experiment ${config.experimentId} is not actively enrolling (state: ${config.experiment.state}), reward not recorded.`, + request, + logger + ); + } - if (!enrollments.length || enrollments.length > 1) { - this.throwConflictError( - `Could not find unique enrollment for user ${user.id} in experiment ${config.experimentId}, reward not recorded.`, - request, - logger - ); - } + const enrollments = await this.individualEnrollmentRepository.findEnrollments(user.id, [config.experimentId]); - const { conditionId } = enrollments[0]; + if (!enrollments.length || enrollments.length > 1) { + this.logAndAbort( + `Could not find unique enrollment for user ${user.id} in experiment ${config.experimentId}, reward not recorded.`, + request, + logger + ); + } - await this.tsRewardRepository.save({ - experimentId: config.experimentId, - conditionId, - userId: user.id, - success, - }); + const { conditionId } = enrollments[0]; - const state = await this.posteriorStateRepository.findByConditionId(conditionId); + await this.tsRewardRepository.save({ + experimentId: config.experimentId, + conditionId, + userId: user.id, + success, + }); - if (!state) { - this.throwConflictError( - `No posterior state found for condition ${conditionId} in experiment ${config.experimentId}, reward not recorded.`, - request, - logger - ); - } + const state = await this.posteriorStateRepository.findByConditionId(conditionId); - await this.applyOrBufferReward(state.id, success, config.batchSize); - - logger.info({ - message: 'Thompson Sampling reward recorded', - experimentId: config.experimentId, - conditionId, - userId: user.id, - success, - }); - - return { message: 'Reward recorded successfully.', request }; - } catch (error) { - if (error instanceof HttpError) throw error; - this.throwConflictError( - `Failed to record reward (userId: ${user.id}, experimentId: ${experimentId ?? 'not provided'}).`, + if (!state) { + this.logAndAbort( + `No posterior state found for condition ${conditionId} in experiment ${config.experimentId}, reward not recorded.`, request, logger ); } + + await this.applyOrBufferReward(state, success, config.batchSize); + + logger.info({ + message: 'Thompson Sampling reward recorded', + experimentId: config.experimentId, + conditionId, + userId: user.id, + success, + }); } /** * Fold a reward into the posterior (successCount/totalCount), or buffer it as pending until - * batchSize reward observations have accumulated for this condition. The raw event is always - * persisted to ThompsonSamplingReward regardless of batching — batching only delays when a - * reward affects which condition gets sampled next, it never drops data. + * batchSize reward observations have accumulated across the whole experiment (all conditions + * combined, not just this one) — batchSize paces how often posteriors move, and a reward for + * any condition is evidence toward that same shared cadence. Once the threshold is hit, every + * condition's pending buffer is flushed, not just the one that tipped it over, so a condition + * with few rewards still gets its pending counts folded in as soon as the batch closes. The raw + * event is always persisted to ThompsonSamplingReward regardless of batching — batching only + * delays when a reward affects which condition gets sampled next, it never drops data. */ - private async applyOrBufferReward(stateId: string, success: boolean, batchSize?: number): Promise { + private async applyOrBufferReward( + state: Pick, + success: boolean, + batchSize?: number + ): Promise { const effectiveBatchSize = batchSize && batchSize > 1 ? batchSize : 1; if (effectiveBatchSize <= 1) { - await this.posteriorStateRepository.increment({ id: stateId }, 'totalCount', 1); + await this.posteriorStateRepository.increment({ id: state.id }, 'totalCount', 1); if (success) { - await this.posteriorStateRepository.increment({ id: stateId }, 'successCount', 1); + await this.posteriorStateRepository.increment({ id: state.id }, 'successCount', 1); } else { - await this.posteriorStateRepository.increment({ id: stateId }, 'failureCount', 1); + await this.posteriorStateRepository.increment({ id: state.id }, 'failureCount', 1); } return; } - await this.posteriorStateRepository.increment({ id: stateId }, 'pendingTotalCount', 1); + await this.posteriorStateRepository.increment({ id: state.id }, 'pendingTotalCount', 1); if (success) { - await this.posteriorStateRepository.increment({ id: stateId }, 'pendingSuccessCount', 1); + await this.posteriorStateRepository.increment({ id: state.id }, 'pendingSuccessCount', 1); } else { - await this.posteriorStateRepository.increment({ id: stateId }, 'pendingFailureCount', 1); + await this.posteriorStateRepository.increment({ id: state.id }, 'pendingFailureCount', 1); } - const refreshedState = await this.posteriorStateRepository.findOne({ where: { id: stateId } }); + const experimentStates = await this.posteriorStateRepository.findByConfigId(state.configId); + const totalPending = experimentStates.reduce((sum, s) => sum + s.pendingTotalCount, 0); - if (refreshedState.pendingTotalCount >= effectiveBatchSize) { - await this.flushPendingRewards( - stateId, - refreshedState.pendingSuccessCount, - refreshedState.pendingFailureCount, - refreshedState.pendingTotalCount + if (totalPending >= effectiveBatchSize) { + await Promise.all( + experimentStates + .filter((s) => s.pendingTotalCount > 0) + .map((s) => this.flushPendingRewards(s.id, s.pendingSuccessCount, s.pendingFailureCount, s.pendingTotalCount)) ); } } @@ -157,18 +190,35 @@ export class ThompsonSamplingRewardService { ); } + /** + * Config lookups are cached — warmupThreshold/minimumDrawDifference/batchSize and the + * experiment's enrolling state change only on an admin edit (ThompsonSamplingExperimentCrudService + * invalidates on write), while a reward can arrive for the same experiment far more often. This + * keeps a burst of rewards from re-querying the DB for data that hasn't moved. Same read-only + * contract as ExperimentService.getCachedValidExperiments(): the returned object is shared across + * callers, never mutate it. + * + * Note: an experiment's enrolling state can also change via ExperimentService.updateState() + * (start/stop), which this cache does not invalidate — that state change is tolerated as + * TTL-bounded staleness, the same tradeoff already accepted by getCachedValidExperiments for the + * assignment path. + */ private async findConfigById( experimentId: string, request: RewardValidator, logger: UpgradeLogger ): Promise { - const config = await this.tsConfigRepository.findOne({ - where: { experimentId }, - relations: { experiment: true }, - }); + const config = await this.cacheService.wrap( + CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX + 'id:' + experimentId, + () => + this.tsConfigRepository.findOne({ + where: { experimentId }, + relations: { experiment: true }, + }) + ); if (!config) { - this.throwConflictError( + this.logAndAbort( `No Thompson Sampling config found for experiment ${experimentId}, reward not recorded.`, request, logger @@ -185,10 +235,13 @@ export class ThompsonSamplingRewardService { logger: UpgradeLogger ): Promise { const { site, target } = decisionPoint; - const configs = await this.tsConfigRepository.findByDecisionPoint(context, site, target); + const configs = await this.cacheService.wrap( + CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX + `dp:${context}:${site}:${target}`, + () => this.tsConfigRepository.findByDecisionPoint(context, site, target) + ); if (configs.length === 0) { - this.throwConflictError( + this.logAndAbort( `No active Thompson Sampling experiment found for decision point (context: ${context}, site: ${site}, target: ${target}).`, request, logger @@ -196,7 +249,7 @@ export class ThompsonSamplingRewardService { } if (configs.length > 1) { - this.throwConflictError( + this.logAndAbort( `Multiple active Thompson Sampling experiments found for decision point (context: ${context}, site: ${site}, target: ${target}); use experimentId to disambiguate.`, request, logger @@ -206,10 +259,8 @@ export class ThompsonSamplingRewardService { return configs[0]; } - private throwConflictError(message: string, request: RewardValidator, logger: UpgradeLogger): never { + private logAndAbort(message: string, request: RewardValidator, logger: UpgradeLogger): never { logger.error({ message, request }); - const error = new HttpError(409, message); - (error as any).type = SERVER_ERROR.ASSIGNMENT_ERROR; - throw error; + throw new RewardProcessingAborted(message); } } diff --git a/packages/backend/src/api/services/ThompsonSamplingService.ts b/packages/backend/src/api/services/ThompsonSamplingService.ts index 73c2e88fa6..1427491cb9 100644 --- a/packages/backend/src/api/services/ThompsonSamplingService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingService.ts @@ -32,6 +32,9 @@ export class ThompsonSamplingService { * awards the round to the highest draw. Win counts are converted to integer percentages via the * Largest Remainder Method so the result always sums to exactly 100. * + * When every condition shares the same alpha/beta (e.g. equal priors and no reward data yet), + * the true win rate is exactly uniform by symmetry — skip the simulation and its sampling noise. + * * @param conditions - Each condition with its current posterior alpha and beta parameters * @param numDraws - Number of simulated draws (default 10 000) * @returns Map of conditionCode → integer percentage in [0, 100]; sums to 100 @@ -43,6 +46,10 @@ export class ThompsonSamplingService { if (conditions.length === 0) return {}; if (conditions.length === 1) return { [conditions[0].code]: 100 }; + if (conditions.every((c) => c.alpha === conditions[0].alpha && c.beta === conditions[0].beta)) { + return this.distributeEvenly(conditions.map((c) => c.code)); + } + const wins = new Map(conditions.map((c) => [c.code, 0])); for (let i = 0; i < numDraws; i++) { @@ -58,17 +65,29 @@ export class ThompsonSamplingService { wins.set(winner, (wins.get(winner) ?? 0) + 1); } - // Largest Remainder Method: floor each raw percentage then distribute the - // remaining integer points to the conditions with the largest fractional parts. - const rawPcts = conditions.map((c) => { - const raw = ((wins.get(c.code) ?? 0) / numDraws) * 100; - return { code: c.code, floor: Math.floor(raw), remainder: raw - Math.floor(raw) }; - }); - const pointsLeft = 100 - rawPcts.reduce((sum, r) => sum + r.floor, 0); - rawPcts.sort((a, b) => b.remainder - a.remainder); + return this.toIntegerPercentages( + conditions.map((c) => ({ code: c.code, raw: ((wins.get(c.code) ?? 0) / numDraws) * 100 })) + ); + } + + // Splits 100 points evenly across codes (Largest Remainder Method handles the non-divisible case). + private distributeEvenly(codes: string[]): Record { + return this.toIntegerPercentages(codes.map((code) => ({ code, raw: 100 / codes.length }))); + } + + // Largest Remainder Method: floor each raw percentage then distribute the + // remaining integer points to the entries with the largest fractional parts. + private toIntegerPercentages(raws: Array<{ code: string; raw: number }>): Record { + const withFloors = raws.map(({ code, raw }) => ({ + code, + floor: Math.floor(raw), + remainder: raw - Math.floor(raw), + })); + const pointsLeft = 100 - withFloors.reduce((sum, r) => sum + r.floor, 0); + withFloors.sort((a, b) => b.remainder - a.remainder); const result: Record = {}; - rawPcts.forEach((r, i) => { + withFloors.forEach((r, i) => { result[r.code] = r.floor + (i < pointsLeft ? 1 : 0); }); return result; @@ -79,7 +98,8 @@ export class ThompsonSamplingService { * * @param conditionCodes - All eligible condition codes for this experiment * @param rewardSummaries - Accumulated reward counts per condition - * @param totalRewardCount - Total number of reward observations across all conditions + * @param totalRewardCount - Total number of reward observations collected across all conditions, + * including any not yet folded into the posteriors by a pending batch flush * @param config - Optional algorithm parameters (priors, warmup, thresholds) */ selectCondition( @@ -116,7 +136,7 @@ export class ThompsonSamplingService { // Fall back to uniform when the top two draws are too close to distinguish if ( - config.minimumDrawDifference !== undefined && + config.minimumDrawDifference && draws.length >= 2 && draws[0].draw - draws[1].draw < config.minimumDrawDifference ) { diff --git a/packages/backend/test/unit/services/CacheService.test.ts b/packages/backend/test/unit/services/CacheService.test.ts index cd8f108c82..1983ae5497 100644 --- a/packages/backend/test/unit/services/CacheService.test.ts +++ b/packages/backend/test/unit/services/CacheService.test.ts @@ -270,6 +270,8 @@ describe('CacheService', () => { // owner's TTL rather than the segments TTL [CACHE_PREFIX.FEATURE_FLAG_PRECOMPUTED_SEGMENT_KEY_PREFIX, 45000], [CACHE_PREFIX.EXPERIMENT_PRECOMPUTED_SEGMENT_KEY_PREFIX, 30000], + // Thompson Sampling config lookups share the experiments bucket/TTL + [CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX, 30000], ])('wraps %s with its category TTL (ms)', async (prefix, expectedTtl) => { const fn = jest.fn().mockResolvedValue('v'); await service.wrap(prefix + 'context', fn); diff --git a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts index 270db9395e..00e5cd3731 100644 --- a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts @@ -31,7 +31,13 @@ import { withinSubjectDPExperiment, } from '../mockdata'; import { GroupEnrollment } from '../../../src/api/models/GroupEnrollment'; -import { ENROLLMENT_CODE, EXPERIMENT_STATE, FILTER_MODE, MARKED_DECISION_POINT_STATUS } from 'upgrade_types'; +import { + ASSIGNMENT_ALGORITHM, + ENROLLMENT_CODE, + EXPERIMENT_STATE, + FILTER_MODE, + MARKED_DECISION_POINT_STATUS, +} from 'upgrade_types'; import { CacheService } from '../../../src/api/services/CacheService'; import { UserStratificationFactorRepository } from '../../../src/api/repositories/UserStratificationRepository'; import { configureLogger } from '../../utils/logger'; @@ -2245,4 +2251,47 @@ describe('Experiment Assignment Service Test', () => { }); }); }); + + describe('[assignThompsonSampling] warmup evidence', () => { + const thompsonExperiment: any = { + id: 'ts-experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-a' }, { id: 'condition-b' }], + }; + const thompsonUser: any = { id: 'user-1' }; + + function makePosteriorState(conditionId: string, totalCount: number, pendingTotalCount: number) { + return { + conditionId, + successCount: 0, + failureCount: 0, + totalCount, + pendingTotalCount, + priorSuccess: 1, + priorFailure: 1, + }; + } + + it('counts rewards still sitting in a pending batch toward totalRewardCount, not just flushed ones', async () => { + testedModule.thompsonSamplingConfigRepository = { + findByExperimentId: sandbox.stub().resolves({ + warmupThreshold: 9, + minimumDrawDifference: undefined, + conditionPosteriorStates: [ + makePosteriorState('condition-a', 2, 3), // 2 flushed + 3 pending + makePosteriorState('condition-b', 1, 4), // 1 flushed + 4 pending + ], + }), + }; + testedModule.thompsonSamplingService = { selectCondition: sandbox.stub().returns('condition-a') }; + + await (testedModule as any).assignThompsonSampling(thompsonExperiment, thompsonUser, loggerMock); + + // Flushed-only totals (2 + 1 = 3) would wrongly stay inside a warmupThreshold of 9. + // The real reward evidence collected so far also includes the pending buffers + // (3 + 4 = 7), for a true total of 10 — past warmup. + const totalRewardCountArg = testedModule.thompsonSamplingService.selectCondition.getCall(0).args[2]; + expect(totalRewardCountArg).toBe(10); + }); + }); }); diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts new file mode 100644 index 0000000000..53f7b65a81 --- /dev/null +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -0,0 +1,59 @@ +import { ThompsonSamplingExperimentCrudService } from '../../../src/api/services/ThompsonSamplingExperimentCrudService'; +import { CACHE_PREFIX } from 'upgrade_types'; + +describe('ThompsonSamplingExperimentCrudService', () => { + let configRepository: any; + let posteriorStateRepository: any; + let cacheService: any; + let service: ThompsonSamplingExperimentCrudService; + + beforeEach(() => { + configRepository = { + save: jest.fn().mockResolvedValue({ id: 'config-1', experimentId: 'experiment-1' }), + update: jest.fn().mockResolvedValue(undefined), + findByExperimentId: jest.fn().mockResolvedValue(undefined), + }; + posteriorStateRepository = { + save: jest.fn().mockResolvedValue(undefined), + update: jest.fn().mockResolvedValue(undefined), + remove: jest.fn().mockResolvedValue(undefined), + }; + cacheService = { resetPrefixCache: jest.fn().mockResolvedValue(undefined) }; + + service = new ThompsonSamplingExperimentCrudService( + configRepository, + posteriorStateRepository, + {} as any, + cacheService + ); + }); + + describe('config cache invalidation', () => { + it('resets the Thompson Sampling config cache prefix after creating a config', async () => { + await service.createConfig('experiment-1', [{ id: 'condition-1' }]); + + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + }); + + it('resets the Thompson Sampling config cache prefix after updating a config', async () => { + await service.updateConfig('experiment-1', { batchSize: 5 }); + + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + }); + + it('resets the cache even when updating priors (a second, later write in updateConfig)', async () => { + configRepository.findByExperimentId.mockResolvedValue({ + id: 'config-1', + conditionPosteriorStates: [{ conditionId: 'condition-1' }], + }); + + await service.updateConfig('experiment-1', { priors: { 'condition-1': { success: 3, failure: 2 } } }); + + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + expect(posteriorStateRepository.update).toHaveBeenCalledWith( + { configId: 'config-1', conditionId: 'condition-1' }, + { priorSuccess: 3, priorFailure: 2 } + ); + }); + }); +}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts index 49fc34727f..da1ca793d7 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts @@ -8,9 +8,24 @@ import { configureLogger } from '../../utils/logger'; const logger = new UpgradeLogger(); const EXPERIMENT_ID = 'experiment-1'; +const CONFIG_ID = 'config-1'; const CONDITION_ID = 'condition-1'; +const CONDITION_A_ID = 'condition-a'; +const CONDITION_B_ID = 'condition-b'; const USER_ID = 'user-1'; +interface PosteriorStateRow { + id: string; + configId: string; + conditionId: string; + successCount: number; + failureCount: number; + totalCount: number; + pendingSuccessCount: number; + pendingFailureCount: number; + pendingTotalCount: number; +} + function makeUser(): RequestedExperimentUser { return { id: USER_ID, requestedUserId: USER_ID } as RequestedExperimentUser; } @@ -19,6 +34,53 @@ function makeRequest(rewardValue: BinaryRewardAllowedValue = BinaryRewardAllowed return { experimentId: EXPERIMENT_ID, rewardValue } as RewardValidator; } +function makeStateRow(id: string, conditionId: string): PosteriorStateRow { + return { + id, + configId: CONFIG_ID, + conditionId, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingSuccessCount: 0, + pendingFailureCount: 0, + pendingTotalCount: 0, + }; +} + +// acceptReward() fires processReward() without awaiting it, so tests that exercise the +// background path need to let its promise chain drain before asserting on side effects. +function flushPromises(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +// The default test double: caching is transparent (always misses through to the source), so +// most tests can ignore caching entirely and assert on repository calls as before. +function makePassthroughCacheService() { + return { + wrap: jest.fn((_key: string, fn: () => Promise) => fn()), + resetPrefixCache: jest.fn().mockResolvedValue(undefined), + }; +} + +// A real (in-memory) cache double, for the tests that specifically exercise caching behavior. +function makeMemoizingCacheService() { + const store = new Map(); + return { + wrap: jest.fn(async (key: string, fn: () => Promise) => { + if (store.has(key)) return store.get(key); + const value = await fn(); + store.set(key, value); + return value; + }), + resetPrefixCache: jest.fn(async (prefix: string) => { + for (const key of Array.from(store.keys())) { + if (key.startsWith(prefix)) store.delete(key); + } + }), + }; +} + describe('ThompsonSamplingRewardService', () => { beforeAll(() => { configureLogger(); @@ -28,19 +90,15 @@ describe('ThompsonSamplingRewardService', () => { let posteriorStateRepository: any; let tsConfigRepository: any; let individualEnrollmentRepository: any; + let cacheService: ReturnType; let service: ThompsonSamplingRewardService; - // In-memory posterior state row, mutated by the mocked increment/update calls so - // assertions can inspect the final counts after one or more recordReward() calls. - let state: { - id: string; - successCount: number; - failureCount: number; - totalCount: number; - pendingSuccessCount: number; - pendingFailureCount: number; - pendingTotalCount: number; - }; + // In-memory posterior state rows, keyed by conditionId, mutated by the mocked + // increment/update calls so assertions can inspect the final counts after one + // or more processReward() calls. Reward flow always resolves the row through + // findByConditionId(conditionId), so the enrollment mock's conditionId is what + // selects which row a given call mutates. + let statesByCondition: Record; function makeConfig(batchSize?: number) { return { @@ -50,28 +108,33 @@ describe('ThompsonSamplingRewardService', () => { }; } + function allStates(): PosteriorStateRow[] { + return Object.values(statesByCondition); + } + + function findRowById(id: string): PosteriorStateRow { + return allStates().find((row) => row.id === id); + } + beforeEach(() => { - state = { - id: 'state-1', - successCount: 0, - failureCount: 0, - totalCount: 0, - pendingSuccessCount: 0, - pendingFailureCount: 0, - pendingTotalCount: 0, + statesByCondition = { + [CONDITION_ID]: makeStateRow('state-1', CONDITION_ID), }; tsRewardRepository = { save: jest.fn().mockResolvedValue(undefined) }; posteriorStateRepository = { - findByConditionId: jest.fn().mockResolvedValue(state), - increment: jest.fn((criteria: { id: string }, column: keyof typeof state, amount: number) => { - (state as any)[column] += amount; + findByConditionId: jest.fn((conditionId: string) => Promise.resolve(statesByCondition[conditionId])), + findByConfigId: jest.fn((configId: string) => + Promise.resolve(allStates().filter((row) => row.configId === configId)) + ), + increment: jest.fn((criteria: { id: string }, column: keyof PosteriorStateRow, amount: number) => { + const row = findRowById(criteria.id); + (row[column] as number) += amount; return Promise.resolve(undefined); }), - findOne: jest.fn().mockImplementation(() => Promise.resolve({ ...state })), - update: jest.fn((criteria: { id: string }, partial: Partial) => { - Object.assign(state, partial); + update: jest.fn((criteria: { id: string }, partial: Partial) => { + Object.assign(findRowById(criteria.id), partial); return Promise.resolve(undefined); }), }; @@ -85,11 +148,14 @@ describe('ThompsonSamplingRewardService', () => { findEnrollments: jest.fn().mockResolvedValue([{ conditionId: CONDITION_ID }]), }; + cacheService = makePassthroughCacheService(); + service = new ThompsonSamplingRewardService( tsRewardRepository, posteriorStateRepository, tsConfigRepository, - individualEnrollmentRepository + individualEnrollmentRepository, + cacheService as any ); }); @@ -97,7 +163,7 @@ describe('ThompsonSamplingRewardService', () => { it('always persists the raw reward event regardless of batching', async () => { tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(5)); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); expect(tsRewardRepository.save).toHaveBeenCalledWith({ experimentId: EXPERIMENT_ID, @@ -110,8 +176,9 @@ describe('ThompsonSamplingRewardService', () => { it('increments totalCount immediately when batchSize is unset (default behavior)', async () => { tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(undefined)); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + const state = statesByCondition[CONDITION_ID]; expect(state.totalCount).toBe(1); expect(state.successCount).toBe(1); expect(state.pendingTotalCount).toBe(0); @@ -120,8 +187,9 @@ describe('ThompsonSamplingRewardService', () => { it('increments totalCount immediately when batchSize is 1', async () => { tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(1)); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + const state = statesByCondition[CONDITION_ID]; expect(state.totalCount).toBe(1); expect(state.successCount).toBe(0); expect(state.failureCount).toBe(1); @@ -129,18 +197,20 @@ describe('ThompsonSamplingRewardService', () => { }); }); - describe('batchSize', () => { + describe('batchSize (single condition)', () => { it('buffers rewards as pending until batchSize is reached', async () => { tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(3)); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + let state = statesByCondition[CONDITION_ID]; expect(state.pendingTotalCount).toBe(1); expect(state.pendingSuccessCount).toBe(1); expect(state.pendingFailureCount).toBe(0); expect(state.totalCount).toBe(0); expect(state.successCount).toBe(0); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + state = statesByCondition[CONDITION_ID]; expect(state.pendingTotalCount).toBe(2); expect(state.pendingSuccessCount).toBe(1); expect(state.pendingFailureCount).toBe(1); @@ -151,10 +221,11 @@ describe('ThompsonSamplingRewardService', () => { it('flushes pending counts into successCount/totalCount once batchSize is reached', async () => { tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(3)); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + const state = statesByCondition[CONDITION_ID]; expect(state.totalCount).toBe(3); expect(state.successCount).toBe(2); expect(state.failureCount).toBe(1); @@ -167,13 +238,15 @@ describe('ThompsonSamplingRewardService', () => { tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(2)); // First batch of 2 flushes... - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + let state = statesByCondition[CONDITION_ID]; expect(state.totalCount).toBe(2); expect(state.successCount).toBe(2); // ...a single reward into the next batch should only be pending, not yet applied. - await service.recordReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + state = statesByCondition[CONDITION_ID]; expect(state.totalCount).toBe(2); expect(state.successCount).toBe(2); expect(state.pendingTotalCount).toBe(1); @@ -191,9 +264,10 @@ describe('ThompsonSamplingRewardService', () => { BinaryRewardAllowedValue.SUCCESS, ]; for (const value of values) { - await service.recordReward(makeUser(), makeRequest(value), logger); + await (service as any).processReward(makeUser(), makeRequest(value), logger); } + const state = statesByCondition[CONDITION_ID]; expect(state.pendingSuccessCount).toBe(2); expect(state.pendingFailureCount).toBe(2); expect(state.pendingFailureCount).toBe(state.pendingTotalCount - state.pendingSuccessCount); @@ -209,9 +283,10 @@ describe('ThompsonSamplingRewardService', () => { BinaryRewardAllowedValue.SUCCESS, ]; for (const value of values) { - await service.recordReward(makeUser(), makeRequest(value), logger); + await (service as any).processReward(makeUser(), makeRequest(value), logger); } + const state = statesByCondition[CONDITION_ID]; expect(state.successCount).toBe(2); expect(state.failureCount).toBe(2); expect(state.failureCount).toBe(state.totalCount - state.successCount); @@ -228,11 +303,180 @@ describe('ThompsonSamplingRewardService', () => { BinaryRewardAllowedValue.FAILURE, ]; for (const value of values) { - await service.recordReward(makeUser(), makeRequest(value), logger); + await (service as any).processReward(makeUser(), makeRequest(value), logger); } + const state = statesByCondition[CONDITION_ID]; expect(state.totalCount + state.pendingTotalCount).toBe(values.length); expect(tsRewardRepository.save).toHaveBeenCalledTimes(values.length); }); }); + + describe('batchSize (across conditions of the same experiment)', () => { + beforeEach(() => { + statesByCondition = { + [CONDITION_A_ID]: makeStateRow('state-a', CONDITION_A_ID), + [CONDITION_B_ID]: makeStateRow('state-b', CONDITION_B_ID), + }; + }); + + function rewardCondition(conditionId: string, value: BinaryRewardAllowedValue) { + individualEnrollmentRepository.findEnrollments = jest.fn().mockResolvedValue([{ conditionId }]); + return (service as any).processReward(makeUser(), makeRequest(value), logger); + } + + it('counts pending rewards for batchSize against the whole experiment, not per condition', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(5)); + + // 4 rewards land on condition A, 1 on condition B — 5 total, so batchSize=5 + // should flush even though neither condition alone reached 5. + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.FAILURE); + + const stateA = statesByCondition[CONDITION_A_ID]; + const stateB = statesByCondition[CONDITION_B_ID]; + expect(stateA.pendingTotalCount).toBe(4); + expect(stateA.totalCount).toBe(0); + expect(stateB.pendingTotalCount).toBe(0); + + await rewardCondition(CONDITION_B_ID, BinaryRewardAllowedValue.SUCCESS); + + // The 5th reward (on B) tips the shared batch over the threshold, so both + // conditions' pending buffers flush together. + expect(stateA.totalCount).toBe(4); + expect(stateA.successCount).toBe(3); + expect(stateA.failureCount).toBe(1); + expect(stateA.pendingTotalCount).toBe(0); + + expect(stateB.totalCount).toBe(1); + expect(stateB.successCount).toBe(1); + expect(stateB.pendingTotalCount).toBe(0); + }); + + it('does not flush a condition with no pending rewards of its own', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(2)); + + // Condition B never receives a reward in this batch; only A's two rewards + // trip the threshold. B's row should be left untouched (still all zeros). + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + + const stateB = statesByCondition[CONDITION_B_ID]; + expect(stateB.totalCount).toBe(0); + expect(stateB.pendingTotalCount).toBe(0); + expect(posteriorStateRepository.update).not.toHaveBeenCalledWith({ id: stateB.id }, expect.anything()); + }); + }); + + describe('acceptReward (quick receipt, background processing)', () => { + it('returns a receipt synchronously, without waiting on the DB', () => { + // Never resolves — if acceptReward awaited this, the test would hang instead of returning. + tsConfigRepository.findOne = jest.fn().mockReturnValue(new Promise(() => undefined)); + + const request = makeRequest(BinaryRewardAllowedValue.SUCCESS); + const result = service.acceptReward(makeUser(), request, logger); + + expect(result).toEqual({ message: 'Reward received and is being processed.', request }); + }); + + it('logs the specific reason (once) when the background reward cannot be recorded', async () => { + individualEnrollmentRepository.findEnrollments = jest.fn().mockResolvedValue([]); // no enrollment found + const loggerMock: any = { info: jest.fn(), error: jest.fn(), warn: jest.fn() }; + + const result = service.acceptReward(makeUser(), makeRequest(), loggerMock); + expect(result.message).toBe('Reward received and is being processed.'); + + await flushPromises(); + + expect(loggerMock.error).toHaveBeenCalledTimes(1); + expect(loggerMock.error).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('Could not find unique enrollment') }) + ); + }); + + it('logs a single generic failure message for an unexpected (non-abort) error', async () => { + tsConfigRepository.findOne = jest.fn().mockRejectedValue(new Error('connection reset')); + const loggerMock: any = { info: jest.fn(), error: jest.fn(), warn: jest.fn() }; + + service.acceptReward(makeUser(), makeRequest(), loggerMock); + await flushPromises(); + + expect(loggerMock.error).toHaveBeenCalledTimes(1); + expect(loggerMock.error).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Unexpected error processing Thompson Sampling reward'), + error: expect.any(Error), + }) + ); + }); + + it('still records the reward in the background after returning the receipt', async () => { + const result = service.acceptReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + expect(result.message).toBe('Reward received and is being processed.'); + expect(tsRewardRepository.save).not.toHaveBeenCalled(); + + await flushPromises(); + + expect(tsRewardRepository.save).toHaveBeenCalledWith({ + experimentId: EXPERIMENT_ID, + conditionId: CONDITION_ID, + userId: USER_ID, + success: true, + }); + }); + }); + + describe('config lookup caching', () => { + beforeEach(() => { + cacheService = makeMemoizingCacheService(); + service = new ThompsonSamplingRewardService( + tsRewardRepository, + posteriorStateRepository, + tsConfigRepository, + individualEnrollmentRepository, + cacheService as any + ); + }); + + it('reuses a cached experimentId lookup across rewards instead of re-querying the DB', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig()); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + expect(tsConfigRepository.findOne).toHaveBeenCalledTimes(1); + // Only the config lookup is cached — the reward itself is still recorded every time. + expect(tsRewardRepository.save).toHaveBeenCalledTimes(2); + }); + + it('reuses a cached decision-point lookup across rewards instead of re-querying the DB', async () => { + tsConfigRepository.findByDecisionPoint = jest.fn().mockResolvedValue([makeConfig()]); + const dpRequest = (): RewardValidator => + ({ + rewardValue: BinaryRewardAllowedValue.SUCCESS, + context: 'context-1', + decisionPoint: { site: 'site-1', target: 'target-1' }, + } as RewardValidator); + + await (service as any).processReward(makeUser(), dpRequest(), logger); + await (service as any).processReward(makeUser(), dpRequest(), logger); + + expect(tsConfigRepository.findByDecisionPoint).toHaveBeenCalledTimes(1); + }); + + it('does not share cache entries between different experiments', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig()); + + await (service as any).processReward(makeUser(), makeRequest(), logger); + await (service as any).processReward( + makeUser(), + { experimentId: 'experiment-2', rewardValue: BinaryRewardAllowedValue.SUCCESS } as RewardValidator, + logger + ); + + expect(tsConfigRepository.findOne).toHaveBeenCalledTimes(2); + }); + }); }); diff --git a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts index 41270a7020..3bc72e7fbd 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts @@ -236,28 +236,42 @@ describe('ThompsonSamplingService', () => { } }); - it('two equal Beta(1,1) arms split near 50/50', () => { + it('two equal Beta(1,1) arms split exactly 50/50 (skips simulation by symmetry)', () => { const conditions = [ { code: 'A', alpha: 1, beta: 1 }, { code: 'B', alpha: 1, beta: 1 }, ]; const weights = service.estimateConditionWeights(conditions); - // With 10k draws and σ≈0.5%, ±10% is ~20σ — essentially never flaky - expect(weights['A']).toBeGreaterThanOrEqual(40); - expect(weights['A']).toBeLessThanOrEqual(60); - expect(weights['B']).toBeGreaterThanOrEqual(40); - expect(weights['B']).toBeLessThanOrEqual(60); + expect(weights).toEqual({ A: 50, B: 50 }); }); - it('four equal arms each get roughly 25%', () => { + it('four equal arms each get exactly 25% regardless of the shared prior strength', () => { const conditions = ['A', 'B', 'C', 'D'].map((code) => ({ code, alpha: 2, beta: 2 })); const weights = service.estimateConditionWeights(conditions); + expect(weights).toEqual({ A: 25, B: 25, C: 25, D: 25 }); + }); + + it('three equal arms split evenly and still sum to 100', () => { + const conditions = ['A', 'B', 'C'].map((code) => ({ code, alpha: 4, beta: 6 })); + const weights = service.estimateConditionWeights(conditions); + const total = Object.values(weights).reduce((sum, w) => sum + w, 0); + expect(total).toBe(100); for (const w of Object.values(weights)) { - expect(w).toBeGreaterThanOrEqual(15); - expect(w).toBeLessThanOrEqual(35); + expect(w).toBeGreaterThanOrEqual(33); + expect(w).toBeLessThanOrEqual(34); } }); + it('unequal alpha/beta still runs the simulation (not the equal-posterior shortcut)', () => { + const conditions = [ + { code: 'A', alpha: 5, beta: 3 }, + { code: 'B', alpha: 3, beta: 5 }, + ]; + const weights = service.estimateConditionWeights(conditions); + expect(weights['A']).toBeGreaterThan(50); + expect(weights['B']).toBeLessThan(50); + }); + it('dominant condition (90 successes / 10 failures) captures most weight', () => { const conditions = [ { code: 'winner', alpha: 1 + 90, beta: 1 + 10 }, diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts index 7f2853d5d6..472dcdcb4a 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts @@ -180,6 +180,10 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { description: 'experiments.upsert-experiment-modal.assignment-algorithm-stratified-random-sampling-description.text', }, + { + value: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + description: 'experiments.upsert-experiment-modal.assignment-algorithm-thompson-sampling-description.text', + }, ]; constructor( @@ -199,10 +203,6 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { description: 'Condition will be assigned within subjects (e.g., participant sees multiple conditions).', }); } - this.assignmentAlgorithms.push({ - value: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, - description: ASSIGNMENT_ALGORITHM_DISPLAY_MAP[ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING], - }); } ngOnInit(): void { diff --git a/packages/frontend/projects/upgrade/src/assets/i18n/en.json b/packages/frontend/projects/upgrade/src/assets/i18n/en.json index 077bc9fb1a..fe430467c6 100644 --- a/packages/frontend/projects/upgrade/src/assets/i18n/en.json +++ b/packages/frontend/projects/upgrade/src/assets/i18n/en.json @@ -195,7 +195,7 @@ "home.new-experiment.design.ts-configurable-policy.batch-size.hint.text": "Number of rewards to collect before updating the model (default: 1)", "home.new-experiment.design.ts-configurable-policy.batch-size.min-error.text": "Batch size must be at least 1", "home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text": "Warmup Threshold", - "home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text": "Use uniform random selection until total enrollments exceed this count (default: 0)", + "home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text": "Use uniform random until the total reward count exceeds this threshold (default: 0)", "home.new-experiment.design.ts-configurable-policy.uniform-threshold.min-error.text": "Warmup threshold must be 0 or higher", "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text": "Minimum Draw Difference", "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.hint.text": "Fall back to uniform selection when the top two sampled draws differ by less than this value (0.0–1.0, default: 0)", @@ -418,6 +418,7 @@ "experiments.upsert-experiment-modal.condition-order-ordered-round-robin-description.text": "Conditions are presented in the same fixed sequence each round.", "experiments.upsert-experiment-modal.assignment-algorithm-random-description.text": "Random assignment according to the configured condition weights.", "experiments.upsert-experiment-modal.assignment-algorithm-stratified-random-sampling-description.text": "Random assignment within balanced subgroups to ensure representation.", + "experiments.upsert-experiment-modal.assignment-algorithm-thompson-sampling-description.text": "Adaptive assignment based on Thompson Sampling.", "experiments.upsert-experiment-modal.stratification-factor-hint.text": "Select a stratification factor for balanced random assignment.", "experiments.upsert-experiment-modal.tags-label.text": "Tags (optional)", "experiments.upsert-experiment-modal.tags-placeholder.text": "Tags separated by commas", diff --git a/packages/types/src/Experiment/enums.ts b/packages/types/src/Experiment/enums.ts index 3980bb971e..2448b0b895 100644 --- a/packages/types/src/Experiment/enums.ts +++ b/packages/types/src/Experiment/enums.ts @@ -364,6 +364,7 @@ export enum CACHE_PREFIX { FEATURE_FLAG_PRECOMPUTED_SEGMENT_KEY_PREFIX = 'featureFlagPrecomputedSegments-', EXPERIMENT_PRECOMPUTED_SEGMENT_KEY_PREFIX = 'experimentPrecomputedSegments-', SETTING_KEY_PREFIX = 'setting-', + THOMPSON_SAMPLING_CONFIG_KEY_PREFIX = 'thompsonSamplingConfig-', } export enum STATUS_INDICATOR_CHIP_TYPE { From f0cd8a984549b9e43161e4bef02d84ba4be011ed Mon Sep 17 00:00:00 2001 From: doswalt Date: Fri, 4 Sep 2026 12:57:54 -0400 Subject: [PATCH 07/28] tidying up inconsistencies and DRY-able opportunities --- CLAUDE.md | 26 ++- .../api/controllers/ExperimentController.ts | 50 +---- ...mpsonSamplingExperimentConfigRepository.ts | 12 ++ .../services/ExperimentAssignmentService.ts | 14 +- .../src/api/services/ImportExportService.ts | 13 +- .../ThompsonSamplingExperimentCrudService.ts | 84 +++++++-- .../services/ThompsonSamplingRewardService.ts | 49 +++-- .../api/services/ThompsonSamplingService.ts | 72 +++++-- .../controllers/ExperimentController.test.ts | 6 +- .../ExperimentAssignmentService.test.ts | 5 +- .../unit/services/ImportExportService.test.ts | 57 ++++++ ...mpsonSamplingExperimentCrudService.test.ts | 175 +++++++++++++++++- .../services/ThompsonSamplingService.test.ts | 56 ++++-- .../experiments/store/experiments.model.ts | 4 + ...iment-conditions-section-card.component.ts | 2 +- ...experiment-conditions-table.component.html | 4 +- .../experiment-conditions-table.component.ts | 3 + ...nt-condition-expandable-row.component.html | 4 +- ...ment-condition-expandable-row.component.ts | 9 +- packages/types/package.json | 3 - packages/types/tsconfig.json | 2 - 21 files changed, 509 insertions(+), 141 deletions(-) create mode 100644 packages/backend/test/unit/services/ImportExportService.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index cde0996f08..ed5dba3ebb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -131,7 +131,7 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - **Reward error type**: `ThompsonSamplingRewardService.throwConflictError` sets `(error as any).type = SERVER_ERROR.ASSIGNMENT_ERROR` explicitly. Without this, a plain `HttpError(409)` falls through to the middleware's httpCode switch where 409 → `SERVER_ERROR.DUPLICATE_KEY` (whose string is the unrelated feature flag message). Added `case SERVER_ERROR.ASSIGNMENT_ERROR` to `ErrorHandlerMiddleware` outer switch. -- **thompsonSamplingConfig missing in API responses**: `getSingleExperiment`, `create`, and `update` all return `ExperimentDTO` without loading `thompsonSamplingConfig` (it's in a separate table). Added `attachThompsonSamplingConfig()` private method to `ExperimentController` that queries the config for THOMPSON_SAMPLING experiments and attaches it; called on all three response paths. Also added `getConfigForExperiment()` to `ThompsonSamplingExperimentCrudService`. +- **thompsonSamplingConfig missing in API responses**: `getSingleExperiment`, `create`, and `update` all return `ExperimentDTO` without loading `thompsonSamplingConfig` (it's in a separate table). Added `attachThompsonSamplingConfig()` private method to `ExperimentController` that queries the config for THOMPSON_SAMPLING experiments and attaches it; called on all three response paths. Also added `getConfigForExperiment()` to `ThompsonSamplingExperimentCrudService`. (This private method was later moved onto `ThompsonSamplingExperimentCrudService` itself as the public `attachConfigToExperiment()` — see "Bulk import/batch-create never created a Thompson Sampling config" below — so it could be reused outside the controller.) - **Form/overview labels still showing old terms**: Updated `packages/frontend/projects/upgrade/src/assets/i18n/en.json` — `"uniform-threshold.label.text"` → "Warmup Threshold", `"tspostdiff-thresh.label.text"` → "Minimum Draw Difference" (with updated hints). The translation keys themselves are unchanged; only the values were updated. @@ -149,8 +149,28 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - **`POST /v6/reward` is now fire-and-forget, with the config lookup cached**: this endpoint sat on the client's response path doing 8-10 sequential, uncached DB round trips (config lookup, enrollment lookup, audit insert, posterior increments, and — when a batch closes — a flush per condition) even though nothing in the client SDK waits on the result to make a UI decision. `ThompsonSamplingRewardService.recordReward()` was split into `acceptReward()` (public, synchronous — returns a "received and is being processed" receipt immediately) and `processReward()` (private, does the actual work in the background via `acceptReward().catch(...)`, never awaited by the controller). A failure that used to become a `409`/`500` HTTP response is now only logged: `logAndAbort()` (renamed from `throwConflictError()`) logs and throws an internal `RewardProcessingAborted` sentinel purely for control flow, which `acceptReward()`'s catch recognizes and does not re-log (an error of any other type gets one generic "Unexpected error processing..." log line). The `/v6/reward` swagger doc's `200` response and description were updated to describe the receipt/async-logging contract, and its `409`/`500` response entries were removed since nothing throws to the HTTP layer here anymore. Separately, `findConfigById`/`findConfigByDecisionPoint` now go through `CacheService.wrap()` under a new `CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX` (bucketed under the existing `experiments` TTL/refresh-threshold config in `CacheService`'s `PREFIX_CATEGORY` map) — the same `wrap()`-based pattern `ExperimentService.getCachedValidExperiments()` already uses, since `warmupThreshold`/`minimumDrawDifference`/`batchSize` and the joined `experiment.state` only change on an admin edit, far less often than rewards arrive. `ThompsonSamplingExperimentCrudService.createConfig()`/`updateConfig()` call a new `invalidateConfigCache()` (`cacheService.resetPrefixCache(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX)`) after writing — a full-prefix reset rather than a single-key delete, because the decision-point-keyed cache entries embed the same config fields and there's no cheap way to know which `context:site:target` keys reference a given experiment (same blunt-but-safe approach `ExperimentService.updateList()` already takes with `EXPERIMENT_KEY_PREFIX` for the analogous problem). `syncConditions()` does not invalidate, since it only touches `conditionPosteriorStates`, which these cached config objects don't include. One accepted gap: `ExperimentService.updateState()` (experiment start/stop) does not invalidate this cache, so a just-stopped experiment can still accept a reward for up to the `experiments` bucket's TTL — the same TTL-bounded staleness `getCachedValidExperiments()` already tolerates for the assignment path. +- **`minimumDrawDifference` verified against MoocLet's `tspostdiff_thresh`**: confirmed the full plumbing is live (frontend form → `ThompsonSamplingConfigValidator` (`0`–`1`) → `ThompsonSamplingExperimentConfig.minimumDrawDifference` → `ExperimentAssignmentService.assignThompsonSampling()` → `ThompsonSamplingService.selectCondition()`'s fallback check) and matches the reference policy's intent. `ts_postdiff_sample`/`thompson_sampling_postdiff` in `policies.py` draw one Beta sample per arm, and if `abs(draw_1 - draw_2) < tspostdiff_thresh`, fall back to uniform random across all arms instead of picking the highest draw — a hedge against picking a "winner" whose lead over its closest rival is really just sampling noise. But that reference implementation is hardcoded for exactly 2 arms (`thompson_sampling_postdiff`'s own docstring: "Assumes only 2 versions"; `ts_postdiff_sample` reuses the same logic unchanged, indexing `list(versions_dict.values())[0]`/`[1]` regardless of how many conditions exist) — with 3+ arms it silently compares whichever two conditions happen to be first in dict order, not the two that are actually closest, and its `else` branch re-draws every arm from scratch for the real pick rather than reusing the draws the diff check already made. `selectCondition()` doesn't have either problem: it draws once per condition, sorts, and compares `draws[0]` vs `draws[1]` (the actual top two by sampled value, however many conditions there are) before reusing that same top draw as the pick — the correct N-arm generalization of the same idea, not a port of the 2-arm-specific code. Added a test (`ThompsonSamplingService.test.ts` → "compares the top two draws by value, not the first two conditions by list order") that would fail under a MoocLet-style hardcoded-first-two comparison but passes under the current top-two-by-value implementation. + +- **Bulk import/batch-create never created a Thompson Sampling config**: `ImportExportService.addBulkExperiments()` (the shared path behind both `/experiments/import` and `/experiments/batch`) called `experimentService.create()` directly with no reference to `ThompsonSamplingExperimentCrudService` — config/posterior-state creation existed only in `ExperimentController.create()`/`update()`. An imported or batch-created `THOMPSON_SAMPLING` experiment got an `Experiment` row but no `ThompsonSamplingExperimentConfig`/`ConditionPosteriorState` rows, so `assignThompsonSampling()` would log an error and return `undefined` for every user, forever. Fixed by moving the per-call-site `if (assignmentAlgorithm === THOMPSON_SAMPLING) {...}` gating (previously duplicated across the controller's `create()`/`update()`/`one()`) into three orchestration methods on `ThompsonSamplingExperimentCrudService`: `createConfigIfApplicable()`, `syncConfigIfApplicable()`, and `attachConfigToExperiment()` (which replaced the controller's private `attachThompsonSamplingConfig()`). `ImportExportService.addBulkExperiments()` and `exportExperiment()` now call these too, so import, batch-create, and export get the same config handling as the single-experiment `POST`/`PUT`/`GET` endpoints. Reward data can't leak through this path regardless of caller: `ThompsonSamplingConfigParams`/`createConfig()` have no field for `successCount`/`failureCount`, so posterior state always starts at Beta(priorSuccess, priorFailure) with zero counts, whatever the source experiment (e.g. one round-tripped through export) had accumulated. + +- **`ConditionRewardSummary.conditionCode` actually held a `conditionId`**: every caller populated it with a condition UUID, never an actual (nullable) `conditionCode`. Renamed the field to `conditionId`, and renamed `ThompsonSamplingService.selectCondition()`'s `conditionCodes` parameter (and its private `uniformRandom()` helper) to `conditionIds`/`conditionId` throughout, so the algorithm-facing types now say what they actually hold. `ThompsonSamplingExperimentCrudService.getRewardsSummary()`'s own `conditionCode`/`code` fields are unrelated and untouched — those hold (or fall back to) a real display-facing condition code, a different concept from the algorithm's internal `conditionId` key. + +- **Beta posterior formula and priors-record construction were duplicated**: `alpha = prior.success + successCount` / `beta = prior.failure + failureCount` was written independently in both `ThompsonSamplingService.selectCondition()` and `ThompsonSamplingExperimentCrudService.getRewardsSummary()`; extracted to `ThompsonSamplingService.computePosterior()`, used by both. The `{ [conditionId]: {success, failure} }` priors-record construction was likewise duplicated between `ExperimentAssignmentService.assignThompsonSampling()` and `ThompsonSamplingExperimentCrudService.attachConfigToExperiment()`; extracted to `ThompsonSamplingService.buildPriorsRecord()`. + +- **`getRewardsSummary()` built a query directly, violating this package's own layering rule** (`packages/backend/CLAUDE.md`: "repositories own ALL query-building; services never build queries directly"): moved the `createQueryBuilder` call (joining `conditionPosteriorStates` and `condition`) into a new `ThompsonSamplingExperimentConfigRepository.findByExperimentIdWithConditions()` method. + +- **Misplaced JSDoc**: the doc comment describing `syncConditions()`'s add/remove-rows behavior was sitting above `getRewardsSummary()` (which is a read-only aggregation with no side effects). Moved to `syncConditions()`; `getRewardsSummary()` got its own accurate doc. + +- **`ThompsonSamplingRewardService.applyOrBufferReward()` mixed concerns and duplicated increment logic**: split into `applyOrBufferReward()` (buffers or immediately applies one reward) and `flushIfBatchReady()` (the experiment-wide pending-count check and flush across every condition). The immediate-apply path (`batchSize` unset/≤1) now routes through the same `flushPendingRewards()` a real batch flush uses, instead of a second hand-rolled copy of the `totalCount`/`successCount`/`failureCount` increment sequence. + +- **Frontend: duplicate `isThompsonSamplingExperiment()` reimplementation**: `experiment-conditions-section-card.component.ts` and `enrollment-condition-expandable-row.component.ts` each defined a local `isThompsonSamplingExperiment(experiment)` that reimplemented the check via `thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment?.assignmentAlgorithm)` instead of calling `ThompsonSamplingHelperService.isThompsonSamplingExperiment(experiment)` — an existing method of the same name/purpose. Both local methods now delegate to it directly (kept as thin per-component wrappers only because their templates already call them by that name in several places). + +- **Frontend: duplicated adaptive-weight tooltip translation key**: the `'experiments.details.conditions.weight-adaptive-tooltip.text'` literal was duplicated 4 times across `experiment-conditions-table.component.html` and `enrollment-condition-expandable-row.component.html`. Extracted to `THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY` in `experiments.model.ts`, referenced from both components' `.ts` files and bound in the templates instead of the literal string. The two Angular Material table structures themselves stay separate (one is a static `matColumnDef` table, the other a dynamic per-key loop over `displayedColumns`), so this only removed the duplicated literal, not the surrounding markup — the `matTooltip`/`matTooltipDisabled`/`matTooltipPosition` attribute repetition is this codebase's established idiom for every tooltip, adaptive-weight or not. + +- **Deliberately not fixed — no shared interface for future adaptive algorithms**: a code review of this branch also flagged that the only seam for a *second* adaptive algorithm is a hardcoded `if (assignmentAlgorithm === THOMPSON_SAMPLING)` check repeated across `ExperimentAssignmentService`, `ExperimentController`, `ExperimentDTO`, and the reward endpoint, with no shared `AdaptiveAssignmentAlgorithm`-style interface Thompson Sampling merely implements one instance of. Left unresolved on purpose — it's a larger architectural change, out of scope for this cleanup pass, and worth designing deliberately once a second algorithm is actually on the table rather than guessing its shape now. + ### Architecture notes -- **conditionId as algorithm key**: `ThompsonSamplingService.selectCondition()` uses condition UUIDs (not `conditionCode`) as identifiers, since `conditionCode` is nullable. `ConditionPosteriorState` rows are keyed by `conditionId`. The `priors` field in `ThompsonSamplingConfigDTO` is therefore also keyed by conditionId. +- **conditionId as algorithm key**: `ThompsonSamplingService.selectCondition()` uses condition UUIDs (not `conditionCode`) as identifiers, since `conditionCode` is nullable. `ConditionPosteriorState` rows, `ConditionRewardSummary.conditionId`, and `selectCondition()`'s own `conditionIds` parameter are all named and keyed accordingly. The `priors` field in `ThompsonSamplingConfigDTO` is therefore also keyed by conditionId. -- **Reward summary endpoint**: `GET /experiments/rewards/:id` delegates to `ThompsonSamplingExperimentCrudService.getRewardsSummary()`, which queries `ConditionPosteriorState` rows joined to conditions, computes `successes`, `failures`, `successRate`, `priorSuccess`, `priorFailure` per condition, and sorts by condition order. Fully implemented. +- **Reward summary endpoint**: `GET /experiments/rewards/:id` delegates to `ThompsonSamplingExperimentCrudService.getRewardsSummary()`, which calls `ThompsonSamplingExperimentConfigRepository.findByExperimentIdWithConditions()` to load `ConditionPosteriorState` rows joined to their conditions, computes `successes`, `failures`, `successRate`, `priorSuccess`, `priorFailure` per condition (via the shared `ThompsonSamplingService.computePosterior()`), and sorts by condition order. Fully implemented. diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index a1300b1677..e7ad6f0073 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -31,14 +31,7 @@ import { ThompsonSamplingExperimentCrudService } from '../services/ThompsonSampl import { Response } from 'express'; import { NotFoundException } from '@nestjs/common/exceptions'; import { ExperimentIdValidator } from '../DTO/ExperimentDTO'; -import { - ASSIGNMENT_ALGORITHM, - CACHE_PREFIX, - IImportError, - LIST_FILTER_MODE, - SERVER_ERROR, - ExperimentRewardsSummary, -} from 'upgrade_types'; +import { CACHE_PREFIX, IImportError, LIST_FILTER_MODE, SERVER_ERROR, ExperimentRewardsSummary } from 'upgrade_types'; import { ImportExportService } from '../services/ImportExportService'; import { getInstanceId } from '../../lib/instanceIdentity'; import { ExperimentSegmentInclusion } from '../models/ExperimentSegmentInclusion'; @@ -917,7 +910,7 @@ export class ExperimentController { @Req() request: AppRequest ): Promise { const experiment = await this.experimentService.getSingleExperiment(id, request.logger); - return this.attachThompsonSamplingConfig(experiment); + return this.thompsonSamplingCrudService.attachConfigToExperiment(experiment); } @Get('/rewards/:id') @@ -1054,15 +1047,9 @@ export class ExperimentController { const createdExperiment = await this.experimentService.create(experiment, currentUser, request.logger); - if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { - await this.thompsonSamplingCrudService.createConfig( - createdExperiment.id, - createdExperiment.conditions, - experiment.thompsonSamplingConfig ?? {} - ); - } + await this.thompsonSamplingCrudService.createConfigIfApplicable(experiment, createdExperiment); - return this.attachThompsonSamplingConfig(createdExperiment); + return this.thompsonSamplingCrudService.attachConfigToExperiment(createdExperiment); } /** @@ -1252,14 +1239,9 @@ export class ExperimentController { const updatedExperiment = await this.experimentService.update({ ...experiment, id }, currentUser, request.logger); - if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { - await this.thompsonSamplingCrudService.syncConditions(id, updatedExperiment.conditions); - if (experiment.thompsonSamplingConfig) { - await this.thompsonSamplingCrudService.updateConfig(id, experiment.thompsonSamplingConfig); - } - } + await this.thompsonSamplingCrudService.syncConfigIfApplicable(experiment, updatedExperiment); - return this.attachThompsonSamplingConfig(updatedExperiment); + return this.thompsonSamplingCrudService.attachConfigToExperiment(updatedExperiment); } /** @@ -1967,24 +1949,4 @@ export class ExperimentController { summary, }; } - - private async attachThompsonSamplingConfig(experiment: ExperimentDTO): Promise { - if (experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { - const config = await this.thompsonSamplingCrudService.getConfigForExperiment(experiment.id); - if (config) { - const priors: Record = {}; - (config.conditionPosteriorStates ?? []).forEach((state) => { - priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; - }); - - experiment.thompsonSamplingConfig = { - warmupThreshold: config.warmupThreshold, - minimumDrawDifference: config.minimumDrawDifference, - batchSize: config.batchSize, - priors, - }; - } - } - return experiment; - } } diff --git a/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts index 0faa50d21c..ed02b91162 100644 --- a/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts +++ b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts @@ -12,6 +12,18 @@ export class ThompsonSamplingExperimentConfigRepository extends Repository { + return this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .leftJoinAndSelect('conditionPosteriorStates.condition', 'condition') + .where('config.experimentId = :experimentId', { experimentId }) + .getOne(); + } + public async findByDecisionPoint( context: string, site: string, diff --git a/packages/backend/src/api/services/ExperimentAssignmentService.ts b/packages/backend/src/api/services/ExperimentAssignmentService.ts index da8a582d0d..8eeec441bb 100644 --- a/packages/backend/src/api/services/ExperimentAssignmentService.ts +++ b/packages/backend/src/api/services/ExperimentAssignmentService.ts @@ -73,12 +73,7 @@ import { ExperimentPrecomputedSegmentService } from './ExperimentPrecomputedSegm import { ExperimentPrecomputedSegment } from '../models/ExperimentPrecomputedSegment'; import { precomputedGroupKey } from './precomputedSegmentHelpers'; import { EntitySegmentMembers, EntitySegmentResolutionInput, SegmentGroupMember } from '../../types'; -import { - ThompsonSamplingService, - ConditionPrior, - ThompsonSamplingConfig, - ConditionRewardSummary, -} from './ThompsonSamplingService'; +import { ThompsonSamplingService, ThompsonSamplingConfig, ConditionRewardSummary } from './ThompsonSamplingService'; import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; export interface FactorialConditionResult { @@ -2092,16 +2087,13 @@ export class ExperimentAssignmentService { const conditionIds = experiment.conditions.map((c) => c.id); const rewardSummaries: ConditionRewardSummary[] = config.conditionPosteriorStates.map((state) => ({ - conditionCode: state.conditionId, + conditionId: state.conditionId, successCount: state.successCount, failureCount: state.failureCount, totalCount: state.totalCount, })); - const priors: Record = {}; - config.conditionPosteriorStates.forEach((state) => { - priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; - }); + const priors = this.thompsonSamplingService.buildPriorsRecord(config.conditionPosteriorStates); // Include pendingTotalCount so a batchSize buffer awaiting flush still counts toward // warmup evidence — rewards are "collected" as soon as recordReward() persists them, diff --git a/packages/backend/src/api/services/ImportExportService.ts b/packages/backend/src/api/services/ImportExportService.ts index 1d077d93c4..a521176da2 100644 --- a/packages/backend/src/api/services/ImportExportService.ts +++ b/packages/backend/src/api/services/ImportExportService.ts @@ -9,13 +9,15 @@ import { In } from 'typeorm'; import { InjectRepository } from '../../typeorm-typedi-extensions'; import { ExperimentRepository } from '../repositories/ExperimentRepository'; import { ExperimentAuditLogRepository } from '../repositories/ExperimentAuditLogRepository'; +import { ThompsonSamplingExperimentCrudService } from './ThompsonSamplingExperimentCrudService'; @Service() export class ImportExportService { constructor( @InjectRepository() protected experimentRepository: ExperimentRepository, @InjectRepository() protected experimentAuditLogRepository: ExperimentAuditLogRepository, - protected experimentService: ExperimentService + protected experimentService: ExperimentService, + protected thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService ) {} public async importExperiments(experiments: ExperimentFile[], user: UserDTO, logger: UpgradeLogger) { @@ -40,7 +42,8 @@ export class ImportExportService { experiments.map(async (experiment) => { try { const result = await this.experimentService.create(experiment, currentUser, logger); - createdExperiments.push(result); + await this.thompsonSamplingCrudService.createConfigIfApplicable(experiment, result); + createdExperiments.push(await this.thompsonSamplingCrudService.attachConfigToExperiment(result)); } catch (error) { logger.error({ message: 'Failed to create experiment during import', @@ -120,8 +123,10 @@ export class ImportExportService { return a.order - b.order; }); - const experimentRecord = this.experimentService.reducedConditionPayload( - this.experimentService.formattingPayload(this.experimentService.formattingConditionPayload(experiment)) + const experimentRecord = await this.thompsonSamplingCrudService.attachConfigToExperiment( + this.experimentService.reducedConditionPayload( + this.experimentService.formattingPayload(this.experimentService.formattingConditionPayload(experiment)) + ) ); this.experimentAuditLogRepository.saveRawJson( diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 9ee31dc7d2..3b980116ff 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -5,7 +5,8 @@ import { ConditionPosteriorStateRepository } from '../repositories/ConditionPost import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; import { ThompsonSamplingService } from './ThompsonSamplingService'; import { CacheService } from './CacheService'; -import { CACHE_PREFIX, ExperimentRewardsSummary } from 'upgrade_types'; +import { ASSIGNMENT_ALGORITHM, CACHE_PREFIX, ExperimentRewardsSummary } from 'upgrade_types'; +import { ExperimentDTO } from '../DTO/ExperimentDTO'; type ConditionRef = { id: string }; @@ -30,6 +31,63 @@ export class ThompsonSamplingExperimentCrudService { return this.configRepository.findByExperimentId(experimentId); } + /** + * Single gate for "does this experiment need a Thompson Sampling config", so every experiment + * creation path (single create, bulk import, batch create) gets config/posterior rows the same + * way instead of each caller re-checking assignmentAlgorithm itself. Only `priors`/`warmupThreshold`/ + * `batchSize`/`minimumDrawDifference` from `experiment.thompsonSamplingConfig` are ever read here — + * there is no field for success/failure counts, so posterior state always starts at zero regardless + * of what the caller's source experiment (e.g. an imported/exported one) previously accumulated. + */ + public async createConfigIfApplicable(experiment: ExperimentDTO, createdExperiment: ExperimentDTO): Promise { + if (experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return; + } + await this.createConfig( + createdExperiment.id, + createdExperiment.conditions, + experiment.thompsonSamplingConfig ?? {} + ); + } + + /** + * Update-path counterpart to createConfigIfApplicable: keeps posterior rows in sync with the + * current condition list and applies any prior/threshold changes, only for Thompson Sampling + * experiments. + */ + public async syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise { + if (experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return; + } + await this.syncConditions(updatedExperiment.id, updatedExperiment.conditions); + if (experiment.thompsonSamplingConfig) { + await this.updateConfig(updatedExperiment.id, experiment.thompsonSamplingConfig); + } + } + + /** + * Populates `experiment.thompsonSamplingConfig` from the stored config/posterior rows for API + * responses and experiment export. Only ever reads `priorSuccess`/`priorFailure` (the Beta seed) — + * never `successCount`/`failureCount` — so exporting an experiment and re-importing it carries the + * configured priors forward without also carrying forward accumulated reward evidence. + */ + public async attachConfigToExperiment(experiment: T): Promise { + if (experiment?.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return experiment; + } + const config = await this.getConfigForExperiment(experiment.id); + if (!config) { + return experiment; + } + experiment.thompsonSamplingConfig = { + warmupThreshold: config.warmupThreshold, + minimumDrawDifference: config.minimumDrawDifference, + batchSize: config.batchSize, + priors: this.thompsonSamplingService.buildPriorsRecord(config.conditionPosteriorStates ?? []), + }; + return experiment; + } + public async createConfig( experimentId: string, conditions: ConditionRef[], @@ -95,16 +153,12 @@ export class ThompsonSamplingExperimentCrudService { } /** - * Keeps ConditionPosteriorState rows in sync with the experiment's current conditions. - * Adds rows for new conditions (using default priors) and removes rows for deleted conditions. + * Per-condition reward totals and estimated win-rate weight for the experiment overview/summary + * display. Read-only aggregation — does not touch ConditionPosteriorState rows (see + * syncConditions() for that). */ public async getRewardsSummary(experimentId: string): Promise { - const config = await this.configRepository - .createQueryBuilder('config') - .leftJoinAndSelect('config.conditionPosteriorStates', 'states') - .leftJoinAndSelect('states.condition', 'condition') - .where('config.experimentId = :experimentId', { experimentId }) - .getOne(); + const config = await this.configRepository.findByExperimentIdWithConditions(experimentId); if (!config) return []; @@ -112,8 +166,12 @@ export class ThompsonSamplingExperimentCrudService { const successes = state.successCount; const failures = state.failureCount; const successRate = state.totalCount > 0 ? ((successes / state.totalCount) * 100).toFixed(1) + '%' : '0.0%'; - const alpha = state.priorSuccess + state.successCount; - const beta = state.priorFailure + state.failureCount; + const { alpha, beta } = this.thompsonSamplingService.computePosterior( + state.priorSuccess, + state.priorFailure, + state.successCount, + state.failureCount + ); return { code: state.condition?.conditionCode ?? state.conditionId, alpha, @@ -140,6 +198,10 @@ export class ThompsonSamplingExperimentCrudService { .sort((a, b) => a.order - b.order); } + /** + * Keeps ConditionPosteriorState rows in sync with the experiment's current conditions. + * Adds rows for new conditions (using default priors) and removes rows for deleted conditions. + */ public async syncConditions(experimentId: string, currentConditions: ConditionRef[]): Promise { const config = await this.configRepository.findByExperimentId(experimentId); if (!config) return; diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index 9ac163afb2..3a6f92ea1c 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -127,13 +127,13 @@ export class ThompsonSamplingRewardService { /** * Fold a reward into the posterior (successCount/totalCount), or buffer it as pending until - * batchSize reward observations have accumulated across the whole experiment (all conditions - * combined, not just this one) — batchSize paces how often posteriors move, and a reward for - * any condition is evidence toward that same shared cadence. Once the threshold is hit, every - * condition's pending buffer is flushed, not just the one that tipped it over, so a condition - * with few rewards still gets its pending counts folded in as soon as the batch closes. The raw - * event is always persisted to ThompsonSamplingReward regardless of batching — batching only - * delays when a reward affects which condition gets sampled next, it never drops data. + * batchSize reward observations have accumulated across the whole experiment — see + * flushIfBatchReady() for why that check spans every condition, not just this one. The raw event + * is always persisted to ThompsonSamplingReward regardless of batching (in processReward(), + * before this is called) — batching only delays when a reward affects which condition gets + * sampled next, it never drops data. An unset/≤1 batchSize applies the reward immediately, via + * the same flushPendingRewards() a real batch flush uses, so there's one code path for "fold a + * reward's counts into successCount/failureCount/totalCount." */ private async applyOrBufferReward( state: Pick, @@ -143,12 +143,7 @@ export class ThompsonSamplingRewardService { const effectiveBatchSize = batchSize && batchSize > 1 ? batchSize : 1; if (effectiveBatchSize <= 1) { - await this.posteriorStateRepository.increment({ id: state.id }, 'totalCount', 1); - if (success) { - await this.posteriorStateRepository.increment({ id: state.id }, 'successCount', 1); - } else { - await this.posteriorStateRepository.increment({ id: state.id }, 'failureCount', 1); - } + await this.flushPendingRewards(state.id, success ? 1 : 0, success ? 0 : 1, 1); return; } @@ -159,16 +154,30 @@ export class ThompsonSamplingRewardService { await this.posteriorStateRepository.increment({ id: state.id }, 'pendingFailureCount', 1); } - const experimentStates = await this.posteriorStateRepository.findByConfigId(state.configId); + await this.flushIfBatchReady(state.configId, effectiveBatchSize); + } + + /** + * batchSize paces how often posteriors move for the experiment as a whole, and a reward for any + * condition is evidence toward that same shared cadence — so the pending count is summed across + * every condition in the config, not just the one that just received a reward. Once the shared + * total reaches batchSize, every condition's pending buffer is flushed, not just the one that + * tipped it over, so a low-volume condition still gets its pending counts folded in as soon as + * the batch closes. + */ + private async flushIfBatchReady(configId: string, effectiveBatchSize: number): Promise { + const experimentStates = await this.posteriorStateRepository.findByConfigId(configId); const totalPending = experimentStates.reduce((sum, s) => sum + s.pendingTotalCount, 0); - if (totalPending >= effectiveBatchSize) { - await Promise.all( - experimentStates - .filter((s) => s.pendingTotalCount > 0) - .map((s) => this.flushPendingRewards(s.id, s.pendingSuccessCount, s.pendingFailureCount, s.pendingTotalCount)) - ); + if (totalPending < effectiveBatchSize) { + return; } + + await Promise.all( + experimentStates + .filter((s) => s.pendingTotalCount > 0) + .map((s) => this.flushPendingRewards(s.id, s.pendingSuccessCount, s.pendingFailureCount, s.pendingTotalCount)) + ); } private async flushPendingRewards( diff --git a/packages/backend/src/api/services/ThompsonSamplingService.ts b/packages/backend/src/api/services/ThompsonSamplingService.ts index 1427491cb9..38abe2276f 100644 --- a/packages/backend/src/api/services/ThompsonSamplingService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingService.ts @@ -6,7 +6,7 @@ export interface ConditionPrior { } export interface ConditionRewardSummary { - conditionCode: string; + conditionId: string; successCount: number; failureCount: number; totalCount: number; @@ -96,40 +96,44 @@ export class ThompsonSamplingService { /** * Select a condition using Thompson Sampling. * - * @param conditionCodes - All eligible condition codes for this experiment + * @param conditionIds - All eligible condition IDs for this experiment * @param rewardSummaries - Accumulated reward counts per condition * @param totalRewardCount - Total number of reward observations collected across all conditions, * including any not yet folded into the posteriors by a pending batch flush * @param config - Optional algorithm parameters (priors, warmup, thresholds) */ selectCondition( - conditionCodes: string[], + conditionIds: string[], rewardSummaries: ConditionRewardSummary[], totalRewardCount: number, config: ThompsonSamplingConfig = {} ): string { - if (conditionCodes.length === 0) { + if (conditionIds.length === 0) { throw new Error('Cannot select from an empty condition list'); } - if (conditionCodes.length === 1) { - return conditionCodes[0]; + if (conditionIds.length === 1) { + return conditionIds[0]; } // Warmup phase: use uniform random until sufficient reward evidence has been collected. // Gated on reward observations (not assignments) — the posteriors only move when rewards // arrive, so that's the right measure of "how much evidence do we actually have." if (config.warmupThreshold !== undefined && totalRewardCount <= config.warmupThreshold) { - return this.uniformRandom(conditionCodes); + return this.uniformRandom(conditionIds); } - const summaryMap = new Map(rewardSummaries.map((s) => [s.conditionCode, s])); - - const draws = conditionCodes.map((code) => { - const summary = summaryMap.get(code); - const prior = config.priors?.[code] ?? DEFAULT_PRIOR; - const alpha = prior.success + (summary?.successCount ?? 0); - const beta = prior.failure + (summary?.failureCount ?? 0); - return { code, draw: this.sampleBeta(alpha, beta) }; + const summaryMap = new Map(rewardSummaries.map((s) => [s.conditionId, s])); + + const draws = conditionIds.map((conditionId) => { + const summary = summaryMap.get(conditionId); + const prior = config.priors?.[conditionId] ?? DEFAULT_PRIOR; + const { alpha, beta } = this.computePosterior( + prior.success, + prior.failure, + summary?.successCount ?? 0, + summary?.failureCount ?? 0 + ); + return { conditionId, draw: this.sampleBeta(alpha, beta) }; }); draws.sort((a, b) => b.draw - a.draw); @@ -140,14 +144,44 @@ export class ThompsonSamplingService { draws.length >= 2 && draws[0].draw - draws[1].draw < config.minimumDrawDifference ) { - return this.uniformRandom(conditionCodes); + return this.uniformRandom(conditionIds); } - return draws[0].code; + return draws[0].conditionId; + } + + /** + * Beta posterior parameters for a condition, given its Beta(priorSuccess, priorFailure) prior and + * its accumulated success/failure counts. Shared by selectCondition() (which condition to draw) + * and ThompsonSamplingExperimentCrudService.getRewardsSummary() (the same math, for display) so + * a future correction to this formula can't be applied to one and missed in the other. + */ + computePosterior( + priorSuccess: number, + priorFailure: number, + successCount: number, + failureCount: number + ): { alpha: number; beta: number } { + return { alpha: priorSuccess + successCount, beta: priorFailure + failureCount }; + } + + /** + * Builds a `{ [conditionId]: { success, failure } }` prior record from posterior state rows. + * Shared by ExperimentAssignmentService (feeding the algorithm's config) and + * ThompsonSamplingExperimentCrudService (attaching config to API responses/export). + */ + buildPriorsRecord( + states: Array<{ conditionId: string; priorSuccess: number; priorFailure: number }> + ): Record { + const priors: Record = {}; + states.forEach((state) => { + priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; + }); + return priors; } - private uniformRandom(conditionCodes: string[]): string { - return conditionCodes[Math.floor(Math.random() * conditionCodes.length)]; + private uniformRandom(conditionIds: string[]): string { + return conditionIds[Math.floor(Math.random() * conditionIds.length)]; } // Beta(α, β) sampled as the ratio of two independent Gamma samples diff --git a/packages/backend/test/unit/controllers/ExperimentController.test.ts b/packages/backend/test/unit/controllers/ExperimentController.test.ts index 827b5768f6..6ca6e745a3 100644 --- a/packages/backend/test/unit/controllers/ExperimentController.test.ts +++ b/packages/backend/test/unit/controllers/ExperimentController.test.ts @@ -34,7 +34,11 @@ describe('Experiment Controller Testing', () => { Container.set(ExperimentService, new ExperimentServiceMock()); Container.set(ExperimentAssignmentService, new ExperimentAssignmentServiceMock()); Container.set(ImportExportService, new ImportExportServiceMock()); - Container.set(ThompsonSamplingExperimentCrudService, {} as any); + Container.set(ThompsonSamplingExperimentCrudService, { + createConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + syncConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + attachConfigToExperiment: jest.fn().mockImplementation((experiment) => Promise.resolve(experiment)), + } as any); }); afterAll(() => { diff --git a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts index 00e5cd3731..f2b714fb79 100644 --- a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts @@ -2283,7 +2283,10 @@ describe('Experiment Assignment Service Test', () => { ], }), }; - testedModule.thompsonSamplingService = { selectCondition: sandbox.stub().returns('condition-a') }; + testedModule.thompsonSamplingService = { + selectCondition: sandbox.stub().returns('condition-a'), + buildPriorsRecord: sandbox.stub().returns({}), + }; await (testedModule as any).assignThompsonSampling(thompsonExperiment, thompsonUser, loggerMock); diff --git a/packages/backend/test/unit/services/ImportExportService.test.ts b/packages/backend/test/unit/services/ImportExportService.test.ts new file mode 100644 index 0000000000..0dd2e0c613 --- /dev/null +++ b/packages/backend/test/unit/services/ImportExportService.test.ts @@ -0,0 +1,57 @@ +import { ImportExportService } from '../../../src/api/services/ImportExportService'; +import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; + +describe('ImportExportService', () => { + let experimentService: any; + let thompsonSamplingCrudService: any; + let service: ImportExportService; + let logger: any; + + beforeEach(() => { + experimentService = { + create: jest.fn().mockImplementation((experiment) => Promise.resolve({ ...experiment })), + }; + thompsonSamplingCrudService = { + createConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + attachConfigToExperiment: jest.fn().mockImplementation((experiment) => Promise.resolve(experiment)), + }; + logger = { info: jest.fn(), error: jest.fn() }; + service = new ImportExportService({} as any, {} as any, experimentService, thompsonSamplingCrudService); + }); + + describe('addBulkExperiments', () => { + it('creates a Thompson Sampling config for an imported/batch-created THOMPSON_SAMPLING experiment', async () => { + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 2, failure: 3 } } }, + } as any; + + await service.addBulkExperiments([experiment], {} as any, logger); + + // Without this call, an imported/batch-created THOMPSON_SAMPLING experiment previously got no + // ThompsonSamplingExperimentConfig/ConditionPosteriorState rows at all (only the single-experiment + // POST /experiments controller path wired this up), so assignment could never select a condition. + expect(thompsonSamplingCrudService.createConfigIfApplicable).toHaveBeenCalledWith( + experiment, + expect.objectContaining({ id: 'experiment-1' }) + ); + }); + + it('attaches the created config onto the returned experiment', async () => { + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + } as any; + thompsonSamplingCrudService.attachConfigToExperiment.mockImplementation((exp: any) => + Promise.resolve({ ...exp, thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } } }) + ); + + const [result] = await service.addBulkExperiments([experiment], {} as any, logger); + + expect(result.thompsonSamplingConfig).toEqual({ priors: { 'condition-1': { success: 1, failure: 1 } } }); + }); + }); +}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts index 53f7b65a81..b5de476f6a 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -1,5 +1,6 @@ import { ThompsonSamplingExperimentCrudService } from '../../../src/api/services/ThompsonSamplingExperimentCrudService'; -import { CACHE_PREFIX } from 'upgrade_types'; +import { ThompsonSamplingService } from '../../../src/api/services/ThompsonSamplingService'; +import { ASSIGNMENT_ALGORITHM, CACHE_PREFIX } from 'upgrade_types'; describe('ThompsonSamplingExperimentCrudService', () => { let configRepository: any; @@ -12,6 +13,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { save: jest.fn().mockResolvedValue({ id: 'config-1', experimentId: 'experiment-1' }), update: jest.fn().mockResolvedValue(undefined), findByExperimentId: jest.fn().mockResolvedValue(undefined), + findByExperimentIdWithConditions: jest.fn().mockResolvedValue(undefined), }; posteriorStateRepository = { save: jest.fn().mockResolvedValue(undefined), @@ -20,10 +22,14 @@ describe('ThompsonSamplingExperimentCrudService', () => { }; cacheService = { resetPrefixCache: jest.fn().mockResolvedValue(undefined) }; + // Real ThompsonSamplingService — it's a pure, dependency-free service, so using the genuine + // computePosterior/buildPriorsRecord/estimateConditionWeights implementations here (instead of + // re-mocking them) keeps this suite honest about what getRewardsSummary/attachConfigToExperiment + // actually compute. service = new ThompsonSamplingExperimentCrudService( configRepository, posteriorStateRepository, - {} as any, + new ThompsonSamplingService(), cacheService ); }); @@ -56,4 +62,169 @@ describe('ThompsonSamplingExperimentCrudService', () => { ); }); }); + + describe('createConfigIfApplicable', () => { + it('does nothing for a non-Thompson-Sampling experiment', async () => { + await service.createConfigIfApplicable( + { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any, + { + id: 'experiment-1', + conditions: [{ id: 'condition-1' }], + } as any + ); + + expect(configRepository.save).not.toHaveBeenCalled(); + expect(posteriorStateRepository.save).not.toHaveBeenCalled(); + }); + + it('creates a config seeded only from priors/thresholds, never from reward counts, for a newly created/imported experiment', async () => { + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { + warmupThreshold: 10, + batchSize: 5, + priors: { 'condition-1': { success: 7, failure: 4 } }, + // A source experiment (e.g. one carried over via export/import) might still have these + // fields on it, but ThompsonSamplingConfigParams has no slot for them and createConfig + // never reads anything but priors/thresholds — so posterior state must still start at 0. + successCount: 999, + totalCount: 999, + }, + } as any; + + await service.createConfigIfApplicable(experiment, { + id: 'experiment-1', + conditions: [{ id: 'condition-1' }], + } as any); + + expect(configRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ experimentId: 'experiment-1', warmupThreshold: 10, batchSize: 5 }) + ); + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + configId: 'config-1', + conditionId: 'condition-1', + priorSuccess: 7, + priorFailure: 4, + successCount: 0, + totalCount: 0, + }); + }); + }); + + describe('syncConfigIfApplicable', () => { + it('does nothing for a non-Thompson-Sampling experiment', async () => { + await service.syncConfigIfApplicable( + { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any, + { + id: 'experiment-1', + conditions: [], + } as any + ); + + expect(configRepository.findByExperimentId).not.toHaveBeenCalled(); + expect(configRepository.update).not.toHaveBeenCalled(); + }); + + it('syncs conditions and applies prior updates for a Thompson Sampling experiment', async () => { + configRepository.findByExperimentId.mockResolvedValue({ + id: 'config-1', + conditionPosteriorStates: [{ conditionId: 'condition-1' }], + }); + + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { priors: { 'condition-1': { success: 2, failure: 1 } } }, + } as any; + + await service.syncConfigIfApplicable(experiment, { + id: 'experiment-1', + conditions: [{ id: 'condition-1' }], + } as any); + + expect(posteriorStateRepository.update).toHaveBeenCalledWith( + { configId: 'config-1', conditionId: 'condition-1' }, + { priorSuccess: 2, priorFailure: 1 } + ); + }); + }); + + describe('attachConfigToExperiment', () => { + it('leaves a non-Thompson-Sampling experiment untouched', async () => { + const experiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any; + + const result = await service.attachConfigToExperiment(experiment); + + expect(result).toBe(experiment); + expect(configRepository.findByExperimentId).not.toHaveBeenCalled(); + }); + + it('attaches only priors (never accumulated reward counts) for a Thompson Sampling experiment', async () => { + configRepository.findByExperimentId.mockResolvedValue({ + warmupThreshold: 10, + minimumDrawDifference: 0.05, + batchSize: 5, + conditionPosteriorStates: [ + { conditionId: 'condition-1', priorSuccess: 3, priorFailure: 2, successCount: 40, failureCount: 12 }, + ], + }); + + const experiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING } as any; + + const result = await service.attachConfigToExperiment(experiment); + + expect(result.thompsonSamplingConfig).toEqual({ + warmupThreshold: 10, + minimumDrawDifference: 0.05, + batchSize: 5, + priors: { 'condition-1': { success: 3, failure: 2 } }, + }); + }); + }); + + describe('getRewardsSummary', () => { + it('returns an empty summary when no config exists', async () => { + const result = await service.getRewardsSummary('experiment-1'); + + expect(result).toEqual([]); + expect(configRepository.findByExperimentIdWithConditions).toHaveBeenCalledWith('experiment-1'); + }); + + it('computes alpha/beta from priors + counts and sorts by condition order', async () => { + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + conditionPosteriorStates: [ + { + conditionId: 'condition-2', + priorSuccess: 1, + priorFailure: 1, + successCount: 5, + failureCount: 5, + totalCount: 10, + condition: { conditionCode: 'B', order: 1 }, + }, + { + conditionId: 'condition-1', + priorSuccess: 2, + priorFailure: 3, + successCount: 8, + failureCount: 2, + totalCount: 10, + condition: { conditionCode: 'A', order: 0 }, + }, + ], + }); + + const result = await service.getRewardsSummary('experiment-1'); + + expect(result.map((r) => r.conditionCode)).toEqual(['A', 'B']); + const [conditionA] = result; + expect(conditionA).toMatchObject({ + conditionCode: 'A', + successes: 8, + failures: 2, + successRate: '80.0%', + priorSuccess: 2, + priorFailure: 3, + }); + }); + }); }); diff --git a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts index 3bc72e7fbd..a215a1964f 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts @@ -47,8 +47,8 @@ describe('ThompsonSamplingService', () => { const conditions = ['A', 'B']; // A has an overwhelmingly dominant posterior; without warmup it would always win const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, - { conditionCode: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, + { conditionId: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionId: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, ]; const config: ThompsonSamplingConfig = { warmupThreshold: 50 }; @@ -63,8 +63,8 @@ describe('ThompsonSamplingService', () => { it('exits warmup once reward count exceeds threshold', () => { const conditions = ['A', 'B']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, - { conditionCode: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, + { conditionId: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionId: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, ]; const config: ThompsonSamplingConfig = { warmupThreshold: 10 }; @@ -84,8 +84,8 @@ describe('ThompsonSamplingService', () => { it('selects the condition with a better reward history reliably', () => { const conditions = ['good', 'bad']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'good', successCount: 90, failureCount: 10, totalCount: 100 }, - { conditionCode: 'bad', successCount: 10, failureCount: 90, totalCount: 100 }, + { conditionId: 'good', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionId: 'bad', successCount: 10, failureCount: 90, totalCount: 100 }, ]; let goodCount = 0; @@ -101,10 +101,10 @@ describe('ThompsonSamplingService', () => { it('handles three or more conditions — clear winner dominates', () => { const conditions = ['A', 'B', 'C', 'D']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 5, failureCount: 95, totalCount: 100 }, - { conditionCode: 'B', successCount: 90, failureCount: 10, totalCount: 100 }, - { conditionCode: 'C', successCount: 10, failureCount: 90, totalCount: 100 }, - { conditionCode: 'D', successCount: 5, failureCount: 95, totalCount: 100 }, + { conditionId: 'A', successCount: 5, failureCount: 95, totalCount: 100 }, + { conditionId: 'B', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionId: 'C', successCount: 10, failureCount: 90, totalCount: 100 }, + { conditionId: 'D', successCount: 5, failureCount: 95, totalCount: 100 }, ]; let bCount = 0; @@ -158,7 +158,7 @@ describe('ThompsonSamplingService', () => { it('handles a condition with no reward summary entry (defaults to prior)', () => { const conditions = ['A', 'B']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 50, failureCount: 50, totalCount: 100 }, + { conditionId: 'A', successCount: 50, failureCount: 50, totalCount: 100 }, // B has no entry — treated as zero rewards, uses prior only ]; @@ -185,8 +185,8 @@ describe('ThompsonSamplingService', () => { it('does not interfere when threshold is zero', () => { const conditions = ['A', 'B']; const rewardSummaries: ConditionRewardSummary[] = [ - { conditionCode: 'A', successCount: 90, failureCount: 10, totalCount: 100 }, - { conditionCode: 'B', successCount: 10, failureCount: 90, totalCount: 100 }, + { conditionId: 'A', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionId: 'B', successCount: 10, failureCount: 90, totalCount: 100 }, ]; const config: ThompsonSamplingConfig = { minimumDrawDifference: 0 }; @@ -199,6 +199,36 @@ describe('ThompsonSamplingService', () => { } expect(aCount / runs).toBeGreaterThan(0.85); }); + + it('compares the top two draws by value, not the first two conditions by list order', () => { + // The MoocLet reference this threshold is based on (tspostdiff_thresh) hardcodes "the + // first two versions" for its diff check, which only happens to work because it assumes + // exactly 2 arms. With 3+ conditions that would mean comparing whichever two conditions + // come first in the list, not the two that are actually closest. Here B's posterior is far + // below A/C's, so a "first two by list order" comparison (B vs A) would see a huge gap and + // almost never fall back to uniform — even though A and C (identically strong) draw values + // within minimumDrawDifference of each other very often and should trigger the fallback. + const conditions = ['B', 'A', 'C']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 99, failureCount: 0, totalCount: 99 }, + { conditionId: 'B', successCount: 0, failureCount: 99, totalCount: 99 }, + { conditionId: 'C', successCount: 99, failureCount: 0, totalCount: 99 }, + ]; + const config: ThompsonSamplingConfig = { minimumDrawDifference: 0.3 }; + + let bCount = 0; + const runs = 500; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs, config) === 'B') { + bCount++; + } + } + + // B only ever wins via the uniform fallback, so a meaningful rate here proves the + // fallback is firing off of A/C's close draws — a first-two-by-list-order comparison + // would keep this near zero instead. + expect(bCount / runs).toBeGreaterThan(0.05); + }); }); }); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts index 498386f2df..6e364d69e8 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts @@ -604,6 +604,10 @@ export const THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS = { MINIMUM_DRAW_DIFFERENCE: 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text', }; +// Shared by the conditions table and the enrollment expandable row — both hide/tooltip the weight +// column the same way for adaptive experiments, and must reference the same translation key. +export const THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY = 'experiments.details.conditions.weight-adaptive-tooltip.text'; + export const EXPERIMENT_ROOT_DISPLAYED_COLUMNS = Object.values(EXPERIMENT_ROOT_COLUMN_NAMES); export interface ExperimentState { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts index 5ed935466a..af64e3c65c 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts @@ -80,7 +80,7 @@ export class ExperimentConditionsSectionCardComponent implements OnInit { } isThompsonSamplingExperiment(experiment: Experiment) { - return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment?.assignmentAlgorithm); + return this.thompsonSamplingHelperService.isThompsonSamplingExperiment(experiment); } onAddConditionClick(appContext: string, experimentId: string): void { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html index bd3ddf97e3..2635ff8848 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html @@ -24,7 +24,7 @@ mat-header-cell *matHeaderCellDef class="weight-column ft-14-600" - [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" + [matTooltip]="WEIGHT_ADAPTIVE_TOOLTIP_KEY | translate" [matTooltipDisabled]="!isThompsonSamplingExperiment" matTooltipPosition="above" > @@ -34,7 +34,7 @@ mat-cell *matCellDef="let condition" class="weight-column ft-14-400" - [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" + [matTooltip]="WEIGHT_ADAPTIVE_TOOLTIP_KEY | translate" [matTooltipDisabled]="!isThompsonSamplingExperiment" matTooltipPosition="above" > diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts index 6fd01d9b71..e5f23e575b 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts @@ -11,6 +11,7 @@ import { ExperimentCondition, ExperimentConditionRowActionEvent, EXPERIMENT_ROW_ACTION, + THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY, } from '../../../../../../../../core/experiments/store/experiments.model'; import { SharedModule } from '../../../../../../../../shared/shared.module'; import { Prior } from 'upgrade_types'; @@ -50,6 +51,8 @@ export class ExperimentConditionsTableComponent { return ['condition', 'weight', 'weightEdit', 'description', 'actions']; } + readonly WEIGHT_ADAPTIVE_TOOLTIP_KEY = THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY; + CONDITION_TRANSLATION_KEYS = { CONDITION: 'experiments.details.conditions.condition.text', DESCRIPTION: 'experiments.details.conditions.description.text', diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html index d03e78e68a..bdad133da2 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html @@ -10,7 +10,7 @@ style="justify-content: left; padding-left: 16px" class="ft-14-600" *matHeaderCellDef - [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" + [matTooltip]="WEIGHT_ADAPTIVE_TOOLTIP_KEY | translate" [matTooltipDisabled]="!isThompsonSamplingExperiment(experiment)" matTooltipPosition="above" > @@ -28,7 +28,7 @@ style="justify-content: left" class="ft-14-400" *matCellDef="let element; let i = dataIndex" - [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" + [matTooltip]="WEIGHT_ADAPTIVE_TOOLTIP_KEY | translate" [matTooltipDisabled]="!isThompsonSamplingExperiment(experiment)" matTooltipPosition="above" > diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts index 0868978fe0..caa86c65d1 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts @@ -1,5 +1,8 @@ import { Component, ChangeDetectionStrategy, Input, OnDestroy, forwardRef } from '@angular/core'; -import { ExperimentVM } from '../../../../../../../../../core/experiments/store/experiments.model'; +import { + ExperimentVM, + THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY, +} from '../../../../../../../../../core/experiments/store/experiments.model'; import { TranslateModule, TranslateService } from '@ngx-translate/core'; import { Subscription } from 'rxjs'; import { MatIconModule } from '@angular/material/icon'; @@ -28,6 +31,8 @@ export class EnrollmentConditionExpandableRowComponent implements OnDestroy { @Input() referenceId: string; @Input() experiment: ExperimentVM; + readonly WEIGHT_ADAPTIVE_TOOLTIP_KEY = THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY; + expandedId = ''; columnHeaders = {}; translateSub: Subscription; @@ -56,7 +61,7 @@ export class EnrollmentConditionExpandableRowComponent implements OnDestroy { } isThompsonSamplingExperiment(experiment: ExperimentVM): boolean { - return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment?.assignmentAlgorithm); + return this.thompsonSamplingHelperService.isThompsonSamplingExperiment(experiment); } toggleExpandableSymbol(id: string): void { diff --git a/packages/types/package.json b/packages/types/package.json index 1318b62954..c466f94714 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -23,9 +23,6 @@ "typescript": "~5.3.3" }, "dependencies": { - "class-transformer": "^0.5.1", - "class-validator": "^0.14.1", - "reflect-metadata": "^0.2.2", "tslib": "2.8.1" } } \ No newline at end of file diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json index f0d80d6d3d..1d5a0024f0 100644 --- a/packages/types/tsconfig.json +++ b/packages/types/tsconfig.json @@ -6,8 +6,6 @@ "sourceMap": true, "declaration": true, "moduleResolution": "node", - "emitDecoratorMetadata": true, - "experimentalDecorators": true, "importHelpers": true, "module": "commonjs", "target": "es2015", From e86c667a1f3e8991204d904cd5ee6275affc19c8 Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:13:28 -0400 Subject: [PATCH 08/28] Change reward endpoint to acceptReward in skill docs Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .claude/skills/setup-perftrace/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/setup-perftrace/SKILL.md b/.claude/skills/setup-perftrace/SKILL.md index 92ef619f57..546a7e660e 100644 --- a/.claude/skills/setup-perftrace/SKILL.md +++ b/.claude/skills/setup-perftrace/SKILL.md @@ -387,7 +387,7 @@ file they live in. Verify against the branch rather than trusting this table if | `PATCH /v6/groupmembership` | `experimentUserService.updateGroupMembership` | controller | | `PATCH /v6/workinggroup` | `experimentUserService.updateWorkingGroup` | controller | | `PATCH /v6/useraliases` | `experimentUserService.setAliasesForUser(aliases=N)` | controller | -| `POST /v6/reward` | `thompsonSamplingRewardService.recordReward` | controller | +| `POST /v6/reward` | `thompsonSamplingRewardService.acceptReward` | controller | | `POST /v6/mark` | `experimentAssignmentService.markExperimentPoint` | controller | | " | 8 spans inside `markExperimentPoint` — `previewUserService.findOneFromCache`, `getCachedExperiments`, `checkUserOrGroupIsGloballyExcluded`, `experimentLevelExclusionInclusion`, `monitoredDecisionPointRepository.findOne`, `saveGroupExclusionDoc`, `updateEnrollmentExclusionDocumentsAndCheckEndingCriteria`, `monitoredDecisionPointRepository.saveRawJson` | `services/ExperimentAssignmentService.ts` | | `POST /v6/assign` | `formatAssignments` (sync) | controller | From 15efa69ee1c72cb0a4d674317f4f44a1f4df0d42 Mon Sep 17 00:00:00 2001 From: doswalt Date: Fri, 4 Sep 2026 13:28:06 -0400 Subject: [PATCH 09/28] a small abstraction to anticipate adding new algorithm type configurations later, avoids larger refactors since we will not likely add many algorithms, and we just dont want to guess and make overly abstract for no reason --- CLAUDE.md | 6 ++- .../api/controllers/ExperimentController.ts | 14 ++++--- ...aptiveExperimentConfigDispatcherService.ts | 39 +++++++++++++++++++ .../AdaptiveExperimentConfigService.ts | 13 +++++++ .../src/api/services/ImportExportService.ts | 10 ++--- .../ThompsonSamplingExperimentCrudService.ts | 3 +- .../unit/services/ImportExportService.test.ts | 10 ++--- 7 files changed, 77 insertions(+), 18 deletions(-) create mode 100644 packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts create mode 100644 packages/backend/src/api/services/AdaptiveExperimentConfigService.ts diff --git a/CLAUDE.md b/CLAUDE.md index ed5dba3ebb..a001b58454 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -167,7 +167,11 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - **Frontend: duplicated adaptive-weight tooltip translation key**: the `'experiments.details.conditions.weight-adaptive-tooltip.text'` literal was duplicated 4 times across `experiment-conditions-table.component.html` and `enrollment-condition-expandable-row.component.html`. Extracted to `THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY` in `experiments.model.ts`, referenced from both components' `.ts` files and bound in the templates instead of the literal string. The two Angular Material table structures themselves stay separate (one is a static `matColumnDef` table, the other a dynamic per-key loop over `displayedColumns`), so this only removed the duplicated literal, not the surrounding markup — the `matTooltip`/`matTooltipDisabled`/`matTooltipPosition` attribute repetition is this codebase's established idiom for every tooltip, adaptive-weight or not. -- **Deliberately not fixed — no shared interface for future adaptive algorithms**: a code review of this branch also flagged that the only seam for a *second* adaptive algorithm is a hardcoded `if (assignmentAlgorithm === THOMPSON_SAMPLING)` check repeated across `ExperimentAssignmentService`, `ExperimentController`, `ExperimentDTO`, and the reward endpoint, with no shared `AdaptiveAssignmentAlgorithm`-style interface Thompson Sampling merely implements one instance of. Left unresolved on purpose — it's a larger architectural change, out of scope for this cleanup pass, and worth designing deliberately once a second algorithm is actually on the table rather than guessing its shape now. +- **Extensibility analysis, and the low-risk pieces of it that were implemented**: a follow-up analysis (`adaptive-algorithm-extensibility-analysis.md`) scoped what to change now vs. wait on, given the expectation of at most 2-3 adaptive algorithms ever, not a general plugin framework. Two "do now" items: + - **Backend config CRUD dispatch (done)**: `ThompsonSamplingExperimentCrudService`'s three self-gated methods (`createConfigIfApplicable`, `syncConfigIfApplicable`, `attachConfigToExperiment`) were extracted into an `AdaptiveExperimentConfigService` interface (`src/api/services/AdaptiveExperimentConfigService.ts`). A new `AdaptiveExperimentConfigDispatcherService` holds an array of implementations (today just Thompson Sampling's) and loops over it for each of the three methods; `ExperimentController` (`one()`, `create()`, `update()`) and `ImportExportService` (`addBulkExperiments()`, `exportExperiment()`) now call the dispatcher instead of `ThompsonSamplingExperimentCrudService` directly. `ExperimentController.thompsonSamplingCrudService` is still injected and used directly only for `GET /experiments/rewards/:id` (`getRewardsSummary()`), which isn't part of the three-method interface and stays Thompson-Sampling-specific. Adding algorithm #2's config service means adding it to the array in `AdaptiveExperimentConfigDispatcherService`'s constructor — no call-site changes. + - **Frontend swappable config form (already satisfied, no change needed)**: the analysis recommended extracting a standalone Thompson-Sampling-specific form component with a small `[existingConfig]`/`(configChange)`/`(validityChange)`-style contract, swapped in by a `@switch` on `assignmentAlgorithm`. Checked `upsert-experiment-modal.component.ts`/`.html` and found this already exists — `TsConfigurablePolicyParametersFormComponent` (`.../upsert-experiment-modal/ts-configurable-policy-parameters-form/`) is a standalone component with exactly that contract (`@Input existingPolicyParams`/`disabled`, `@Output parametersChange`/`validationChange`/`formChanged`), hosted behind a single `@if (assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING)` in the parent modal. A single `@if` is simpler than `@switch` for one case and converts trivially to `@switch`/`@else if` when algorithm #2 needs its own form component — not worth doing preemptively. + - **Left alone, per the analysis**: the reward pipeline split (waits for a second reward-consuming algorithm to know its real shape), the conditions-table column-swapping, renaming the Thompson-Sampling-named tables/summary shape to something generic, and any dynamic component/plugin registry. Same reasoning as the original "no shared interface" note below still applies to the *assignment/reward* seam — only the config-CRUD seam was formalized. +- **Deliberately not fixed — no shared interface for the assignment/reward seam**: the only seam for a *second* adaptive algorithm's assignment and reward handling is still a hardcoded `if (assignmentAlgorithm === THOMPSON_SAMPLING)` check in `ExperimentAssignmentService`, `ExperimentDTO`, and the reward endpoint (the config-CRUD seam described above is now the exception). Left unresolved on purpose — it's a larger architectural change, out of scope for this cleanup pass, and worth designing deliberately once a second algorithm is actually on the table rather than guessing its shape now. ### Architecture notes diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index e7ad6f0073..65c8852dc2 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -28,6 +28,7 @@ import { AppRequest, PaginationResponse } from '../../types'; import { ExperimentDTO, ExperimentFile, ValidatedExperimentError } from '../DTO/ExperimentDTO'; import { ExperimentIds } from './validators/ExperimentIdsValidator'; import { ThompsonSamplingExperimentCrudService } from '../services/ThompsonSamplingExperimentCrudService'; +import { AdaptiveExperimentConfigDispatcherService } from '../services/AdaptiveExperimentConfigDispatcherService'; import { Response } from 'express'; import { NotFoundException } from '@nestjs/common/exceptions'; import { ExperimentIdValidator } from '../DTO/ExperimentDTO'; @@ -654,7 +655,8 @@ export class ExperimentController { public experimentAssignmentService: ExperimentAssignmentService, public importExportService: ImportExportService, public cacheService: CacheService, - public thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService + public thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService, + public adaptiveExperimentConfigDispatcher: AdaptiveExperimentConfigDispatcherService ) {} /** @@ -910,7 +912,7 @@ export class ExperimentController { @Req() request: AppRequest ): Promise { const experiment = await this.experimentService.getSingleExperiment(id, request.logger); - return this.thompsonSamplingCrudService.attachConfigToExperiment(experiment); + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(experiment); } @Get('/rewards/:id') @@ -1047,9 +1049,9 @@ export class ExperimentController { const createdExperiment = await this.experimentService.create(experiment, currentUser, request.logger); - await this.thompsonSamplingCrudService.createConfigIfApplicable(experiment, createdExperiment); + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, createdExperiment); - return this.thompsonSamplingCrudService.attachConfigToExperiment(createdExperiment); + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(createdExperiment); } /** @@ -1239,9 +1241,9 @@ export class ExperimentController { const updatedExperiment = await this.experimentService.update({ ...experiment, id }, currentUser, request.logger); - await this.thompsonSamplingCrudService.syncConfigIfApplicable(experiment, updatedExperiment); + await this.adaptiveExperimentConfigDispatcher.syncConfigIfApplicable(experiment, updatedExperiment); - return this.thompsonSamplingCrudService.attachConfigToExperiment(updatedExperiment); + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(updatedExperiment); } /** diff --git a/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts b/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts new file mode 100644 index 0000000000..9058fbd854 --- /dev/null +++ b/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts @@ -0,0 +1,39 @@ +import { Service } from 'typedi'; +import { ExperimentDTO } from '../DTO/ExperimentDTO'; +import { AdaptiveExperimentConfigService } from './AdaptiveExperimentConfigService'; +import { ThompsonSamplingExperimentCrudService } from './ThompsonSamplingExperimentCrudService'; + +/** + * Fans the config lifecycle for an experiment out to every registered adaptive + * algorithm's config service. Each service self-gates on assignmentAlgorithm, so + * adding a second adaptive algorithm means adding its service to `services` here -- + * no branching needed at the call sites in ExperimentController/ImportExportService. + */ +@Service() +export class AdaptiveExperimentConfigDispatcherService implements AdaptiveExperimentConfigService { + private readonly services: AdaptiveExperimentConfigService[]; + + constructor(thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService) { + this.services = [thompsonSamplingCrudService]; + } + + public async createConfigIfApplicable(experiment: ExperimentDTO, createdExperiment: ExperimentDTO): Promise { + for (const service of this.services) { + await service.createConfigIfApplicable(experiment, createdExperiment); + } + } + + public async syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise { + for (const service of this.services) { + await service.syncConfigIfApplicable(experiment, updatedExperiment); + } + } + + public async attachConfigToExperiment(experiment: T): Promise { + let result = experiment; + for (const service of this.services) { + result = await service.attachConfigToExperiment(result); + } + return result; + } +} diff --git a/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts b/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts new file mode 100644 index 0000000000..ca44a11bbd --- /dev/null +++ b/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts @@ -0,0 +1,13 @@ +import { ExperimentDTO } from '../DTO/ExperimentDTO'; + +/** + * Contract for an adaptive assignment algorithm's per-experiment config lifecycle + * (Thompson Sampling today). Each implementation checks experiment.assignmentAlgorithm + * itself and no-ops when it doesn't apply, so callers can dispatch to every registered + * implementation without branching on algorithm. + */ +export interface AdaptiveExperimentConfigService { + createConfigIfApplicable(experiment: ExperimentDTO, createdExperiment: ExperimentDTO): Promise; + syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise; + attachConfigToExperiment(experiment: T): Promise; +} diff --git a/packages/backend/src/api/services/ImportExportService.ts b/packages/backend/src/api/services/ImportExportService.ts index a521176da2..6cf69b5f2b 100644 --- a/packages/backend/src/api/services/ImportExportService.ts +++ b/packages/backend/src/api/services/ImportExportService.ts @@ -9,7 +9,7 @@ import { In } from 'typeorm'; import { InjectRepository } from '../../typeorm-typedi-extensions'; import { ExperimentRepository } from '../repositories/ExperimentRepository'; import { ExperimentAuditLogRepository } from '../repositories/ExperimentAuditLogRepository'; -import { ThompsonSamplingExperimentCrudService } from './ThompsonSamplingExperimentCrudService'; +import { AdaptiveExperimentConfigDispatcherService } from './AdaptiveExperimentConfigDispatcherService'; @Service() export class ImportExportService { @@ -17,7 +17,7 @@ export class ImportExportService { @InjectRepository() protected experimentRepository: ExperimentRepository, @InjectRepository() protected experimentAuditLogRepository: ExperimentAuditLogRepository, protected experimentService: ExperimentService, - protected thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService + protected adaptiveExperimentConfigDispatcher: AdaptiveExperimentConfigDispatcherService ) {} public async importExperiments(experiments: ExperimentFile[], user: UserDTO, logger: UpgradeLogger) { @@ -42,8 +42,8 @@ export class ImportExportService { experiments.map(async (experiment) => { try { const result = await this.experimentService.create(experiment, currentUser, logger); - await this.thompsonSamplingCrudService.createConfigIfApplicable(experiment, result); - createdExperiments.push(await this.thompsonSamplingCrudService.attachConfigToExperiment(result)); + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, result); + createdExperiments.push(await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(result)); } catch (error) { logger.error({ message: 'Failed to create experiment during import', @@ -123,7 +123,7 @@ export class ImportExportService { return a.order - b.order; }); - const experimentRecord = await this.thompsonSamplingCrudService.attachConfigToExperiment( + const experimentRecord = await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment( this.experimentService.reducedConditionPayload( this.experimentService.formattingPayload(this.experimentService.formattingConditionPayload(experiment)) ) diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 3b980116ff..b186a81502 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -7,6 +7,7 @@ import { ThompsonSamplingService } from './ThompsonSamplingService'; import { CacheService } from './CacheService'; import { ASSIGNMENT_ALGORITHM, CACHE_PREFIX, ExperimentRewardsSummary } from 'upgrade_types'; import { ExperimentDTO } from '../DTO/ExperimentDTO'; +import { AdaptiveExperimentConfigService } from './AdaptiveExperimentConfigService'; type ConditionRef = { id: string }; @@ -19,7 +20,7 @@ export interface ThompsonSamplingConfigParams { } @Service() -export class ThompsonSamplingExperimentCrudService { +export class ThompsonSamplingExperimentCrudService implements AdaptiveExperimentConfigService { constructor( @InjectRepository() private configRepository: ThompsonSamplingExperimentConfigRepository, @InjectRepository() private posteriorStateRepository: ConditionPosteriorStateRepository, diff --git a/packages/backend/test/unit/services/ImportExportService.test.ts b/packages/backend/test/unit/services/ImportExportService.test.ts index 0dd2e0c613..0540f6774b 100644 --- a/packages/backend/test/unit/services/ImportExportService.test.ts +++ b/packages/backend/test/unit/services/ImportExportService.test.ts @@ -3,7 +3,7 @@ import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; describe('ImportExportService', () => { let experimentService: any; - let thompsonSamplingCrudService: any; + let adaptiveExperimentConfigDispatcher: any; let service: ImportExportService; let logger: any; @@ -11,12 +11,12 @@ describe('ImportExportService', () => { experimentService = { create: jest.fn().mockImplementation((experiment) => Promise.resolve({ ...experiment })), }; - thompsonSamplingCrudService = { + adaptiveExperimentConfigDispatcher = { createConfigIfApplicable: jest.fn().mockResolvedValue(undefined), attachConfigToExperiment: jest.fn().mockImplementation((experiment) => Promise.resolve(experiment)), }; logger = { info: jest.fn(), error: jest.fn() }; - service = new ImportExportService({} as any, {} as any, experimentService, thompsonSamplingCrudService); + service = new ImportExportService({} as any, {} as any, experimentService, adaptiveExperimentConfigDispatcher); }); describe('addBulkExperiments', () => { @@ -33,7 +33,7 @@ describe('ImportExportService', () => { // Without this call, an imported/batch-created THOMPSON_SAMPLING experiment previously got no // ThompsonSamplingExperimentConfig/ConditionPosteriorState rows at all (only the single-experiment // POST /experiments controller path wired this up), so assignment could never select a condition. - expect(thompsonSamplingCrudService.createConfigIfApplicable).toHaveBeenCalledWith( + expect(adaptiveExperimentConfigDispatcher.createConfigIfApplicable).toHaveBeenCalledWith( experiment, expect.objectContaining({ id: 'experiment-1' }) ); @@ -45,7 +45,7 @@ describe('ImportExportService', () => { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [{ id: 'condition-1' }], } as any; - thompsonSamplingCrudService.attachConfigToExperiment.mockImplementation((exp: any) => + adaptiveExperimentConfigDispatcher.attachConfigToExperiment.mockImplementation((exp: any) => Promise.resolve({ ...exp, thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } } }) ); From f1bc4d27ebce4b96a84871c2ce80089a4b3eea34 Mon Sep 17 00:00:00 2001 From: doswalt Date: Fri, 4 Sep 2026 15:50:29 -0400 Subject: [PATCH 10/28] fix 8 live issues from Copilot code review: algorithm-transition config sync, concurrent reward race, TS+within-subjects validation, orphaned experiment on partial create failure, weight-map collision, partial-update field nulling, and stuck TS form validity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified each finding against current HEAD before fixing (several of Copilot's 24 comments were already stale, posted against earlier commits) — see CLAUDE.md for the full rundown of what was fixed vs. already resolved vs. out of scope. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 10 ++ packages/backend/src/api/DTO/ExperimentDTO.ts | 32 +++++ .../api/controllers/ExperimentController.ts | 10 +- ...mpsonSamplingExperimentConfigRepository.ts | 27 +++-- .../src/api/services/ImportExportService.ts | 9 +- .../ThompsonSamplingExperimentCrudService.ts | 71 +++++++++-- .../services/ThompsonSamplingRewardService.ts | 111 +++++++++--------- .../test/unit/DTO/ExperimentDTO.test.ts | 46 ++++++++ ...mpsonSamplingExperimentCrudService.test.ts | 94 ++++++++++++++- .../ThompsonSamplingRewardService.test.ts | 47 ++++++-- .../upsert-experiment-modal.component.ts | 30 +++++ 11 files changed, 392 insertions(+), 95 deletions(-) create mode 100644 packages/backend/test/unit/DTO/ExperimentDTO.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index a001b58454..987b8c211a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -173,6 +173,16 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - **Left alone, per the analysis**: the reward pipeline split (waits for a second reward-consuming algorithm to know its real shape), the conditions-table column-swapping, renaming the Thompson-Sampling-named tables/summary shape to something generic, and any dynamic component/plugin registry. Same reasoning as the original "no shared interface" note below still applies to the *assignment/reward* seam — only the config-CRUD seam was formalized. - **Deliberately not fixed — no shared interface for the assignment/reward seam**: the only seam for a *second* adaptive algorithm's assignment and reward handling is still a hardcoded `if (assignmentAlgorithm === THOMPSON_SAMPLING)` check in `ExperimentAssignmentService`, `ExperimentDTO`, and the reward endpoint (the config-CRUD seam described above is now the exception). Left unresolved on purpose — it's a larger architectural change, out of scope for this cleanup pass, and worth designing deliberately once a second algorithm is actually on the table rather than guessing its shape now. +- **Copilot code-review fixes**: a Copilot review of this branch surfaced 24 comments; after checking each against the current code (several were already stale, having been posted against earlier commits), 8 were live, ranked, and fixed: + - **Algorithm-transition asymmetry, both directions**: `ThompsonSamplingExperimentCrudService.syncConfigIfApplicable()` previously assumed a config already existed on update. Switching an experiment *to* Thompson Sampling via edit never created one (assignment silently broke forever); switching *away from* it never deleted the old one (the reward endpoint could keep treating a now-non-adaptive experiment as Thompson Sampling). Fixed both: creates the config if missing when switching in, deletes it via a new `deleteConfigIfExists()` when switching out. `ThompsonSamplingExperimentConfigRepository.findByDecisionPoint()`/`findConfigsForActivelyEnrollingExperiments()` also gained an `assignmentAlgorithm` filter as defense-in-depth against any config row that outlives the delete. + - **Concurrent reward race**: `applyOrBufferReward()`/`flushPendingRewards()` used separate increment/read/reset calls with no locking, so two rewards for the same experiment arriving close together could double-apply or lose pending counts. Rewritten to run inside one transaction taking a `pessimistic_write` lock (ordered by id) across every `ConditionPosteriorState` row for the config before mutating any of them. + - **Thompson Sampling + Within-Subjects**: Within-Subjects assignment never stores a condition on the individual enrollment (tracked per-repeat via `RepeatedEnrollment` instead), so Thompson Sampling's reward path — which reads that enrollment's `conditionId` — could never succeed for that combination, and nothing rejected it. Added `IsAssignmentAlgorithmCompatibleWithUnit()` to `ExperimentDTO` (backend) and disabled/auto-reset the Thompson Sampling option when Within-Subjects is selected in `upsert-experiment-modal.component.ts` (frontend). + - **No rollback on partial create failure**: `ExperimentController.create()` and `ImportExportService.addBulkExperiments()` committed the `Experiment` row before creating its adaptive config; a config-creation failure left an orphaned, permanently-broken experiment behind. Both now delete the just-created experiment if `createConfigIfApplicable()` throws. + - **`getRewardsSummary()` weight-map collision**: was keyed by `conditionCode`, which has no uniqueness constraint (only `ExperimentCondition.twoCharacterId` is unique) — two conditions sharing a code would collide and one would silently show the other's `estimatedWeight`. Now keyed by `conditionId`, matching the pattern the algorithm-facing code already uses for the same reason. + - **`updateConfig()` nulling out omitted fields**: `params.field ?? null` cleared any field a partial payload omitted. Now only fields actually present in `params` are included in the update. + - **`isTSFormValid$` stuck after leaving Thompson Sampling**: if the TS sub-form was invalid at the moment the algorithm was switched away, the child component was destroyed without emitting a final validity event, permanently disabling Save. `checkForAlgorithmChange()`'s else-branch now resets `isTSFormValid$`/`isTSFormChanged$`. + - Not fixed (explicitly out of scope, per this branch's data): the migration's default/zero-seeded posterior state for remapped `ts_configurable` experiments — no such experiments exist in the environments this migrates to. + ### Architecture notes - **conditionId as algorithm key**: `ThompsonSamplingService.selectCondition()` uses condition UUIDs (not `conditionCode`) as identifiers, since `conditionCode` is nullable. `ConditionPosteriorState` rows, `ConditionRewardSummary.conditionId`, and `selectCondition()`'s own `conditionIds` parameter are all named and keyed accordingly. The `priors` field in `ThompsonSamplingConfigDTO` is therefore also keyed by conditionId. diff --git a/packages/backend/src/api/DTO/ExperimentDTO.ts b/packages/backend/src/api/DTO/ExperimentDTO.ts index 0dfd2a7773..513718a480 100644 --- a/packages/backend/src/api/DTO/ExperimentDTO.ts +++ b/packages/backend/src/api/DTO/ExperimentDTO.ts @@ -381,6 +381,7 @@ abstract class BaseExperimentWithoutPayload { @IsOptional() @IsEnum(ASSIGNMENT_ALGORITHM) + @IsAssignmentAlgorithmCompatibleWithUnit() public assignmentAlgorithm?: ASSIGNMENT_ALGORITHM; // TODO add conditional validity here ie endOn is null @@ -497,6 +498,37 @@ function IsAssignmentUnitGroupConsistent(validationOptions?: ValidationOptions) }; } +/** + * Within-Subjects assignment never runs through assignExperiment()/assignThompsonSampling() -- + * the individual enrollment's condition is always stored as null, with the per-repeat condition + * tracked separately via RepeatedEnrollment instead. Thompson Sampling's reward path then reads + * that null conditionId when trying to record a reward, which can never succeed. Reject the + * combination outright rather than let it silently produce an experiment whose rewards can never + * be recorded. + */ +function IsAssignmentAlgorithmCompatibleWithUnit(validationOptions?: ValidationOptions) { + return function (object: any, propertyName: string) { + registerDecorator({ + name: 'isAssignmentAlgorithmCompatibleWithUnit', + target: object.constructor, + propertyName: propertyName, + options: validationOptions, + validator: { + validate(assignmentAlgorithmValue: any, args: ValidationArguments) { + const experiment = args.object as any; + return !( + assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING && + experiment.assignmentUnit === ASSIGNMENT_UNIT.WITHIN_SUBJECTS + ); + }, + defaultMessage() { + return 'Thompson Sampling cannot be used with Within-Subjects assignment: rewards cannot be attributed to a condition under that assignment unit.'; + }, + }, + }); + }; +} + const MAX_NUMBER_INPUT = 1_000_000; const MIN_PRIOR_VALUE = 1; diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index 65c8852dc2..fee7748a75 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -1049,7 +1049,15 @@ export class ExperimentController { const createdExperiment = await this.experimentService.create(experiment, currentUser, request.logger); - await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, createdExperiment); + try { + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, createdExperiment); + } catch (error) { + // The experiment row already committed above. Without this, a failed adaptive-config write + // would leave a Thompson Sampling experiment with no config/posterior rows behind -- invisible + // and permanently unable to assign a condition. Remove it rather than leave it orphaned. + await this.experimentService.delete(createdExperiment.id, currentUser, { logger: request.logger }); + throw error; + } return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(createdExperiment); } diff --git a/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts index ed02b91162..5575051cf2 100644 --- a/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts +++ b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts @@ -1,7 +1,7 @@ import { Repository } from 'typeorm'; import { EntityRepository } from '../../typeorm-typedi-extensions'; import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; -import { EXPERIMENT_STATE } from 'upgrade_types'; +import { ASSIGNMENT_ALGORITHM, EXPERIMENT_STATE } from 'upgrade_types'; @EntityRepository(ThompsonSamplingExperimentConfig) export class ThompsonSamplingExperimentConfigRepository extends Repository { @@ -29,15 +29,21 @@ export class ThompsonSamplingExperimentConfigRepository extends Repository { - return this.createQueryBuilder('config') - .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') - .leftJoinAndSelect('config.experiment', 'experiment') - .leftJoinAndSelect('experiment.partitions', 'decisionPoint') - .where('experiment.state = :state', { state: EXPERIMENT_STATE.ENROLLING }) - .andWhere(':context = ANY(experiment.context)', { context }) - .andWhere('decisionPoint.site = :site', { site }) - .andWhere('decisionPoint.target = :target', { target }) - .getMany(); + return ( + this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .leftJoinAndSelect('config.experiment', 'experiment') + .leftJoinAndSelect('experiment.partitions', 'decisionPoint') + .where('experiment.state = :state', { state: EXPERIMENT_STATE.ENROLLING }) + // Defends against a stale config row surviving an algorithm change away from Thompson + // Sampling (deleteConfigIfExists() is the primary fix -- this guards against any config row + // that outlives it, e.g. one created before that fix shipped). + .andWhere('experiment.assignmentAlgorithm = :algorithm', { algorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }) + .andWhere(':context = ANY(experiment.context)', { context }) + .andWhere('decisionPoint.site = :site', { site }) + .andWhere('decisionPoint.target = :target', { target }) + .getMany() + ); } public async findConfigsForActivelyEnrollingExperiments(): Promise { @@ -45,6 +51,7 @@ export class ThompsonSamplingExperimentConfigRepository extends Repository { try { const result = await this.experimentService.create(experiment, currentUser, logger); - await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, result); + try { + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, result); + } catch (configError) { + // Same reasoning as the single-experiment POST /experiments path: don't leave an + // orphaned, config-less Thompson Sampling experiment behind when this step fails. + await this.experimentService.delete(result.id, currentUser, { logger }); + throw configError; + } createdExperiments.push(await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(result)); } catch (error) { logger.error({ diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index b186a81502..0c9a8467ba 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -54,12 +54,28 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment /** * Update-path counterpart to createConfigIfApplicable: keeps posterior rows in sync with the * current condition list and applies any prior/threshold changes, only for Thompson Sampling - * experiments. + * experiments. Also handles both directions of an algorithm change on an existing experiment: + * switching TO Thompson Sampling creates the config that createConfigIfApplicable never got a + * chance to (this is an update, not the original create), and switching AWAY FROM it deletes any + * config left over from before, so the reward path can't keep treating a now-non-adaptive + * experiment as Thompson Sampling. */ public async syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise { if (experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + await this.deleteConfigIfExists(updatedExperiment.id); return; } + + const existingConfig = await this.getConfigForExperiment(updatedExperiment.id); + if (!existingConfig) { + await this.createConfig( + updatedExperiment.id, + updatedExperiment.conditions, + experiment.thompsonSamplingConfig ?? {} + ); + return; + } + await this.syncConditions(updatedExperiment.id, updatedExperiment.conditions); if (experiment.thompsonSamplingConfig) { await this.updateConfig(updatedExperiment.id, experiment.thompsonSamplingConfig); @@ -120,14 +136,24 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment } public async updateConfig(experimentId: string, params: ThompsonSamplingConfigParams): Promise { - await this.configRepository.update( - { experimentId }, - { - warmupThreshold: params.warmupThreshold ?? null, - minimumDrawDifference: params.minimumDrawDifference ?? null, - batchSize: params.batchSize ?? null, - } - ); + // Only touch fields the caller actually provided -- `params.field ?? null` would otherwise + // silently clear any field a partial payload omits, since each is independently optional. + const fieldsToUpdate: Partial< + Pick + > = {}; + if (params.warmupThreshold !== undefined) { + fieldsToUpdate.warmupThreshold = params.warmupThreshold; + } + if (params.minimumDrawDifference !== undefined) { + fieldsToUpdate.minimumDrawDifference = params.minimumDrawDifference; + } + if (params.batchSize !== undefined) { + fieldsToUpdate.batchSize = params.batchSize; + } + + if (Object.keys(fieldsToUpdate).length > 0) { + await this.configRepository.update({ experimentId }, fieldsToUpdate); + } await this.invalidateConfigCache(); @@ -174,7 +200,7 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment state.failureCount ); return { - code: state.condition?.conditionCode ?? state.conditionId, + conditionId: state.conditionId, alpha, beta, conditionCode: state.condition?.conditionCode ?? state.conditionId, @@ -187,14 +213,17 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment }; }); + // Keyed by conditionId, not conditionCode: conditionCode has no uniqueness constraint (only + // ExperimentCondition.twoCharacterId is unique), so two conditions sharing a code would + // otherwise collide in the weight map and silently swap estimatedWeight values. const weightMap = this.thompsonSamplingService.estimateConditionWeights( - rows.map((r) => ({ code: r.code, alpha: r.alpha, beta: r.beta })) + rows.map((r) => ({ code: r.conditionId, alpha: r.alpha, beta: r.beta })) ); return rows - .map(({ code: _code, alpha: _alpha, beta: _beta, ...rest }) => ({ + .map(({ conditionId, alpha: _alpha, beta: _beta, ...rest }) => ({ ...rest, - estimatedWeight: weightMap[rest.conditionCode], + estimatedWeight: weightMap[conditionId], })) .sort((a, b) => a.order - b.order); } @@ -230,6 +259,22 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment } } + /** + * Deletes the config (and, via ON DELETE CASCADE, its posterior-state rows) when an experiment + * that used to be Thompson Sampling is switched to a different algorithm. Without this, the + * config row would linger and ThompsonSamplingExperimentConfigRepository's enrolling-state-only + * queries would keep finding it, letting the reward path treat a now-non-adaptive experiment as + * if it were still Thompson Sampling. + */ + private async deleteConfigIfExists(experimentId: string): Promise { + const config = await this.configRepository.findByExperimentId(experimentId); + if (!config) { + return; + } + await this.configRepository.remove(config); + await this.invalidateConfigCache(); + } + /** * Clears every cached config lookup ThompsonSamplingRewardService may have made — both the * by-experimentId and by-decision-point keys share this prefix. A targeted delete of just the diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index 3a6f92ea1c..a44d4e62e0 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -1,4 +1,5 @@ import { Service } from 'typedi'; +import { EntityManager } from 'typeorm'; import { InjectRepository } from '../../typeorm-typedi-extensions'; import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; import { BinaryRewardAllowedValue, CACHE_PREFIX, EXPERIMENT_STATE } from 'upgrade_types'; @@ -127,13 +128,22 @@ export class ThompsonSamplingRewardService { /** * Fold a reward into the posterior (successCount/totalCount), or buffer it as pending until - * batchSize reward observations have accumulated across the whole experiment — see - * flushIfBatchReady() for why that check spans every condition, not just this one. The raw event - * is always persisted to ThompsonSamplingReward regardless of batching (in processReward(), - * before this is called) — batching only delays when a reward affects which condition gets - * sampled next, it never drops data. An unset/≤1 batchSize applies the reward immediately, via - * the same flushPendingRewards() a real batch flush uses, so there's one code path for "fold a - * reward's counts into successCount/failureCount/totalCount." + * batchSize reward observations have accumulated across the whole experiment. The raw event is + * always persisted to ThompsonSamplingReward regardless of batching (in processReward(), before + * this is called) — batching only delays when a reward affects which condition gets sampled + * next, it never drops data. An unset/≤1 batchSize applies the reward immediately. + * + * Everything below runs inside one transaction that takes a pessimistic write lock on every + * ConditionPosteriorState row for this config up front (ordered by id, to avoid deadlocking + * against a concurrent reward that locks the same rows). batchSize paces how often posteriors + * move for the experiment as a whole, and a reward for any condition is evidence toward that + * same shared cadence, so the "is the batch ready" check has to see a consistent snapshot across + * every condition, not just the one that just received a reward — without the lock, two rewards + * arriving close together could both read the same pending totals and double-apply them, or one + * could have its just-buffered increment silently overwritten by the other's flush-reset. Once + * the shared total reaches batchSize, every condition's pending buffer is flushed, not just the + * one that tipped it over, so a low-volume condition still gets its pending counts folded in as + * soon as the batch closes. */ private async applyOrBufferReward( state: Pick, @@ -142,61 +152,52 @@ export class ThompsonSamplingRewardService { ): Promise { const effectiveBatchSize = batchSize && batchSize > 1 ? batchSize : 1; - if (effectiveBatchSize <= 1) { - await this.flushPendingRewards(state.id, success ? 1 : 0, success ? 0 : 1, 1); - return; - } + await this.posteriorStateRepository.manager.transaction(async (manager) => { + const experimentStates = await manager + .createQueryBuilder(ConditionPosteriorState, 'state') + .where('state.configId = :configId', { configId: state.configId }) + .orderBy('state.id', 'ASC') + .setLock('pessimistic_write') + .getMany(); + + const current = experimentStates.find((s) => s.id === state.id); + if (!current) { + return; + } - await this.posteriorStateRepository.increment({ id: state.id }, 'pendingTotalCount', 1); - if (success) { - await this.posteriorStateRepository.increment({ id: state.id }, 'pendingSuccessCount', 1); - } else { - await this.posteriorStateRepository.increment({ id: state.id }, 'pendingFailureCount', 1); - } + current.pendingTotalCount += 1; + if (success) { + current.pendingSuccessCount += 1; + } else { + current.pendingFailureCount += 1; + } - await this.flushIfBatchReady(state.configId, effectiveBatchSize); - } + if (effectiveBatchSize <= 1) { + await this.flushPendingRewards(manager, current); + return; + } - /** - * batchSize paces how often posteriors move for the experiment as a whole, and a reward for any - * condition is evidence toward that same shared cadence — so the pending count is summed across - * every condition in the config, not just the one that just received a reward. Once the shared - * total reaches batchSize, every condition's pending buffer is flushed, not just the one that - * tipped it over, so a low-volume condition still gets its pending counts folded in as soon as - * the batch closes. - */ - private async flushIfBatchReady(configId: string, effectiveBatchSize: number): Promise { - const experimentStates = await this.posteriorStateRepository.findByConfigId(configId); - const totalPending = experimentStates.reduce((sum, s) => sum + s.pendingTotalCount, 0); + await manager.save(current); - if (totalPending < effectiveBatchSize) { - return; - } + const totalPending = experimentStates.reduce((sum, s) => sum + s.pendingTotalCount, 0); + if (totalPending < effectiveBatchSize) { + return; + } - await Promise.all( - experimentStates - .filter((s) => s.pendingTotalCount > 0) - .map((s) => this.flushPendingRewards(s.id, s.pendingSuccessCount, s.pendingFailureCount, s.pendingTotalCount)) - ); + await Promise.all( + experimentStates.filter((s) => s.pendingTotalCount > 0).map((s) => this.flushPendingRewards(manager, s)) + ); + }); } - private async flushPendingRewards( - stateId: string, - pendingSuccessCount: number, - pendingFailureCount: number, - pendingTotalCount: number - ): Promise { - await this.posteriorStateRepository.increment({ id: stateId }, 'totalCount', pendingTotalCount); - if (pendingSuccessCount > 0) { - await this.posteriorStateRepository.increment({ id: stateId }, 'successCount', pendingSuccessCount); - } - if (pendingFailureCount > 0) { - await this.posteriorStateRepository.increment({ id: stateId }, 'failureCount', pendingFailureCount); - } - await this.posteriorStateRepository.update( - { id: stateId }, - { pendingSuccessCount: 0, pendingFailureCount: 0, pendingTotalCount: 0 } - ); + private async flushPendingRewards(manager: EntityManager, state: ConditionPosteriorState): Promise { + state.totalCount += state.pendingTotalCount; + state.successCount += state.pendingSuccessCount; + state.failureCount += state.pendingFailureCount; + state.pendingTotalCount = 0; + state.pendingSuccessCount = 0; + state.pendingFailureCount = 0; + await manager.save(state); } /** diff --git a/packages/backend/test/unit/DTO/ExperimentDTO.test.ts b/packages/backend/test/unit/DTO/ExperimentDTO.test.ts new file mode 100644 index 0000000000..4262c712c0 --- /dev/null +++ b/packages/backend/test/unit/DTO/ExperimentDTO.test.ts @@ -0,0 +1,46 @@ +import 'reflect-metadata'; +import { validate } from 'class-validator'; +import { plainToInstance } from 'class-transformer'; +import { ExperimentDTO } from '../../../src/api/DTO/ExperimentDTO'; +import { ASSIGNMENT_UNIT, ASSIGNMENT_ALGORITHM } from 'upgrade_types'; + +describe('ExperimentDTO', () => { + describe('assignmentAlgorithm / assignmentUnit compatibility', () => { + it('rejects Thompson Sampling combined with Within-Subjects assignment', async () => { + // Within-Subjects assignment never stores a condition on the individual enrollment (it's + // tracked per-repeat via RepeatedEnrollment instead), which is what Thompson Sampling's + // reward path reads to attribute a reward to a condition -- so this combination can never + // record a reward and must be rejected up front. + const dto = plainToInstance(ExperimentDTO, { + assignmentUnit: ASSIGNMENT_UNIT.WITHIN_SUBJECTS, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + }); + + const errors = await validate(dto, { skipMissingProperties: true }); + + expect(errors.some((e) => e.property === 'assignmentAlgorithm')).toBe(true); + }); + + it('allows Thompson Sampling with Individual assignment', async () => { + const dto = plainToInstance(ExperimentDTO, { + assignmentUnit: ASSIGNMENT_UNIT.INDIVIDUAL, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + }); + + const errors = await validate(dto, { skipMissingProperties: true }); + + expect(errors.some((e) => e.property === 'assignmentAlgorithm')).toBe(false); + }); + + it('allows Within-Subjects assignment with a non-Thompson-Sampling algorithm', async () => { + const dto = plainToInstance(ExperimentDTO, { + assignmentUnit: ASSIGNMENT_UNIT.WITHIN_SUBJECTS, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + }); + + const errors = await validate(dto, { skipMissingProperties: true }); + + expect(errors.some((e) => e.property === 'assignmentAlgorithm')).toBe(false); + }); + }); +}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts index b5de476f6a..1a6094891a 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -12,6 +12,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { configRepository = { save: jest.fn().mockResolvedValue({ id: 'config-1', experimentId: 'experiment-1' }), update: jest.fn().mockResolvedValue(undefined), + remove: jest.fn().mockResolvedValue(undefined), findByExperimentId: jest.fn().mockResolvedValue(undefined), findByExperimentIdWithConditions: jest.fn().mockResolvedValue(undefined), }; @@ -63,6 +64,20 @@ describe('ThompsonSamplingExperimentCrudService', () => { }); }); + describe('updateConfig', () => { + it('only writes fields the caller actually provided, leaving omitted fields untouched', async () => { + await service.updateConfig('experiment-1', { batchSize: 5 }); + + expect(configRepository.update).toHaveBeenCalledWith({ experimentId: 'experiment-1' }, { batchSize: 5 }); + }); + + it('does not issue an update call at all when no threshold/batchSize fields are provided', async () => { + await service.updateConfig('experiment-1', { priors: { 'condition-1': { success: 1, failure: 1 } } }); + + expect(configRepository.update).not.toHaveBeenCalled(); + }); + }); + describe('createConfigIfApplicable', () => { it('does nothing for a non-Thompson-Sampling experiment', async () => { await service.createConfigIfApplicable( @@ -112,7 +127,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { }); describe('syncConfigIfApplicable', () => { - it('does nothing for a non-Thompson-Sampling experiment', async () => { + it('does nothing for a non-Thompson-Sampling experiment with no existing config', async () => { await service.syncConfigIfApplicable( { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any, { @@ -121,10 +136,52 @@ describe('ThompsonSamplingExperimentCrudService', () => { } as any ); - expect(configRepository.findByExperimentId).not.toHaveBeenCalled(); + expect(configRepository.remove).not.toHaveBeenCalled(); expect(configRepository.update).not.toHaveBeenCalled(); }); + it('deletes a stale config left over from before the experiment switched away from Thompson Sampling', async () => { + const staleConfig = { id: 'config-1', conditionPosteriorStates: [{ conditionId: 'condition-1' }] }; + configRepository.findByExperimentId.mockResolvedValue(staleConfig); + + await service.syncConfigIfApplicable( + { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any, + { + id: 'experiment-1', + conditions: [], + } as any + ); + + expect(configRepository.remove).toHaveBeenCalledWith(staleConfig); + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + }); + + it('creates a config when an experiment is switched to Thompson Sampling on an update rather than at create time', async () => { + configRepository.findByExperimentId.mockResolvedValue(undefined); + + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { batchSize: 5, priors: { 'condition-1': { success: 2, failure: 1 } } }, + } as any; + + await service.syncConfigIfApplicable(experiment, { + id: 'experiment-1', + conditions: [{ id: 'condition-1' }], + } as any); + + expect(configRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ experimentId: 'experiment-1', batchSize: 5 }) + ); + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + configId: 'config-1', + conditionId: 'condition-1', + priorSuccess: 2, + priorFailure: 1, + successCount: 0, + totalCount: 0, + }); + }); + it('syncs conditions and applies prior updates for a Thompson Sampling experiment', async () => { configRepository.findByExperimentId.mockResolvedValue({ id: 'config-1', @@ -226,5 +283,38 @@ describe('ThompsonSamplingExperimentCrudService', () => { priorFailure: 3, }); }); + + it('keys weight estimation by conditionId, not conditionCode, so two conditions sharing a code do not collide', async () => { + // conditionCode has no uniqueness constraint -- give both conditions the same one, but with + // vastly different posteriors, so a code-keyed weight map (the bug) would collapse them into + // a single shared value instead of each reflecting its own evidence. + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + conditionPosteriorStates: [ + { + conditionId: 'condition-1', + priorSuccess: 1000, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + condition: { conditionCode: 'DUPLICATE', order: 0 }, + }, + { + conditionId: 'condition-2', + priorSuccess: 1, + priorFailure: 1000, + successCount: 0, + failureCount: 0, + totalCount: 0, + condition: { conditionCode: 'DUPLICATE', order: 1 }, + }, + ], + }); + + const [strong, weak] = await service.getRewardsSummary('experiment-1'); + + expect(strong.estimatedWeight).toBeGreaterThan(90); + expect(weak.estimatedWeight).toBeLessThan(10); + }); }); }); diff --git a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts index da1ca793d7..08ff6aef6e 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts @@ -116,6 +116,36 @@ describe('ThompsonSamplingRewardService', () => { return allStates().find((row) => row.id === id); } + // Fakes just enough of TypeORM's EntityManager for applyOrBufferReward()'s transaction: a + // transaction() that runs the callback inline (no real DB transaction/lock semantics -- those + // aren't meaningfully unit-testable without a real Postgres instance), a createQueryBuilder() + // that filters the in-memory rows by configId (the only clause the service issues), and a + // save() that persists in-memory since getMany() already hands back references into + // statesByCondition, not copies. + function makeFakeManager() { + const manager: any = { + transaction: (work: (m: any) => Promise) => work(manager), + createQueryBuilder: () => { + let configIdFilter: string | undefined; + const builder: any = { + where: (_cond: string, params: { configId: string }) => { + configIdFilter = params.configId; + return builder; + }, + orderBy: () => builder, + setLock: () => builder, + getMany: () => Promise.resolve(allStates().filter((row) => row.configId === configIdFilter)), + }; + return builder; + }, + save: jest.fn((entity: PosteriorStateRow) => { + Object.assign(findRowById(entity.id), entity); + return Promise.resolve(entity); + }), + }; + return manager; + } + beforeEach(() => { statesByCondition = { [CONDITION_ID]: makeStateRow('state-1', CONDITION_ID), @@ -125,18 +155,7 @@ describe('ThompsonSamplingRewardService', () => { posteriorStateRepository = { findByConditionId: jest.fn((conditionId: string) => Promise.resolve(statesByCondition[conditionId])), - findByConfigId: jest.fn((configId: string) => - Promise.resolve(allStates().filter((row) => row.configId === configId)) - ), - increment: jest.fn((criteria: { id: string }, column: keyof PosteriorStateRow, amount: number) => { - const row = findRowById(criteria.id); - (row[column] as number) += amount; - return Promise.resolve(undefined); - }), - update: jest.fn((criteria: { id: string }, partial: Partial) => { - Object.assign(findRowById(criteria.id), partial); - return Promise.resolve(undefined); - }), + manager: makeFakeManager(), }; tsConfigRepository = { @@ -366,7 +385,9 @@ describe('ThompsonSamplingRewardService', () => { const stateB = statesByCondition[CONDITION_B_ID]; expect(stateB.totalCount).toBe(0); expect(stateB.pendingTotalCount).toBe(0); - expect(posteriorStateRepository.update).not.toHaveBeenCalledWith({ id: stateB.id }, expect.anything()); + expect(posteriorStateRepository.manager.save).not.toHaveBeenCalledWith( + expect.objectContaining({ id: stateB.id }) + ); }); }); diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts index 472dcdcb4a..f9069c65e5 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts @@ -209,6 +209,7 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { this.experimentService.fetchContextMetaData(); this.stratificationFactorsService.fetchStratificationFactors(true); this.createExperimentForm(); + this.updateAssignmentAlgorithms(); // Set up subscriptions this.listenForContextMetaData(); @@ -480,6 +481,18 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { groupTypeControl?.updateValueAndValidity(); consistencyRuleControl?.updateValueAndValidity(); conditionOrderControl?.updateValueAndValidity(); + + // Thompson Sampling can't attribute rewards under Within-Subjects assignment (see + // updateAssignmentAlgorithms()) -- disable it going forward, and bump an already-selected + // Thompson Sampling algorithm back to Random rather than leave an invalid combination + // sitting in the form. + this.updateAssignmentAlgorithms(); + if ( + assignmentUnit === ASSIGNMENT_UNIT.WITHIN_SUBJECTS && + this.assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING + ) { + this.experimentForm.get('assignmentAlgorithm')?.setValue(ASSIGNMENT_ALGORITHM.RANDOM); + } }) ); } @@ -495,6 +508,12 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { } } else { this.thompsonSamplingConfigFormValue = undefined; + // The TS sub-form is about to be removed from the DOM (its @if goes false) without emitting + // a final validity event. Without resetting these, a form that was invalid at the moment of + // switching away would leave isTSFormValid$ stuck at false, permanently disabling Save for + // an otherwise-valid non-TS experiment until the modal is reopened. + this.isTSFormValid$.next(true); + this.isTSFormChanged$.next(false); } } @@ -507,6 +526,17 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { if (stratifiedAlgorithm) { (stratifiedAlgorithm as any).disabled = this.allStratificationFactors.length === 0; } + + // Thompson Sampling can't be used with Within-Subjects assignment: that assignment unit never + // stores a condition on the individual enrollment (it's tracked per-repeat instead), which is + // what Thompson Sampling's reward path reads to attribute a reward to a condition. + const thompsonSamplingAlgorithm = this.assignmentAlgorithms.find( + (alg) => alg.value === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING + ); + + if (thompsonSamplingAlgorithm) { + (thompsonSamplingAlgorithm as any).disabled = this.unitOfAssignmentValue === ASSIGNMENT_UNIT.WITHIN_SUBJECTS; + } } validateStratificationFactorSelection(algorithm: ASSIGNMENT_ALGORITHM): void { From 2ca97d7d7d9fc41590cb2af2f2e35e69aef0f541 Mon Sep 17 00:00:00 2001 From: doswalt Date: Wed, 9 Sep 2026 15:20:49 -0400 Subject: [PATCH 11/28] handle bad prior id mapping, fix sync edge case --- .../api/controllers/ExperimentController.ts | 40 ++++- ...aptiveExperimentConfigDispatcherService.ts | 8 +- .../AdaptiveExperimentConfigService.ts | 12 +- .../src/api/services/ImportExportService.ts | 10 +- .../ThompsonSamplingExperimentCrudService.ts | 52 +++++- ...ExperimentControllerAdaptiveConfig.test.ts | 153 ++++++++++++++++++ .../unit/services/ImportExportService.test.ts | 29 +++- ...mpsonSamplingExperimentCrudService.test.ts | 63 ++++++++ 8 files changed, 355 insertions(+), 12 deletions(-) create mode 100644 packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index fee7748a75..8449b593b5 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -1047,10 +1047,18 @@ export class ExperimentController { throw new BadRequestError(contextValidationError); } + // Captured before create() runs: ExperimentService.create() replaces every condition's id with + // a freshly generated one in place, so `experiment.conditions[].id` no longer matches whatever + // ids the client used to key thompsonSamplingConfig.priors by the time create() returns. + const originalConditionIds = experiment.conditions?.map((condition) => condition.id); const createdExperiment = await this.experimentService.create(experiment, currentUser, request.logger); try { - await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, createdExperiment); + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable( + experiment, + createdExperiment, + originalConditionIds + ); } catch (error) { // The experiment row already committed above. Without this, a failed adaptive-config write // would leave a Thompson Sampling experiment with no config/posterior rows behind -- invisible @@ -1247,9 +1255,37 @@ export class ExperimentController { throw new BadRequestError(contextValidationError); } + const previousExperiment = await this.experimentService.getSingleExperiment(id, request.logger); + const updatedExperiment = await this.experimentService.update({ ...experiment, id }, currentUser, request.logger); - await this.adaptiveExperimentConfigDispatcher.syncConfigIfApplicable(experiment, updatedExperiment); + try { + await this.adaptiveExperimentConfigDispatcher.syncConfigIfApplicable(experiment, updatedExperiment); + } catch (error) { + // The base experiment update above already committed (e.g. assignmentAlgorithm switched to + // THOMPSON_SAMPLING). Without reverting, a failed config sync would leave that change in place + // with no config/posterior rows -- invisible and permanently unable to assign a condition, the + // same failure mode create() already guards against by deleting the just-created experiment. + // Restore the pre-update experiment, then re-run the config sync against the reverted state so + // any config/posterior rows the failed attempt did manage to write get cleaned up too. Best + // effort: a failure here is logged rather than allowed to replace/mask the original error. + if (previousExperiment) { + try { + const revertedExperiment = await this.experimentService.update( + previousExperiment, + currentUser, + request.logger + ); + await this.adaptiveExperimentConfigDispatcher.syncConfigIfApplicable(previousExperiment, revertedExperiment); + } catch (revertError) { + request.logger.error({ + message: `Failed to fully revert experiment ${id} after adaptive config sync failure`, + error: revertError, + }); + } + } + throw error; + } return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(updatedExperiment); } diff --git a/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts b/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts index 9058fbd854..5d2f551332 100644 --- a/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts +++ b/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts @@ -17,9 +17,13 @@ export class AdaptiveExperimentConfigDispatcherService implements AdaptiveExperi this.services = [thompsonSamplingCrudService]; } - public async createConfigIfApplicable(experiment: ExperimentDTO, createdExperiment: ExperimentDTO): Promise { + public async createConfigIfApplicable( + experiment: ExperimentDTO, + createdExperiment: ExperimentDTO, + originalConditionIds?: string[] + ): Promise { for (const service of this.services) { - await service.createConfigIfApplicable(experiment, createdExperiment); + await service.createConfigIfApplicable(experiment, createdExperiment, originalConditionIds); } } diff --git a/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts b/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts index ca44a11bbd..7c2792d804 100644 --- a/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts +++ b/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts @@ -7,7 +7,17 @@ import { ExperimentDTO } from '../DTO/ExperimentDTO'; * implementation without branching on algorithm. */ export interface AdaptiveExperimentConfigService { - createConfigIfApplicable(experiment: ExperimentDTO, createdExperiment: ExperimentDTO): Promise; + /** + * `originalConditionIds` is the condition ID list exactly as submitted by the caller, captured + * before ExperimentService.create()/deduceConditions() replace every condition ID with a freshly + * generated one. Implementations that key caller-supplied data (e.g. priors) by condition ID need + * this to remap those keys onto `createdExperiment`'s actual (post-creation) condition IDs. + */ + createConfigIfApplicable( + experiment: ExperimentDTO, + createdExperiment: ExperimentDTO, + originalConditionIds?: string[] + ): Promise; syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise; attachConfigToExperiment(experiment: T): Promise; } diff --git a/packages/backend/src/api/services/ImportExportService.ts b/packages/backend/src/api/services/ImportExportService.ts index e92d220721..59b1759c4b 100644 --- a/packages/backend/src/api/services/ImportExportService.ts +++ b/packages/backend/src/api/services/ImportExportService.ts @@ -41,9 +41,17 @@ export class ImportExportService { await Promise.all( experiments.map(async (experiment) => { try { + // Captured before create() runs -- see ExperimentController.create() for why: create() + // mutates condition ids in place, so this is the last point they still match whatever ids + // (from the import file, or a batch-create caller) thompsonSamplingConfig.priors is keyed by. + const originalConditionIds = experiment.conditions?.map((condition) => condition.id); const result = await this.experimentService.create(experiment, currentUser, logger); try { - await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable(experiment, result); + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable( + experiment, + result, + originalConditionIds + ); } catch (configError) { // Same reasoning as the single-experiment POST /experiments path: don't leave an // orphaned, config-less Thompson Sampling experiment behind when this step fails. diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 0c9a8467ba..6b06481e01 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -39,16 +39,58 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment * `batchSize`/`minimumDrawDifference` from `experiment.thompsonSamplingConfig` are ever read here — * there is no field for success/failure counts, so posterior state always starts at zero regardless * of what the caller's source experiment (e.g. an imported/exported one) previously accumulated. + * + * `priors` is keyed by whatever condition IDs the caller submitted (client-generated temp IDs on + * create, or the previously-exported IDs on import), but ExperimentService.create()/deduceConditions() + * replace every condition ID with a freshly generated one before/while persisting — so those keys + * never match `createdExperiment.conditions[].id` on their own. `originalConditionIds` (captured by + * the caller before create() ran) lets remapPriorsToNewConditionIds() translate them. */ - public async createConfigIfApplicable(experiment: ExperimentDTO, createdExperiment: ExperimentDTO): Promise { + public async createConfigIfApplicable( + experiment: ExperimentDTO, + createdExperiment: ExperimentDTO, + originalConditionIds?: string[] + ): Promise { if (experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { return; } - await this.createConfig( - createdExperiment.id, - createdExperiment.conditions, - experiment.thompsonSamplingConfig ?? {} + const remappedConfig = this.remapPriorsToNewConditionIds( + experiment.thompsonSamplingConfig, + originalConditionIds, + createdExperiment.conditions ); + await this.createConfig(createdExperiment.id, createdExperiment.conditions, remappedConfig ?? {}); + } + + /** + * Translates a priors record keyed by pre-creation condition IDs onto the condition IDs the + * experiment actually ended up with. Both `originalConditionIds` and `newConditions` are produced + * by order-preserving map/forEach transforms all the way through ExperimentService's create/import + * pipeline (conditions are never reordered, only replaced in place), so corresponding entries at the + * same array index refer to the same condition — there is no other stable, unique-per-condition key + * available to correlate on (ExperimentCondition.twoCharacterId was removed; conditionCode is not + * guaranteed unique). Without this, every condition would silently fall back to the default + * Beta(1,1) prior whenever the caller's condition IDs get regenerated. + */ + private remapPriorsToNewConditionIds( + config: ThompsonSamplingConfigParams | undefined, + originalConditionIds: string[] | undefined, + newConditions: ConditionRef[] + ): ThompsonSamplingConfigParams | undefined { + if (!config?.priors || !originalConditionIds) { + return config; + } + + const remappedPriors: Record = {}; + originalConditionIds.forEach((oldId, index) => { + const prior = config.priors?.[oldId]; + const newId = newConditions[index]?.id; + if (prior && newId) { + remappedPriors[newId] = prior; + } + }); + + return { ...config, priors: remappedPriors }; } /** diff --git a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts new file mode 100644 index 0000000000..c431ec3cdb --- /dev/null +++ b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts @@ -0,0 +1,153 @@ +import { ExperimentController } from '../../../src/api/controllers/ExperimentController'; +import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; + +describe('ExperimentController adaptive config wiring', () => { + let experimentService: any; + let adaptiveExperimentConfigDispatcher: any; + let controller: ExperimentController; + let request: any; + + beforeEach(() => { + experimentService = { + validateExperimentContext: jest.fn().mockReturnValue(undefined), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn().mockResolvedValue(undefined), + getSingleExperiment: jest.fn(), + }; + adaptiveExperimentConfigDispatcher = { + createConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + syncConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + attachConfigToExperiment: jest.fn().mockImplementation((experiment) => Promise.resolve(experiment)), + }; + request = { logger: { child: jest.fn(), error: jest.fn(), info: jest.fn() } }; + + controller = new ExperimentController( + experimentService, + {} as any, + {} as any, + {} as any, + {} as any, + adaptiveExperimentConfigDispatcher + ); + }); + + describe('create()', () => { + it('captures condition ids before create() runs and forwards them to createConfigIfApplicable', async () => { + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'client-temp-id-1' }, { id: 'client-temp-id-2' }], + thompsonSamplingConfig: { priors: { 'client-temp-id-1': { success: 2, failure: 3 } } }, + } as any; + // Mirrors what ExperimentService.create() actually does: mutate condition ids in place. + experimentService.create.mockImplementation((exp: any) => { + exp.conditions.forEach((condition: any, index: number) => { + condition.id = `server-id-${index + 1}`; + }); + return Promise.resolve({ ...exp, id: 'experiment-1' }); + }); + + await controller.create(experiment, {} as any, request); + + expect(adaptiveExperimentConfigDispatcher.createConfigIfApplicable).toHaveBeenCalledWith( + expect.objectContaining({ conditions: [{ id: 'server-id-1' }, { id: 'server-id-2' }] }), + expect.objectContaining({ id: 'experiment-1' }), + ['client-temp-id-1', 'client-temp-id-2'] + ); + }); + + it('deletes the just-created experiment and rethrows when config creation fails', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + experimentService.create.mockResolvedValue({ id: 'experiment-1', conditions: [] }); + const configError = new Error('config create failed'); + adaptiveExperimentConfigDispatcher.createConfigIfApplicable.mockRejectedValue(configError); + + await expect(controller.create(experiment, {} as any, request)).rejects.toThrow(configError); + + expect(experimentService.delete).toHaveBeenCalledWith('experiment-1', {}, { logger: request.logger }); + }); + }); + + describe('update()', () => { + it('does not touch previous state on the happy path', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: [] }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenCalledTimes(1); + }); + + it('reverts the experiment and re-syncs the config when syncConfigIfApplicable fails', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + const updatedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const revertedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + const syncError = new Error('config sync failed'); + + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValueOnce(updatedExperiment).mockResolvedValueOnce(revertedExperiment); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable + .mockRejectedValueOnce(syncError) + .mockResolvedValueOnce(undefined); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + // First update is the caller's requested change; second is the revert back to the pre-update state. + expect(experimentService.update).toHaveBeenNthCalledWith(1, { ...experiment, id: 'experiment-1' }, {}, request.logger); + expect(experimentService.update).toHaveBeenNthCalledWith(2, previousExperiment, {}, request.logger); + // First sync is the caller's requested change (which failed); second cleans up against the reverted state. + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenNthCalledWith( + 1, + experiment, + updatedExperiment + ); + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenNthCalledWith( + 2, + previousExperiment, + revertedExperiment + ); + }); + + it('still throws the original error, logged rather than masked, when the revert attempt itself fails', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + const syncError = new Error('config sync failed'); + const revertError = new Error('revert update failed'); + + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update + .mockResolvedValueOnce({ id: 'experiment-1' }) + .mockRejectedValueOnce(revertError); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable.mockRejectedValue(syncError); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + expect(request.logger.error).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('Failed to fully revert'), error: revertError }) + ); + }); + + it('skips the revert (but still throws) when no previous experiment can be found', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const syncError = new Error('config sync failed'); + + experimentService.getSingleExperiment.mockResolvedValue(undefined); + experimentService.update.mockResolvedValue({ id: 'experiment-1' }); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable.mockRejectedValue(syncError); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/backend/test/unit/services/ImportExportService.test.ts b/packages/backend/test/unit/services/ImportExportService.test.ts index 0540f6774b..ce9339360a 100644 --- a/packages/backend/test/unit/services/ImportExportService.test.ts +++ b/packages/backend/test/unit/services/ImportExportService.test.ts @@ -35,7 +35,34 @@ describe('ImportExportService', () => { // POST /experiments controller path wired this up), so assignment could never select a condition. expect(adaptiveExperimentConfigDispatcher.createConfigIfApplicable).toHaveBeenCalledWith( experiment, - expect.objectContaining({ id: 'experiment-1' }) + expect.objectContaining({ id: 'experiment-1' }), + ['condition-1'] + ); + }); + + it('captures condition ids before create() runs, so a caller that regenerates them can still be remapped', async () => { + // ExperimentService.create() regenerates condition ids in place; a caller whose mock (or the + // real service) does this must not lose the pre-creation ids the imported/batch-created + // experiment's priors are keyed by -- addBulkExperiments has to snapshot them beforehand. + experimentService.create.mockImplementation((experiment: any) => + Promise.resolve({ + ...experiment, + conditions: experiment.conditions.map((condition: any) => ({ ...condition, id: `new-${condition.id}` })), + }) + ); + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 2, failure: 3 } } }, + } as any; + + await service.addBulkExperiments([experiment], {} as any, logger); + + expect(adaptiveExperimentConfigDispatcher.createConfigIfApplicable).toHaveBeenCalledWith( + experiment, + expect.objectContaining({ conditions: [{ id: 'new-condition-1' }] }), + ['condition-1'] ); }); diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts index 1a6094891a..7158015f55 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -124,6 +124,69 @@ describe('ThompsonSamplingExperimentCrudService', () => { totalCount: 0, }); }); + + it('remaps priors keyed by pre-creation condition ids onto the actual created condition ids', async () => { + // ExperimentService.create()/deduceConditions() regenerate every condition id in place, so the + // client-submitted (or previously-exported) ids the priors record is keyed by never match + // createdExperiment.conditions[].id on their own -- without originalConditionIds to remap + // through, this would silently fall back to the default Beta(1,1) prior for every condition. + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { + priors: { + 'client-temp-id-1': { success: 7, failure: 4 }, + 'client-temp-id-2': { success: 3, failure: 9 }, + }, + }, + } as any; + + await service.createConfigIfApplicable( + experiment, + { + id: 'experiment-1', + conditions: [{ id: 'server-id-1' }, { id: 'server-id-2' }], + } as any, + ['client-temp-id-1', 'client-temp-id-2'] + ); + + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + configId: 'config-1', + conditionId: 'server-id-1', + priorSuccess: 7, + priorFailure: 4, + successCount: 0, + totalCount: 0, + }); + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + configId: 'config-1', + conditionId: 'server-id-2', + priorSuccess: 3, + priorFailure: 9, + successCount: 0, + totalCount: 0, + }); + }); + + it('falls back to default priors when originalConditionIds is not provided', async () => { + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { priors: { 'client-temp-id-1': { success: 7, failure: 4 } } }, + } as any; + + await service.createConfigIfApplicable(experiment, { + id: 'experiment-1', + conditions: [{ id: 'server-id-1' }], + } as any); + + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + configId: 'config-1', + conditionId: 'server-id-1', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + totalCount: 0, + }); + }); }); describe('syncConfigIfApplicable', () => { From 2453c2b7a75a357fc75b29c0ec6f92fe133901cf Mon Sep 17 00:00:00 2001 From: doswalt Date: Wed, 9 Sep 2026 16:30:00 -0400 Subject: [PATCH 12/28] fix lint --- .../ExperimentControllerAdaptiveConfig.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts index c431ec3cdb..54f43c646e 100644 --- a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts +++ b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts @@ -99,7 +99,12 @@ describe('ExperimentController adaptive config wiring', () => { ); // First update is the caller's requested change; second is the revert back to the pre-update state. - expect(experimentService.update).toHaveBeenNthCalledWith(1, { ...experiment, id: 'experiment-1' }, {}, request.logger); + expect(experimentService.update).toHaveBeenNthCalledWith( + 1, + { ...experiment, id: 'experiment-1' }, + {}, + request.logger + ); expect(experimentService.update).toHaveBeenNthCalledWith(2, previousExperiment, {}, request.logger); // First sync is the caller's requested change (which failed); second cleans up against the reverted state. expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenNthCalledWith( @@ -121,9 +126,7 @@ describe('ExperimentController adaptive config wiring', () => { const revertError = new Error('revert update failed'); experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); - experimentService.update - .mockResolvedValueOnce({ id: 'experiment-1' }) - .mockRejectedValueOnce(revertError); + experimentService.update.mockResolvedValueOnce({ id: 'experiment-1' }).mockRejectedValueOnce(revertError); adaptiveExperimentConfigDispatcher.syncConfigIfApplicable.mockRejectedValue(syncError); await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( From 4378f554d77c0d203edbf6df4dc5aff39e363c47 Mon Sep 17 00:00:00 2001 From: doswalt Date: Thu, 10 Sep 2026 13:33:04 -0400 Subject: [PATCH 13/28] fix: key condition priors by conditionId, not conditionCode conditionCode is nullable and not unique; the priors record and edit-condition-prior-modal form now key by conditionId, matching the backend's ConditionRewardSummary/ThompsonSamplingService convention. conditionCode is retained only for the modal's display label. Co-Authored-By: Claude Sonnet 5 --- .../edit-condition-prior-modal.component.ts | 7 ++++--- .../experiment-conditions-table.component.ts | 4 ++-- .../src/app/shared/services/common-dialog.service.ts | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts index f9fe434a9a..d8627614bd 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts @@ -16,6 +16,7 @@ import { Prior } from 'upgrade_types'; import { SharedModule } from '../../../../../shared/shared.module'; export interface ConditionPriorUpdate { + conditionId: string; conditionCode: string; successes: number; failures: number; @@ -65,7 +66,7 @@ export class EditConditionPriorModalComponent implements OnInit { const conditionsFormArray = this.formBuilder.array( this.conditions.map((condition) => this.formBuilder.group({ - conditionCode: [condition.conditionCode], + conditionId: [condition.conditionId], successes: [condition.successes, validators.successes], failures: [condition.failures, validators.failures], }) @@ -96,8 +97,8 @@ export class EditConditionPriorModalComponent implements OnInit { if (this.priorForm.valid) { const result: Record = {}; this.conditionsFormArray.controls.forEach((control) => { - const conditionCode = control.get('conditionCode')?.value; - result[conditionCode] = { + const conditionId = control.get('conditionId')?.value; + result[conditionId] = { success: control.get('successes')?.value, failure: control.get('failures')?.value, }; diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts index e5f23e575b..6106979e2f 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.ts @@ -63,11 +63,11 @@ export class ExperimentConditionsTableComponent { }; getPriorSuccesses(condition: ExperimentCondition): number { - return this.prior?.[condition.conditionCode]?.success ?? 1; + return this.prior?.[condition.id]?.success ?? 1; } getPriorFailures(condition: ExperimentCondition): number { - return this.prior?.[condition.conditionCode]?.failure ?? 1; + return this.prior?.[condition.id]?.failure ?? 1; } onEditButtonClick(condition: ExperimentCondition): void { diff --git a/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts b/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts index cd9596a972..31cf18d105 100644 --- a/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts +++ b/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts @@ -623,9 +623,10 @@ export class DialogService { existingPrior?: Record ): Observable> { const conditionPriorUpdates: ConditionPriorUpdate[] = conditions.map((condition) => ({ + conditionId: condition.id, conditionCode: condition.conditionCode, - successes: existingPrior?.[condition.conditionCode]?.success ?? 1, - failures: existingPrior?.[condition.conditionCode]?.failure ?? 1, + successes: existingPrior?.[condition.id]?.success ?? 1, + failures: existingPrior?.[condition.id]?.failure ?? 1, })); const dialogRef = this.dialog.open(EditConditionPriorModalComponent, { From cd0542e8c3466c33e26f91a0afe5a548f2907bf1 Mon Sep 17 00:00:00 2001 From: doswalt Date: Thu, 10 Sep 2026 16:22:55 -0400 Subject: [PATCH 14/28] fix: non-nullable TS config defaults, drop stale uniform_random enum value, regroup reward table header - ThompsonSamplingExperimentConfig.warmupThreshold/minimumDrawDifference/batchSize are no longer nullable; createConfig() and the nativeThompsonSampling migration now seed real defaults (0, 0, 1) instead of null, so experiments bootstrapped by that migration's INSERT (or created with a partial config payload) never end up with an unusable null threshold. - Removed 'uniform_random' from the nativeThompsonSampling migration's up()/down() enum definitions. It's a retired MoocLet-era value (dropped from the DB enum in migration 1738974972012, and from the shared TS enum earlier); the migration's enum-recreate boilerplate had been copy-pasted from an older migration that still carried it, silently reintroducing a value the app can no longer represent. - Reward feedback table: grouped the Successes/Failures columns under a shared bottom-aligned header row (Count/Prior/Posterior sub-labels) instead of one flat header row, and Minimum Draw Difference now displays with one decimal place. Co-Authored-By: Claude Sonnet 5 --- .../ThompsonSamplingExperimentConfig.ts | 12 ++++----- .../ThompsonSamplingExperimentCrudService.ts | 6 ++--- .../1788362726319-nativeThompsonSampling.ts | 10 +++---- .../thompson-sampling-helper.service.ts | 2 +- ...igurable-reward-count-table.component.html | 26 +++++++++++++++---- ...igurable-reward-count-table.component.scss | 11 ++++++++ ...nfigurable-reward-count-table.component.ts | 4 +++ ...section-card-overview-details.component.ts | 2 +- .../projects/upgrade/src/assets/i18n/en.json | 1 + 9 files changed, 53 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts index 80bda352d6..8b7fe87e43 100644 --- a/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts +++ b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts @@ -16,16 +16,16 @@ export class ThompsonSamplingExperimentConfig extends BaseModel { experimentId?: string; /** Use uniform random selection until total reward observations exceed this count. */ - @Column({ nullable: true }) - warmupThreshold?: number; + @Column({ default: 0 }) + warmupThreshold: number; /** Fall back to uniform when the top two sampled draws differ by less than this value. */ - @Column({ nullable: true, type: 'float' }) - minimumDrawDifference?: number; + @Column({ type: 'float', default: 0 }) + minimumDrawDifference: number; /** Update posteriors every N reward events rather than on every reward. */ - @Column({ nullable: true }) - batchSize?: number; + @Column({ default: 1 }) + batchSize: number; @OneToMany(() => ConditionPosteriorState, (state) => state.config, { cascade: true, diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 6b06481e01..2246c75f73 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -154,9 +154,9 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment ): Promise { const config = await this.configRepository.save({ experimentId, - warmupThreshold: params.warmupThreshold ?? null, - minimumDrawDifference: params.minimumDrawDifference ?? null, - batchSize: params.batchSize ?? null, + warmupThreshold: params.warmupThreshold ?? 0, + minimumDrawDifference: params.minimumDrawDifference ?? 0, + batchSize: params.batchSize ?? 1, }); await Promise.all( diff --git a/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts index bff994353f..f48d2818cc 100644 --- a/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts +++ b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts @@ -20,7 +20,7 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` ); await queryRunner.query( - `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'thompson_sampling')` + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'thompson_sampling')` ); await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); await queryRunner.query(` @@ -44,9 +44,9 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { `CREATE TABLE "thompson_sampling_experiment_config" ( "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "experimentId" uuid, - "warmupThreshold" integer, - "minimumDrawDifference" double precision, - "batchSize" integer, + "warmupThreshold" integer NOT NULL DEFAULT 0, + "minimumDrawDifference" double precision NOT NULL DEFAULT 0, + "batchSize" integer NOT NULL DEFAULT 1, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), "versionNumber" integer NOT NULL, @@ -157,7 +157,7 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` ); await queryRunner.query( - `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'uniform_random', 'ts_configurable')` + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'ts_configurable')` ); await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); await queryRunner.query( diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts index debd885c7e..46e9a16c18 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts @@ -34,7 +34,7 @@ export function formatThompsonSamplingConfigDetails( { labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.WARMUP_THRESHOLD, value: config?.warmupThreshold }, { labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.MINIMUM_DRAW_DIFFERENCE, - value: config?.minimumDrawDifference, + value: config?.minimumDrawDifference?.toFixed(1), }, ]; } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html index ef201791d0..b64b735305 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html @@ -8,7 +8,7 @@ - + + + + + + + + + + + + + diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss index bd3540bf4c..417345c858 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss @@ -28,7 +28,10 @@ th { padding-left: 0; color: var(--darker-grey); + vertical-align: bottom; + } + &:first-of-type th { &:first-child { border-top-left-radius: 4px; } @@ -37,6 +40,14 @@ border-top-right-radius: 4px; } } + + &:last-of-type th { + border-top: 1px solid var(--light-grey-2); + } + } + + .group-header { + text-align: center; } } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts index fe198430cd..1c327542e3 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts @@ -18,6 +18,10 @@ export class TSConfigurableRewardCountTableComponent { @Input() dataSource: ExperimentRewardsSummary = []; @Input() isLoading = false; + groupHeaderColumns = ['conditionCode', 'successesGroup', 'spacer', 'failuresGroup', 'estimatedWeight']; + + subHeaderColumns = ['successes', 'successPrior', 'successPosterior', 'failures', 'failurePrior', 'failurePosterior']; + displayedColumns = [ 'conditionCode', 'successes', diff --git a/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts b/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts index 044f81fb69..0609c270bd 100644 --- a/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts +++ b/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts @@ -11,7 +11,7 @@ export interface KeyValueFormat { export interface BullettedListKeyValueFormat { labelKey: string; - value: number; + value: number | string; } /** diff --git a/packages/frontend/projects/upgrade/src/assets/i18n/en.json b/packages/frontend/projects/upgrade/src/assets/i18n/en.json index fe430467c6..64ea7024be 100644 --- a/packages/frontend/projects/upgrade/src/assets/i18n/en.json +++ b/packages/frontend/projects/upgrade/src/assets/i18n/en.json @@ -578,6 +578,7 @@ "experiments.details.export-metrics-data.menu-item.text": "Export Metrics Data", "experiments.details.posteriors.condition.text": "Condition", "experiments.details.posteriors.successes.text": "Successes", + "experiments.details.posteriors.count.text": "Count", "experiments.details.posteriors.prior.text": "Prior", "experiments.details.posteriors.posterior.text": "Posterior", "experiments.details.posteriors.failures.text": "Failures", From 0a7d0a9bb8fd64187fd16378b072c1450403bb1f Mon Sep 17 00:00:00 2001 From: doswalt Date: Fri, 11 Sep 2026 14:01:13 -0400 Subject: [PATCH 15/28] fix: Thompson Sampling /mark trusts client-reported condition; exclude adaptive algorithms from /batch-assign /mark previously routed THOMPSON_SAMPLING through assignExperiment(), re-running the non-deterministic Beta-sampling draw for any not-yet-enrolled user. That's safe for RANDOM/STRATIFIED_RANDOM_SAMPLING (pure functions of experimentId/userId/weights) but for Thompson Sampling meant the persisted condition could differ from what /assign had already returned to the client, mis-attributing whatever reward the user later generates. Now mirrors stratified random and within-subjects: trust the client-reported condition directly rather than replaying the algorithm. /batch-assign neither persists an enrollment nor pairs with a /mark call, so an adaptive draw there has no coherent product meaning and no way to reconcile with /assign or /mark. Filtered out until a real use case exists; has no current callers. Both decisions documented in CLAUDE.md, with unit test coverage. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 4 ++ .../services/ExperimentAssignmentService.ts | 34 +++++++--- .../ExperimentAssignmentService.test.ts | 65 +++++++++++++++++++ 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 987b8c211a..185ca222d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -173,6 +173,10 @@ Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. - **Left alone, per the analysis**: the reward pipeline split (waits for a second reward-consuming algorithm to know its real shape), the conditions-table column-swapping, renaming the Thompson-Sampling-named tables/summary shape to something generic, and any dynamic component/plugin registry. Same reasoning as the original "no shared interface" note below still applies to the *assignment/reward* seam — only the config-CRUD seam was formalized. - **Deliberately not fixed — no shared interface for the assignment/reward seam**: the only seam for a *second* adaptive algorithm's assignment and reward handling is still a hardcoded `if (assignmentAlgorithm === THOMPSON_SAMPLING)` check in `ExperimentAssignmentService`, `ExperimentDTO`, and the reward endpoint (the config-CRUD seam described above is now the exception). Left unresolved on purpose — it's a larger architectural change, out of scope for this cleanup pass, and worth designing deliberately once a second algorithm is actually on the table rather than guessing its shape now. +- **`/mark` trusts the client-reported condition for Thompson Sampling, instead of replaying the algorithm**: `updateEnrollmentExclusion()` used to route `THOMPSON_SAMPLING` (like `RANDOM`) through `assignExperiment()` → `assignThompsonSampling()` for any not-yet-enrolled user — a fresh, non-deterministic Beta-sampling draw (unseeded `Math.random()` in `sampleGamma()`/`sampleBeta()`, plus posteriors that shift as other users' rewards arrive between calls). That's safe for `RANDOM`/`STRATIFIED_RANDOM_SAMPLING`, which are pure functions of `(experimentId, userId, weights)` and always reproduce the same answer — but for Thompson Sampling it meant the condition persisted at `/mark` could differ from the condition `/assign` had already returned to the client, silently mis-recording which condition a user actually saw (and therefore mis-attributing whatever reward they later generate). Fixed by adding `THOMPSON_SAMPLING` alongside the existing `STRATIFIED_RANDOM_SAMPLING` special case in both the group and individual branches: look up the client-reported `condition` string against `experiment.conditions` by `conditionCode` and trust it directly, the same way stratified random and within-subjects already do, rather than calling `assignExperiment()`. The "already enrolled" case is unaffected either way — `conditionAssigned` is only ever persisted when `!individualEnrollment`/`!groupEnrollment`, so this only changes behavior for the first mark that creates the enrollment. + +- **Adaptive algorithms excluded from `/batch-assign`**: `BatchAssignController` → `ExperimentAssignmentService.getBatchExperimentConditions()` filters out `THOMPSON_SAMPLING` experiments before assigning (a one-line `.filter()` on `assignmentAlgorithm`, with a comment at the filter site). Reasoning: unlike `/assign`/`/mark`, this endpoint neither persists an enrollment nor is paired with any follow-up call, so an adaptive draw here would be a live, non-deterministic bandit sample disconnected from reward learning and from whatever `/assign` or `/mark` would separately say for the same user at another time — there's no coherent product meaning for it. `/batch-assign` also has no current callers and a known-separate code path from `/assign` (per this repo's own note that only `/assign` produces the `experimentSegmentInclusion` SQL — `/batch-assign` isn't exercised by the load test), so rather than design the right seam now (e.g. trusting a client-reported condition the way `/mark` now does, when there's no real `/mark` pairing to trust it against), adaptive experiments are just filtered out until a real use case forces the question. Covered by a unit test (`ExperimentAssignmentService.test.ts` → "should exclude Thompson Sampling (adaptive) experiments from batch assignment"). + - **Copilot code-review fixes**: a Copilot review of this branch surfaced 24 comments; after checking each against the current code (several were already stale, having been posted against earlier commits), 8 were live, ranked, and fixed: - **Algorithm-transition asymmetry, both directions**: `ThompsonSamplingExperimentCrudService.syncConfigIfApplicable()` previously assumed a config already existed on update. Switching an experiment *to* Thompson Sampling via edit never created one (assignment silently broke forever); switching *away from* it never deleted the old one (the reward endpoint could keep treating a now-non-adaptive experiment as Thompson Sampling). Fixed both: creates the config if missing when switching in, deletes it via a new `deleteConfigIfExists()` when switching out. `ThompsonSamplingExperimentConfigRepository.findByDecisionPoint()`/`findConfigsForActivelyEnrollingExperiments()` also gained an `assignmentAlgorithm` filter as defense-in-depth against any config row that outlives the delete. - **Concurrent reward race**: `applyOrBufferReward()`/`flushPendingRewards()` used separate increment/read/reset calls with no locking, so two rewards for the same experiment arriving close together could double-apply or lose pending counts. Rewritten to run inside one transaction taking a `pessimistic_write` lock (ordered by id) across every `ConditionPosteriorState` row for the config before mutating any of them. diff --git a/packages/backend/src/api/services/ExperimentAssignmentService.ts b/packages/backend/src/api/services/ExperimentAssignmentService.ts index 8eeec441bb..b64cd459d5 100644 --- a/packages/backend/src/api/services/ExperimentAssignmentService.ts +++ b/packages/backend/src/api/services/ExperimentAssignmentService.ts @@ -572,7 +572,15 @@ export class ExperimentAssignmentService { logger.info({ message: `getAllExperimentConditions: User: ${experimentUserDocs.map((doc) => doc.id).join(', ')}`, }); - const experiments: Experiment[] = await this.getExperimentsForContextAndDecisionPoint(context, site, target); + // Adaptive algorithms (currently just Thompson Sampling) are deliberately excluded from batch-assign. + // This endpoint neither persists an enrollment nor is paired with a /mark call, so an adaptive draw here + // is a live, non-deterministic bandit sample disconnected from reward learning and from whatever /assign + // or /mark would separately say for the same user -- there's no coherent product meaning for it. This + // endpoint has no current callers, so rather than design that seam now, adaptive experiments are just + // filtered out until a real use case forces the question. + const experiments: Experiment[] = ( + await this.getExperimentsForContextAndDecisionPoint(context, site, target) + ).filter((experiment) => experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING); if (experiments.length === 0) { return {}; @@ -1847,7 +1855,10 @@ export class ExperimentAssignmentService { const promiseArray = []; let conditionAssigned; if (!noGroupSpecified && !invalidGroup) { - if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING) { + if ( + experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING || + experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING + ) { conditionAssigned = experiment.conditions.find((expCondition) => expCondition.conditionCode === condition); } else { conditionAssigned = await this.assignExperiment( @@ -1944,14 +1955,17 @@ export class ExperimentAssignmentService { }; await this.repeatedEnrollmentRepository.save(RepeatedEnrollmentDocument); } else { - const conditionAssigned = await this.assignExperiment( - user, - experiment, - individualEnrollment, - groupEnrollment, - individualExclusion, - groupExclusion - ); + const conditionAssigned = + experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING + ? experiment.conditions.find((expCondition) => expCondition.conditionCode === condition) + : await this.assignExperiment( + user, + experiment, + individualEnrollment, + groupEnrollment, + individualExclusion, + groupExclusion + ); if (!individualEnrollment && !individualExclusion && conditionAssigned) { const individualEnrollmentDocument: Omit = { diff --git a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts index f2b714fb79..39885facec 100644 --- a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts @@ -2038,6 +2038,21 @@ describe('Experiment Assignment Service Test', () => { expect(result).toEqual({}); }); + it('should exclude Thompson Sampling (adaptive) experiments from batch assignment', async () => { + const context = 'home'; + const site = 'CurriculumSequence'; + const target = 'W1'; + const userDocs = [{ id: 'user1', group: { schoolId: ['school1'] }, workingGroup: {} }]; + const exp = structuredClone(simpleIndividualAssignmentExperiment) as any; + exp.assignmentAlgorithm = ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + + testedModule.experimentRepository.getValidExperimentsForContextAndDecisionPoint = sandbox.stub().resolves([exp]); + + const result = await testedModule.getBatchExperimentConditions(userDocs, context, site, target, loggerMock); + + expect(result).toEqual({}); + }); + it('should return batch experiment conditions for multiple users with simple individual experiment', async () => { const context = 'home'; const site = 'CurriculumSequence'; @@ -2297,4 +2312,54 @@ describe('Experiment Assignment Service Test', () => { expect(totalRewardCountArg).toBe(10); }); }); + + describe('[updateEnrollmentExclusion] Thompson Sampling trusts the client-reported condition', () => { + const experiment: any = { + id: 'ts-experiment-2', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + assignmentUnit: 'individual', + consistencyRule: 'individual', + state: 'enrolling', + conditions: [ + { id: 'condition-a', conditionCode: 'ConditionA' }, + { id: 'condition-b', conditionCode: 'ConditionB' }, + ], + }; + const user: any = { id: 'user-1', workingGroup: {} }; + const decisionPoint: any = { id: 'dp-1', site: 'site1', target: 'target1' }; + + it('persists the client-reported condition on first mark without re-running assignThompsonSampling', async () => { + testedModule.individualEnrollmentRepository.save = sandbox.stub().resolves(undefined); + // If mark ever falls back to assignExperiment() -> assignThompsonSampling() for this algorithm, + // this stub throws -- proving the fix (trusting the client-reported condition, like stratified + // random and within-subjects) rather than silently passing on an incidental TypeError. + testedModule.thompsonSamplingConfigRepository = { + findByExperimentId: sandbox + .stub() + .rejects(new Error('assignThompsonSampling should not run when mark trusts the client condition')), + }; + + await (testedModule as any).updateEnrollmentExclusion( + user, + experiment, + decisionPoint, + { + individualEnrollment: undefined, + individualExclusion: undefined, + groupEnrollment: undefined, + groupExclusion: undefined, + }, + { isUserExcluded: false, isGroupExcluded: false }, + [], + MARKED_DECISION_POINT_STATUS.CONDITION_APPLIED, + 'ConditionB', + undefined, + loggerMock + ); + + sinon.assert.calledOnce(testedModule.individualEnrollmentRepository.save); + const savedDoc = testedModule.individualEnrollmentRepository.save.getCall(0).args[0]; + expect(savedDoc.condition).toEqual(experiment.conditions[1]); + }); + }); }); From ee764f1dbf47c49b753febd583230a5a0cbb3403 Mon Sep 17 00:00:00 2001 From: doswalt Date: Sat, 12 Sep 2026 00:59:18 -0400 Subject: [PATCH 16/28] feat: reward feedback card shows batch/warmup progress, header alignment fixes Reward Feedback card now surfaces two experiment-wide status lines above the conditions table, sourced from the rewards summary endpoint: - Pending rewards: N/batchSize, cycling 0..batchSize-1 as rewards buffer and reset to 0 on each batch flush. Always 0/1 (and shown muted) when batchSize is 1 or less, since nothing is ever buffered. - Algorithm in Effect: "Random Assignment (N/threshold)" during warmup, or "Thompson Sampling" once past it. Shown muted when warmupThreshold is 0 (no warmup configured), since the label is then trivially always true. ExperimentRewardsSummary (upgrade_types) changed from a bare per-condition array to { conditions, pendingRewardsCount, totalRewardCount, warmupThreshold, batchSize }; getRewardsSummary() computes the two new aggregate fields from the same evidence-counting formula already used at assignment time. Reward count table header: Condition and Est. Weight now get a real (blank- topped) cell in both header rows instead of a rowspan, so every header label sits in an identically-sized row and lines up under a shared vertical-align: middle. Added a second spacer column ahead of Est. Weight for breathing room, and gave the existing spacer a real row2 cell too so the row1/row2 divider reaches all the way across instead of stopping at the rowspan gap. Co-Authored-By: Claude Sonnet 5 --- .../ThompsonSamplingExperimentCrudService.ts | 31 ++++- ...mpsonSamplingExperimentCrudService.test.ts | 84 +++++++++++- .../store/experiments.effects.spec.ts | 42 +++--- .../store/experiments.reducer.spec.ts | 120 +++++++++++------- .../store/experiments.selector.spec.ts | 102 +++++++++------ .../store/experiments.selectors.ts | 22 +++- ...eward-feedback-section-card.component.html | 51 +++++++- ...eward-feedback-section-card.component.scss | 27 ++++ ...-reward-feedback-section-card.component.ts | 3 + ...igurable-reward-count-table.component.html | 35 ++++- ...igurable-reward-count-table.component.scss | 2 +- ...nfigurable-reward-count-table.component.ts | 31 ++++- .../projects/upgrade/src/assets/i18n/en.json | 6 + packages/types/src/Experiment/interfaces.ts | 14 +- 14 files changed, 441 insertions(+), 129 deletions(-) create mode 100644 packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.scss diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 2246c75f73..dc03ce1cda 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -222,14 +222,27 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment } /** - * Per-condition reward totals and estimated win-rate weight for the experiment overview/summary - * display. Read-only aggregation — does not touch ConditionPosteriorState rows (see - * syncConditions() for that). + * Per-condition reward totals and estimated win-rate weight, plus experiment-wide batch/warmup + * progress, for the reward feedback card display. Read-only aggregation — does not touch + * ConditionPosteriorState rows (see syncConditions() for that). */ public async getRewardsSummary(experimentId: string): Promise { const config = await this.configRepository.findByExperimentIdWithConditions(experimentId); - if (!config) return []; + if (!config) { + return { conditions: [], pendingRewardsCount: 0, totalRewardCount: 0, warmupThreshold: 0, batchSize: 1 }; + } + + // A reward is only ever buffered (pendingTotalCount > 0) when batchSize > 1 -- unset/<=1 applies + // immediately, so there's nothing to sum and no batch to cycle through. + const pendingRewardsCount = + config.batchSize > 1 ? config.conditionPosteriorStates.reduce((sum, s) => sum + s.pendingTotalCount, 0) : 0; + // Same measure warmupThreshold gates on in ThompsonSamplingService/ExperimentAssignmentService: + // flushed evidence plus whatever's still sitting in a pending batch. + const totalRewardCount = config.conditionPosteriorStates.reduce( + (sum, s) => sum + s.totalCount + s.pendingTotalCount, + 0 + ); const rows = config.conditionPosteriorStates.map((state) => { const successes = state.successCount; @@ -262,12 +275,20 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment rows.map((r) => ({ code: r.conditionId, alpha: r.alpha, beta: r.beta })) ); - return rows + const conditions = rows .map(({ conditionId, alpha: _alpha, beta: _beta, ...rest }) => ({ ...rest, estimatedWeight: weightMap[conditionId], })) .sort((a, b) => a.order - b.order); + + return { + conditions, + pendingRewardsCount, + totalRewardCount, + warmupThreshold: config.warmupThreshold, + batchSize: config.batchSize, + }; } /** diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts index 7158015f55..82702de599 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -305,12 +305,20 @@ describe('ThompsonSamplingExperimentCrudService', () => { it('returns an empty summary when no config exists', async () => { const result = await service.getRewardsSummary('experiment-1'); - expect(result).toEqual([]); + expect(result).toEqual({ + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }); expect(configRepository.findByExperimentIdWithConditions).toHaveBeenCalledWith('experiment-1'); }); it('computes alpha/beta from priors + counts and sorts by condition order', async () => { configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 100, + batchSize: 1, conditionPosteriorStates: [ { conditionId: 'condition-2', @@ -319,6 +327,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { successCount: 5, failureCount: 5, totalCount: 10, + pendingTotalCount: 0, condition: { conditionCode: 'B', order: 1 }, }, { @@ -328,6 +337,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { successCount: 8, failureCount: 2, totalCount: 10, + pendingTotalCount: 0, condition: { conditionCode: 'A', order: 0 }, }, ], @@ -335,8 +345,8 @@ describe('ThompsonSamplingExperimentCrudService', () => { const result = await service.getRewardsSummary('experiment-1'); - expect(result.map((r) => r.conditionCode)).toEqual(['A', 'B']); - const [conditionA] = result; + expect(result.conditions.map((r) => r.conditionCode)).toEqual(['A', 'B']); + const [conditionA] = result.conditions; expect(conditionA).toMatchObject({ conditionCode: 'A', successes: 8, @@ -345,6 +355,9 @@ describe('ThompsonSamplingExperimentCrudService', () => { priorSuccess: 2, priorFailure: 3, }); + expect(result.totalRewardCount).toBe(20); + expect(result.warmupThreshold).toBe(100); + expect(result.batchSize).toBe(1); }); it('keys weight estimation by conditionId, not conditionCode, so two conditions sharing a code do not collide', async () => { @@ -352,6 +365,8 @@ describe('ThompsonSamplingExperimentCrudService', () => { // vastly different posteriors, so a code-keyed weight map (the bug) would collapse them into // a single shared value instead of each reflecting its own evidence. configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 0, + batchSize: 1, conditionPosteriorStates: [ { conditionId: 'condition-1', @@ -360,6 +375,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { successCount: 0, failureCount: 0, totalCount: 0, + pendingTotalCount: 0, condition: { conditionCode: 'DUPLICATE', order: 0 }, }, { @@ -369,15 +385,75 @@ describe('ThompsonSamplingExperimentCrudService', () => { successCount: 0, failureCount: 0, totalCount: 0, + pendingTotalCount: 0, condition: { conditionCode: 'DUPLICATE', order: 1 }, }, ], }); - const [strong, weak] = await service.getRewardsSummary('experiment-1'); + const { + conditions: [strong, weak], + } = await service.getRewardsSummary('experiment-1'); expect(strong.estimatedWeight).toBeGreaterThan(90); expect(weak.estimatedWeight).toBeLessThan(10); }); + + it('sums pendingTotalCount across conditions when batchSize > 1', async () => { + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 0, + batchSize: 5, + conditionPosteriorStates: [ + { + conditionId: 'condition-1', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 2, + condition: { conditionCode: 'A', order: 0 }, + }, + { + conditionId: 'condition-2', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 1, + condition: { conditionCode: 'B', order: 1 }, + }, + ], + }); + + const result = await service.getRewardsSummary('experiment-1'); + + expect(result.pendingRewardsCount).toBe(3); + expect(result.totalRewardCount).toBe(3); + }); + + it('always reports 0 pending rewards when batchSize is 1, even if a row has a stale pendingTotalCount', async () => { + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 0, + batchSize: 1, + conditionPosteriorStates: [ + { + conditionId: 'condition-1', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 3, + condition: { conditionCode: 'A', order: 0 }, + }, + ], + }); + + const result = await service.getRewardsSummary('experiment-1'); + + expect(result.pendingRewardsCount).toBe(0); + }); }); }); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts index 9636a92320..70bb42cb09 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts @@ -1379,24 +1379,30 @@ describe('ExperimentEffects', () => { describe('fetchRewardsDataForExperiment$', () => { it('should dispatch actionFetchRewardsDataForExperimentSuccess on successful fetch', fakeAsync(() => { const experimentId = 'test-experiment-123'; - const mockRewardsSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; + const mockRewardsSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + { + conditionCode: 'Treatment', + successes: 8, + failures: 7, + total: 15, + successRate: '53.3%', + order: 1, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 30, + warmupThreshold: 0, + batchSize: 1, + }; experimentDataService.fetchRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts index 91e87b2c8e..1a4c8db59a 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts @@ -736,24 +736,30 @@ describe('ExperimentsReducer', () => { previousState.isLoadingRewardsSummary = true; previousState.rewardsSummaries = {}; - const mockRewardsSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; + const mockRewardsSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + { + conditionCode: 'Treatment', + successes: 8, + failures: 7, + total: 15, + successRate: '53.3%', + order: 1, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 30, + warmupThreshold: 0, + batchSize: 1, + }; const testAction: Action = actionFetchRewardsDataForExperimentSuccess({ experimentId: 'exp-123', @@ -768,32 +774,44 @@ describe('ExperimentsReducer', () => { }); it('action "actionFetchRewardsDataForExperimentSuccess" should update existing rewards summary', () => { - const oldSummary = [ - { - conditionCode: 'Control', - successes: 5, - failures: 5, - total: 10, - successRate: '50.0%', - order: 0, - }, - ]; + const oldSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 5, + failures: 5, + total: 10, + successRate: '50.0%', + order: 0, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 10, + warmupThreshold: 0, + batchSize: 1, + }; const previousState = { ...initialState }; previousState.rewardsSummaries = { 'exp-123': oldSummary, }; - const newSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - ]; + const newSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 15, + warmupThreshold: 0, + batchSize: 1, + }; const testAction: Action = actionFetchRewardsDataForExperimentSuccess({ experimentId: 'exp-123', @@ -807,8 +825,20 @@ describe('ExperimentsReducer', () => { }); it('action "actionFetchRewardsDataForExperimentSuccess" should preserve other experiment summaries', () => { - const summary1 = [{ conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }]; - const summary2 = [{ conditionCode: 'B', successes: 2, failures: 0, total: 2, successRate: '100%', order: 0 }]; + const summary1 = { + conditions: [{ conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }], + pendingRewardsCount: 0, + totalRewardCount: 1, + warmupThreshold: 0, + batchSize: 1, + }; + const summary2 = { + conditions: [{ conditionCode: 'B', successes: 2, failures: 0, total: 2, successRate: '100%', order: 0 }], + pendingRewardsCount: 0, + totalRewardCount: 2, + warmupThreshold: 0, + batchSize: 1, + }; const previousState = { ...initialState }; previousState.rewardsSummaries = { @@ -841,9 +871,13 @@ describe('ExperimentsReducer', () => { }); it('action "actionFetchRewardsDataForExperimentFailure" should not modify rewardsSummaries', () => { - const existingSummary = [ - { conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }, - ]; + const existingSummary = { + conditions: [{ conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }], + pendingRewardsCount: 0, + totalRewardCount: 1, + warmupThreshold: 0, + batchSize: 1, + }; const previousState = { ...initialState }; previousState.rewardsSummaries = { diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts index 06f55bd5d7..9fd8992f5c 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts @@ -821,24 +821,30 @@ describe('Experiments Selectors', () => { describe('#selectRewardsDataForSelectedExperiment', () => { it('should return rewards summary for selected experiment', () => { - const mockRewardsSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; + const mockRewardsSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + { + conditionCode: 'Treatment', + successes: 8, + failures: 7, + total: 15, + successRate: '53.3%', + order: 1, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 30, + warmupThreshold: 0, + batchSize: 1, + }; const state = { ...mockState, @@ -860,24 +866,30 @@ describe('Experiments Selectors', () => { rewardsSummaries: {}, }; - const expectedDefault = [ - { - conditionCode: 'control', - failures: 0, - order: 1, - successRate: 'n/a', - successes: 0, - total: 0, - }, - { - conditionCode: 'variant', - failures: 0, - order: 2, - successRate: 'n/a', - successes: 0, - total: 0, - }, - ]; + const expectedDefault = { + conditions: [ + { + conditionCode: 'control', + failures: 0, + order: 1, + successRate: 'n/a', + successes: 0, + total: 0, + }, + { + conditionCode: 'variant', + failures: 0, + order: 2, + successRate: 'n/a', + successes: 0, + total: 0, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }; const selectedExperiment = state.experiments.find((exp) => exp.id === '1f12cd8f-7ff9-4731-a4eb-7104918ed252'); @@ -886,11 +898,17 @@ describe('Experiments Selectors', () => { expect(result).toEqual(expectedDefault); }); - it('should return empty array when experiment is null', () => { + it('should return an empty summary when experiment is null', () => { const state = { ...mockState, rewardsSummaries: { - 'some-id': [], + 'some-id': { + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }, }, }; @@ -898,7 +916,13 @@ describe('Experiments Selectors', () => { const result = selectRewardsDataForSelectedExperiment.projector(selectedExperiment, state); - expect(result).toEqual([]); + expect(result).toEqual({ + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }); }); }); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts index d39e0724a5..72bcd621eb 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts @@ -347,23 +347,33 @@ export const selectRewardsDataForSelectedExperiment = createSelector( selectSelectedExperiment, selectExperimentState, (experiment: ExperimentVM, state: ExperimentState): ExperimentRewardsSummary => { + const emptySummary: ExperimentRewardsSummary = { + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }; + if (!experiment || !experiment.id) { - return []; + return emptySummary; } const rewardsSummary = state.rewardsSummaries[experiment.id]; if (!rewardsSummary) { - const defaultRewardsSummary: ExperimentRewardsSummary = experiment.conditions.map((condition) => { - return { + return { + ...emptySummary, + conditions: experiment.conditions.map((condition) => ({ conditionCode: condition.conditionCode, successes: 0, failures: 0, total: 0, successRate: 'n/a', order: condition.order, - }; - }); - return defaultRewardsSummary; + })), + warmupThreshold: experiment.thompsonSamplingConfig?.warmupThreshold ?? 0, + batchSize: experiment.thompsonSamplingConfig?.batchSize ?? 1, + }; } return rewardsSummary; diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html index ce1daaffc2..fc04334b8f 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html @@ -1,3 +1,4 @@ +@if (rewardsSummary$ | async; as summary) { @if (isSectionCardExpanded) { - + + + @let isBatchingActive = summary.batchSize > 1; @let isWarmupConfigured = summary.warmupThreshold > 0; @let + isWarmupActive = isWarmupConfigured && summary.totalRewardCount <= summary.warmupThreshold; +
+
+ {{ 'experiments.details.posteriors.pending-rewards.text' | translate }}: + {{ summary.pendingRewardsCount }}/{{ summary.batchSize }} +
+
+ {{ 'experiments.details.posteriors.algorithm-in-effect.text' | translate }}: + + @if (isWarmupActive) { + {{ + 'experiments.details.posteriors.algorithm-warmup.text' + | translate : { rewardCount: summary.totalRewardCount, warmupThreshold: summary.warmupThreshold } + }} + } @else { + {{ 'experiments.details.posteriors.algorithm-active.text' | translate }} + } + +
+
+ + } +} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.scss b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.scss new file mode 100644 index 0000000000..755207a37e --- /dev/null +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.scss @@ -0,0 +1,27 @@ +// Mirrors CommonSectionCardOverviewDetailsComponent's key-value look (bold label, normal-weight +// dark-grey value, stacked rows) -- not reused directly since these two rows need tooltips and +// conditional/parameterized content that component doesn't support. +.reward-status-section { + padding: 22px 32px 0; + + .item-key-value { + min-height: 32px; + display: flex; + align-items: flex-start; + column-gap: 8px; + cursor: default; + + .item-value { + color: var(--dark-grey); + } + + // Default/not-in-play configuration (batchSize <= 1, warmupThreshold 0) -- shown, not hidden, + // but muted to signal there's nothing meaningful to track here. + &.inactive { + .ft-14-600, + .item-value { + color: var(--light-grey); + } + } + } +} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts index d049fb4bdc..69476f84a2 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts @@ -1,6 +1,7 @@ import { ChangeDetectionStrategy, Component, inject, Input } from '@angular/core'; import { CommonModule } from '@angular/common'; import { TranslateModule } from '@ngx-translate/core'; +import { MatTooltipModule } from '@angular/material/tooltip'; import { CommonSectionCardComponent } from '@shared-component-lib/common-section-card/common-section-card.component'; import { CommonSectionCardTitleHeaderComponent } from '@shared-component-lib/common-section-card-title-header/common-section-card-title-header.component'; import { CommonSectionCardActionButtonsComponent } from '@shared-component-lib/common-section-card-action-buttons/common-section-card-action-buttons.component'; @@ -18,9 +19,11 @@ import { ExperimentRewardsSummary } from 'upgrade_types'; CommonSectionCardTitleHeaderComponent, CommonSectionCardActionButtonsComponent, TranslateModule, + MatTooltipModule, TSConfigurableRewardCountTableComponent, ], templateUrl: './experiment-reward-feedback-section-card.component.html', + styleUrl: './experiment-reward-feedback-section-card.component.scss', changeDetection: ChangeDetectionStrategy.OnPush, }) export class ExperimentRewardFeedbackSectionCardComponent { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html index b64b735305..6986fcd58f 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html @@ -6,9 +6,16 @@
+ {{ 'experiments.details.posteriors.condition.text' | translate }} @@ -16,10 +16,17 @@ + {{ 'experiments.details.posteriors.successes.text' | translate }} + - {{ 'experiments.details.posteriors.successes.text' | translate }} + {{ 'experiments.details.posteriors.count.text' | translate }} {{ row.successes }} @@ -48,14 +55,21 @@ - + {{ 'experiments.details.posteriors.failures.text' | translate }} + - {{ 'experiments.details.posteriors.failures.text' | translate }} + {{ 'experiments.details.posteriors.count.text' | translate }} {{ row.failures }} @@ -87,6 +101,7 @@ -
+ + + + + - + + + + + - + + + + + + + + + + + + @@ -96,12 +121,16 @@ + + + + +
+ {{ 'experiments.details.posteriors.condition.text' | translate }} @@ -53,9 +60,27 @@ ; +export interface ExperimentRewardsSummary { + conditions: ExperimentRewardsByCondition[]; + /** + * Rewards buffered since the last batch flush, summed across all conditions. Cycles from 0 up + * to (but never reaching) batchSize, resetting to 0 once the shared batch closes. Always 0 when + * batchSize is 1 or less, since a reward is applied immediately rather than buffered. + */ + pendingRewardsCount: number; + /** Reward evidence collected so far (flushed + pending), the same measure warmupThreshold gates on. */ + totalRewardCount: number; + warmupThreshold: number; + batchSize: number; +} From 1bc226293475ac06b21fb287400f579db698102a Mon Sep 17 00:00:00 2001 From: doswalt Date: Mon, 14 Sep 2026 10:26:33 -0400 Subject: [PATCH 17/28] include priors on imports, tighten edge-cases --- .../api/controllers/ExperimentController.ts | 3 + .../src/api/services/ExperimentService.ts | 10 +++ .../src/api/services/ImportExportService.ts | 1 - .../services/ThompsonSamplingRewardService.ts | 59 +++++++++-------- ...ExperimentControllerAdaptiveConfig.test.ts | 40 +++++++++++ .../unit/services/ExperimentService.test.ts | 37 +++++++++++ .../ThompsonSamplingRewardService.test.ts | 66 ++++++++++++++----- 7 files changed, 171 insertions(+), 45 deletions(-) diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index 8449b593b5..28ac1e3201 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -1256,6 +1256,9 @@ export class ExperimentController { } const previousExperiment = await this.experimentService.getSingleExperiment(id, request.logger); + if (previousExperiment) { + await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(previousExperiment); + } const updatedExperiment = await this.experimentService.update({ ...experiment, id }, currentUser, request.logger); diff --git a/packages/backend/src/api/services/ExperimentService.ts b/packages/backend/src/api/services/ExperimentService.ts index d3300fbc6e..e9eda11e34 100644 --- a/packages/backend/src/api/services/ExperimentService.ts +++ b/packages/backend/src/api/services/ExperimentService.ts @@ -1724,6 +1724,16 @@ export class ExperimentService { if (result.revertTo && this.allIdMap[result.revertTo]) { result.revertTo = this.allIdMap[result.revertTo]; } + if (result.thompsonSamplingConfig?.priors) { + const remappedPriors = {}; + Object.entries(result.thompsonSamplingConfig.priors).forEach(([oldConditionId, prior]) => { + const newConditionId = this.allIdMap[oldConditionId]; + if (newConditionId) { + remappedPriors[newConditionId] = prior; + } + }); + result.thompsonSamplingConfig.priors = remappedPriors; + } } deduceConditionPayload(result) { diff --git a/packages/backend/src/api/services/ImportExportService.ts b/packages/backend/src/api/services/ImportExportService.ts index 59b1759c4b..a09d8e02dd 100644 --- a/packages/backend/src/api/services/ImportExportService.ts +++ b/packages/backend/src/api/services/ImportExportService.ts @@ -43,7 +43,6 @@ export class ImportExportService { try { // Captured before create() runs -- see ExperimentController.create() for why: create() // mutates condition ids in place, so this is the last point they still match whatever ids - // (from the import file, or a batch-create caller) thompsonSamplingConfig.priors is keyed by. const originalConditionIds = experiment.conditions?.map((condition) => condition.id); const result = await this.experimentService.create(experiment, currentUser, logger); try { diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index a44d4e62e0..c94d9c3dcf 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -3,12 +3,12 @@ import { EntityManager } from 'typeorm'; import { InjectRepository } from '../../typeorm-typedi-extensions'; import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; import { BinaryRewardAllowedValue, CACHE_PREFIX, EXPERIMENT_STATE } from 'upgrade_types'; -import { ThompsonSamplingRewardRepository } from '../repositories/ThompsonSamplingRewardRepository'; import { ConditionPosteriorStateRepository } from '../repositories/ConditionPosteriorStateRepository'; import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; import { IndividualEnrollmentRepository } from '../repositories/IndividualEnrollmentRepository'; import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; import { ConditionPosteriorState } from '../models/ConditionPosteriorState'; +import { ThompsonSamplingReward } from '../models/ThompsonSamplingReward'; import { CacheService } from './CacheService'; import { RewardValidator } from '../controllers/validators/RewardValidator'; import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; @@ -28,8 +28,6 @@ class RewardProcessingAborted extends Error {} @Service() export class ThompsonSamplingRewardService { constructor( - @InjectRepository() - private tsRewardRepository: ThompsonSamplingRewardRepository, @InjectRepository() private posteriorStateRepository: ConditionPosteriorStateRepository, @InjectRepository() @@ -98,13 +96,6 @@ export class ThompsonSamplingRewardService { const { conditionId } = enrollments[0]; - await this.tsRewardRepository.save({ - experimentId: config.experimentId, - conditionId, - userId: user.id, - success, - }); - const state = await this.posteriorStateRepository.findByConditionId(conditionId); if (!state) { @@ -115,7 +106,7 @@ export class ThompsonSamplingRewardService { ); } - await this.applyOrBufferReward(state, success, config.batchSize); + await this.recordRewardAtomically(config.experimentId, conditionId, user.id, success, state, config.batchSize); logger.info({ message: 'Thompson Sampling reward recorded', @@ -127,32 +118,42 @@ export class ThompsonSamplingRewardService { } /** - * Fold a reward into the posterior (successCount/totalCount), or buffer it as pending until - * batchSize reward observations have accumulated across the whole experiment. The raw event is - * always persisted to ThompsonSamplingReward regardless of batching (in processReward(), before - * this is called) — batching only delays when a reward affects which condition gets sampled - * next, it never drops data. An unset/≤1 batchSize applies the reward immediately. + * Writes the ThompsonSamplingReward audit row and folds the reward into the posterior + * (successCount/totalCount) -- or buffers it as pending until batchSize reward observations have + * accumulated across the whole experiment -- inside one transaction. Committing both together + * means a failure partway through (the locked query, a save) rolls back the audit row too, so + * there's no window where a reward is durably recorded but never reflected in the posteriors. + * An unset/≤1 batchSize applies the reward immediately. * - * Everything below runs inside one transaction that takes a pessimistic write lock on every - * ConditionPosteriorState row for this config up front (ordered by id, to avoid deadlocking - * against a concurrent reward that locks the same rows). batchSize paces how often posteriors - * move for the experiment as a whole, and a reward for any condition is evidence toward that - * same shared cadence, so the "is the batch ready" check has to see a consistent snapshot across - * every condition, not just the one that just received a reward — without the lock, two rewards - * arriving close together could both read the same pending totals and double-apply them, or one - * could have its just-buffered increment silently overwritten by the other's flush-reset. Once - * the shared total reaches batchSize, every condition's pending buffer is flushed, not just the - * one that tipped it over, so a low-volume condition still gets its pending counts folded in as - * soon as the batch closes. + * The posterior update takes a pessimistic write lock on every ConditionPosteriorState row for + * this config up front (ordered by id, to avoid deadlocking against a concurrent reward that + * locks the same rows). batchSize paces how often posteriors move for the experiment as a whole, + * and a reward for any condition is evidence toward that same shared cadence, so the "is the + * batch ready" check has to see a consistent snapshot across every condition, not just the one + * that just received a reward — without the lock, two rewards arriving close together could both + * read the same pending totals and double-apply them, or one could have its just-buffered + * increment silently overwritten by the other's flush-reset. Once the shared total reaches + * batchSize, every condition's pending buffer is flushed, not just the one that tipped it over, + * so a low-volume condition still gets its pending counts folded in as soon as the batch closes. */ - private async applyOrBufferReward( - state: Pick, + private async recordRewardAtomically( + experimentId: string, + conditionId: string, + userId: string, success: boolean, + state: Pick, batchSize?: number ): Promise { const effectiveBatchSize = batchSize && batchSize > 1 ? batchSize : 1; await this.posteriorStateRepository.manager.transaction(async (manager) => { + await manager.save(ThompsonSamplingReward, { + experimentId, + conditionId, + userId, + success, + }); + const experimentStates = await manager .createQueryBuilder(ConditionPosteriorState, 'state') .where('state.configId = :configId', { configId: state.configId }) diff --git a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts index 54f43c646e..3f88182ca2 100644 --- a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts +++ b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts @@ -119,6 +119,46 @@ describe('ExperimentController adaptive config wiring', () => { ); }); + it('snapshots thompsonSamplingConfig onto the previous experiment before the forward update runs, so a revert restores it', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const updatedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const revertedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const syncError = new Error('config sync failed'); + const originalConfig = { warmupThreshold: 42, batchSize: 3, minimumDrawDifference: 0.1, priors: {} }; + const callOrder: string[] = []; + + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockImplementation(() => { + callOrder.push('update'); + return Promise.resolve( + callOrder.filter((c) => c === 'update').length === 1 ? updatedExperiment : revertedExperiment + ); + }); + // Mimics attachConfigToExperiment()'s real behavior: mutate the passed-in object in place. + // If this ran fresh inside the catch block instead of up front, it would see whatever a + // partially-failed sync had already committed to the DB -- not the true pre-update values. + adaptiveExperimentConfigDispatcher.attachConfigToExperiment.mockImplementation((exp: any) => { + callOrder.push('attachConfig'); + exp.thompsonSamplingConfig = originalConfig; + return Promise.resolve(exp); + }); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable + .mockRejectedValueOnce(syncError) + .mockResolvedValueOnce(undefined); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + expect(callOrder).toEqual(['attachConfig', 'update', 'update']); + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ thompsonSamplingConfig: originalConfig }), + revertedExperiment + ); + }); + it('still throws the original error, logged rather than masked, when the revert attempt itself fails', async () => { const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; diff --git a/packages/backend/test/unit/services/ExperimentService.test.ts b/packages/backend/test/unit/services/ExperimentService.test.ts index 8865c40bd9..7a443c967b 100644 --- a/packages/backend/test/unit/services/ExperimentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentService.test.ts @@ -491,6 +491,43 @@ describe('ExperimentService Testing', () => { }); }); + describe('deduceConditions()', () => { + it('remaps thompsonSamplingConfig.priors onto the newly generated condition ids', () => { + const result: any = { + conditions: [ + { id: 'old-condition-1', levelCombinationElements: [] }, + { id: 'old-condition-2', levelCombinationElements: [] }, + ], + thompsonSamplingConfig: { + priors: { + 'old-condition-1': { success: 3, failure: 1 }, + 'old-condition-2': { success: 1, failure: 5 }, + }, + }, + }; + + service.deduceConditions(result); + + const [newCondition1, newCondition2] = result.conditions; + expect(newCondition1.id).not.toBe('old-condition-1'); + expect(newCondition2.id).not.toBe('old-condition-2'); + expect(result.thompsonSamplingConfig.priors).toEqual({ + [newCondition1.id]: { success: 3, failure: 1 }, + [newCondition2.id]: { success: 1, failure: 5 }, + }); + expect(result.thompsonSamplingConfig.priors['old-condition-1']).toBeUndefined(); + }); + + it('leaves experiments without a thompsonSamplingConfig unaffected', () => { + const result: any = { + conditions: [{ id: 'old-condition-1', levelCombinationElements: [] }], + }; + + expect(() => service.deduceConditions(result)).not.toThrow(); + expect(result.thompsonSamplingConfig).toBeUndefined(); + }); + }); + describe('update()', () => { it('should successfully update an experiment with basic changes', async () => { const result = await service.update(mockExperimentDTO, mockUser, logger); diff --git a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts index 08ff6aef6e..f87c32ca54 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts @@ -86,7 +86,6 @@ describe('ThompsonSamplingRewardService', () => { configureLogger(); }); - let tsRewardRepository: any; let posteriorStateRepository: any; let tsConfigRepository: any; let individualEnrollmentRepository: any; @@ -100,6 +99,11 @@ describe('ThompsonSamplingRewardService', () => { // selects which row a given call mutates. let statesByCondition: Record; + // ThompsonSamplingReward audit rows saved via manager.save(ThompsonSamplingReward, plainObject) -- + // the two-arg form, distinct from the single-arg save(entityInstance) used for + // ConditionPosteriorState updates. Tests assert against this instead of a repository mock. + let savedRewards: Array<{ experimentId: string; conditionId: string; userId: string; success: boolean }>; + function makeConfig(batchSize?: number) { return { experimentId: EXPERIMENT_ID, @@ -116,12 +120,14 @@ describe('ThompsonSamplingRewardService', () => { return allStates().find((row) => row.id === id); } - // Fakes just enough of TypeORM's EntityManager for applyOrBufferReward()'s transaction: a + // Fakes just enough of TypeORM's EntityManager for recordRewardAtomically()'s transaction: a // transaction() that runs the callback inline (no real DB transaction/lock semantics -- those // aren't meaningfully unit-testable without a real Postgres instance), a createQueryBuilder() // that filters the in-memory rows by configId (the only clause the service issues), and a - // save() that persists in-memory since getMany() already hands back references into - // statesByCondition, not copies. + // save() that handles both call shapes the service uses: the single-arg entity-instance form + // for ConditionPosteriorState updates (persisted in-memory, since getMany() already hands back + // references into statesByCondition, not copies), and the two-arg (EntityClass, plainObject) + // form for the ThompsonSamplingReward audit insert (recorded into savedRewards). function makeFakeManager() { const manager: any = { transaction: (work: (m: any) => Promise) => work(manager), @@ -138,9 +144,13 @@ describe('ThompsonSamplingRewardService', () => { }; return builder; }, - save: jest.fn((entity: PosteriorStateRow) => { - Object.assign(findRowById(entity.id), entity); - return Promise.resolve(entity); + save: jest.fn((entityOrClass: any, maybeEntity?: any) => { + if (maybeEntity !== undefined) { + savedRewards.push(maybeEntity); + return Promise.resolve(maybeEntity); + } + Object.assign(findRowById(entityOrClass.id), entityOrClass); + return Promise.resolve(entityOrClass); }), }; return manager; @@ -151,7 +161,7 @@ describe('ThompsonSamplingRewardService', () => { [CONDITION_ID]: makeStateRow('state-1', CONDITION_ID), }; - tsRewardRepository = { save: jest.fn().mockResolvedValue(undefined) }; + savedRewards = []; posteriorStateRepository = { findByConditionId: jest.fn((conditionId: string) => Promise.resolve(statesByCondition[conditionId])), @@ -170,7 +180,6 @@ describe('ThompsonSamplingRewardService', () => { cacheService = makePassthroughCacheService(); service = new ThompsonSamplingRewardService( - tsRewardRepository, posteriorStateRepository, tsConfigRepository, individualEnrollmentRepository, @@ -178,13 +187,41 @@ describe('ThompsonSamplingRewardService', () => { ); }); + describe('recordRewardAtomically (audit row + posterior update as one unit)', () => { + it('propagates a posterior-update failure instead of leaving the audit row committed on its own', async () => { + const dbError = new Error('connection reset mid-transaction'); + // Fail only the ConditionPosteriorState save (single-arg form); the ThompsonSamplingReward + // audit save (two-arg form) still succeeds first, same as the real save order in + // recordRewardAtomically(). + posteriorStateRepository.manager.save = jest.fn((entityOrClass: any, maybeEntity?: any) => { + if (maybeEntity !== undefined) { + savedRewards.push(maybeEntity); + return Promise.resolve(maybeEntity); + } + return Promise.reject(dbError); + }); + + await expect( + (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger) + ).rejects.toThrow(dbError); + + // Both saves ran through the one manager passed into posteriorStateRepository.manager.transaction() + // -- in a real transaction, this failure rolls the audit insert back with it, rather than + // leaving an "acknowledged" reward whose audit row is durable but never reached the posteriors. + // makeFakeManager() has no real rollback semantics (see its own comment), so this proves the two + // writes are coupled in one atomic unit and that a failure surfaces instead of being swallowed -- + // not that the in-memory rollback itself occurs. + expect(savedRewards).toHaveLength(1); + }); + }); + describe('warmup threshold (reward count)', () => { it('always persists the raw reward event regardless of batching', async () => { tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(5)); await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); - expect(tsRewardRepository.save).toHaveBeenCalledWith({ + expect(savedRewards).toContainEqual({ experimentId: EXPERIMENT_ID, conditionId: CONDITION_ID, userId: USER_ID, @@ -327,7 +364,7 @@ describe('ThompsonSamplingRewardService', () => { const state = statesByCondition[CONDITION_ID]; expect(state.totalCount + state.pendingTotalCount).toBe(values.length); - expect(tsRewardRepository.save).toHaveBeenCalledTimes(values.length); + expect(savedRewards).toHaveLength(values.length); }); }); @@ -436,11 +473,11 @@ describe('ThompsonSamplingRewardService', () => { it('still records the reward in the background after returning the receipt', async () => { const result = service.acceptReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); expect(result.message).toBe('Reward received and is being processed.'); - expect(tsRewardRepository.save).not.toHaveBeenCalled(); + expect(savedRewards).toHaveLength(0); await flushPromises(); - expect(tsRewardRepository.save).toHaveBeenCalledWith({ + expect(savedRewards).toContainEqual({ experimentId: EXPERIMENT_ID, conditionId: CONDITION_ID, userId: USER_ID, @@ -453,7 +490,6 @@ describe('ThompsonSamplingRewardService', () => { beforeEach(() => { cacheService = makeMemoizingCacheService(); service = new ThompsonSamplingRewardService( - tsRewardRepository, posteriorStateRepository, tsConfigRepository, individualEnrollmentRepository, @@ -469,7 +505,7 @@ describe('ThompsonSamplingRewardService', () => { expect(tsConfigRepository.findOne).toHaveBeenCalledTimes(1); // Only the config lookup is cached — the reward itself is still recorded every time. - expect(tsRewardRepository.save).toHaveBeenCalledTimes(2); + expect(savedRewards).toHaveLength(2); }); it('reuses a cached decision-point lookup across rewards instead of re-querying the DB', async () => { From 722f21d0b887000749c97c77592078e2a1abb688 Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:06:12 -0400 Subject: [PATCH 18/28] Throw error if posterior state no longer exists Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../backend/src/api/services/ThompsonSamplingRewardService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index c94d9c3dcf..aea04452b1 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -163,7 +163,7 @@ export class ThompsonSamplingRewardService { const current = experimentStates.find((s) => s.id === state.id); if (!current) { - return; + throw new Error(`Posterior state ${state.id} no longer exists`); } current.pendingTotalCount += 1; From 058f648649dcd434bbda9465f0cb03a014add35b Mon Sep 17 00:00:00 2001 From: doswalt Date: Mon, 14 Sep 2026 11:12:53 -0400 Subject: [PATCH 19/28] add adaptive-quicktest and a fix for warmup of zero semantics --- clientlibs/js/package.json | 3 +- clientlibs/js/quickTestAdaptive.ts | 218 ++++++++++++++++++ .../api/services/ThompsonSamplingService.ts | 9 +- .../services/ThompsonSamplingService.test.ts | 21 ++ 4 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 clientlibs/js/quickTestAdaptive.ts diff --git a/clientlibs/js/package.json b/clientlibs/js/package.json index d2c781be38..e76e9936ee 100644 --- a/clientlibs/js/package.json +++ b/clientlibs/js/package.json @@ -50,7 +50,8 @@ "docs:markdown": "typedoc --options typedoc.json", "test": "jest", "test:coverage": "jest --coverage", - "quicktest": "([ -d dist ] || yarn build) && ts-node quickTest.ts" + "quicktest": "([ -d dist ] || yarn build) && ts-node quickTest.ts", + "quicktest:adaptive": "([ -d dist ] || yarn build) && ts-node quickTestAdaptive.ts" }, "keywords": [], "author": "", diff --git a/clientlibs/js/quickTestAdaptive.ts b/clientlibs/js/quickTestAdaptive.ts new file mode 100644 index 0000000000..a7dae9078b --- /dev/null +++ b/clientlibs/js/quickTestAdaptive.ts @@ -0,0 +1,218 @@ +// to run: npx ts-node clientlibs/js/quickTestAdaptive.ts +// +// Manual smoke test for Thompson Sampling (adaptive) experiments end-to-end: +// 1. Creates a real experiment via the admin API (POST /experiments) and starts enrollment. +// 2. Simulates a batch of synthetic users each calling /v6/init -> /v6/assign -> /v6/mark -> +// /v6/reward, the same sequence a real client would. +// 3. Prints the reward summary (GET /experiments/rewards/:id) -- the same data the frontend's +// Reward Feedback card reads -- so you can watch "Pending rewards" cycle with batchSize and +// "Algorithm in Effect" flip from Random Assignment to Thompson Sampling as warmupThreshold +// is crossed, without opening a browser. +// 4. Deletes the experiment when done (see CLEANUP_AFTER_RUN below). +// +// Local dev only. See ADMIN_TOKEN below for why. + +import axios, { AxiosError } from 'axios'; +import UpgradeClient from './dist/node'; + +const URL = { + // 3030 is the standard docker-compose port (see root CLAUDE.md); a git worktree set up via + // /new-worktree auto-assigns its own port instead (check packages/backend/.env's APP_PORT) -- + // update this if you're running in a worktree. + LOCAL: 'http://localhost:3032', + ECS_QA: 'https://apps.qa-cli.net/upgrade-service', + ECS_STAGING: 'https://apps.qa-cli.com/upgrade-service', +}; + +// ------------------------------------------------------------------------------------------- +// Admin auth +// ------------------------------------------------------------------------------------------- +// authorizationChecker.ts (packages/backend/src/auth/) bypasses real Google token validation +// for this exact string, attaching a dev admin user instead -- but only when the target +// server's GOOGLE_AUTH_TOKEN_REQUIRED env var is false (check packages/backend/.env; this is +// this worktree's current local setting, not a given for every environment). This will NOT +// work against a real deployed server. Value must match FAKE_DEV_CREDENTIAL in +// packages/types/src/User/index.ts -- hardcoded rather than imported from upgrade_types +// because this file runs directly under ts-node (see quickTest.ts), which doesn't resolve the +// upgrade_types path alias at runtime the way a webpack-built consumer does. +const ADMIN_TOKEN = 'fake-dev-user-google-credential'; + +// ------------------------------------------------------------------------------------------- +// Config -- edit these to change what gets created/simulated +// ------------------------------------------------------------------------------------------- +const hostUrl = URL.LOCAL; +const adminApiUrl = hostUrl + '/api'; +const context = 'upgrade-internal'; +const site = 'quicktest-adaptive-site'; +const target = 'quicktest-adaptive-target'; + +// Adaptive algorithm parameters -- see packages/frontend .../thompson-sampling-helper.service.ts +// and the Reward Feedback card for how these show up in the UI. +const BATCH_SIZE = 3; // rewards buffered before posteriors update; watch "Pending rewards" cycle 0..batchSize-1 +const WARMUP_THRESHOLD = 4; // reward-count gate before real TS sampling kicks in; watch "Algorithm in Effect" flip +const MINIMUM_DRAW_DIFFERENCE = 0; + +const CONDITIONS = [ + { tempId: 'quicktest-cond-control', conditionCode: 'control', priorSuccess: 1, priorFailure: 1 }, + { tempId: 'quicktest-cond-variant', conditionCode: 'variant', priorSuccess: 1, priorFailure: 1 }, +]; + +const NUM_SIMULATED_USERS = 10; +// Each simulated user's reward outcome. A fixed pattern by default so runs are reproducible -- +// swap in `Math.random() < 0.7 ? 'SUCCESS' : 'FAILURE'` if you want noisy data instead. +function rewardForUser(index: number): 'SUCCESS' | 'FAILURE' { + return index % 3 === 0 ? 'FAILURE' : 'SUCCESS'; +} + +const CLEANUP_AFTER_RUN = false; // delete the created experiment when the script finishes + +// ------------------------------------------------------------------------------------------- + +const adminClient = axios.create({ + baseURL: adminApiUrl, + headers: { Authorization: `Bearer ${ADMIN_TOKEN}` }, +}); + +quickTestAdaptive(); + +/** main test *******************************************************************************/ +async function quickTestAdaptive() { + const experiment = await createAdaptiveExperiment(); + if (!experiment) return; + + console.log(`\n[Created experiment]: ${experiment.id} (${experiment.name})`); + console.log( + '[Conditions]:', + experiment.conditions.map((c: { conditionCode: string; id: string }) => `${c.conditionCode}=${c.id}`).join(', ') + ); + + await setExperimentState(experiment.id, 'enrolling'); + console.log('[Experiment state]: enrolling'); + + for (let i = 0; i < NUM_SIMULATED_USERS; i++) { + await simulateUser(i, experiment.id); + } + + // /v6/reward is fire-and-forget (POST /v6/reward acknowledges before the DB write happens -- + // see ThompsonSamplingRewardService.acceptReward()), so give the background processing a beat + // to finish before reading the summary back, or the last few rewards may not show up yet. + await sleep(1000); + + await printRewardsSummary(experiment.id); + + if (CLEANUP_AFTER_RUN) { + await deleteExperiment(experiment.id); + console.log(`\n[Cleaned up]: deleted experiment ${experiment.id}`); + } else { + console.log(`\n[Left in place]: experiment ${experiment.id} -- delete manually when done.`); + } +} + +/** admin API calls (experiment CRUD) *******************************************************/ + +async function createAdaptiveExperiment(): Promise<{ + id: string; + name: string; + conditions: { id: string; conditionCode: string }[]; +} | null> { + const payload = { + name: `quicktest-adaptive-${Date.now()}`, + description: 'Created by clientlibs/js/quickTestAdaptive.ts -- safe to delete.', + context: [context], + state: 'inactive', + consistencyRule: 'individual', + assignmentUnit: 'individual', + postExperimentRule: 'continue', + tags: ['quicktest'], + filterMode: 'includeAll', // excludeAll would exclude every user unless individually/group included via a segment + type: 'Simple', + assignmentAlgorithm: 'thompson_sampling', + // Conditions/partitions need a client-supplied id even though the server regenerates its + // own -- ExperimentService.create() remaps thompsonSamplingConfig.priors from these ids onto + // the server-generated ones automatically (see ThompsonSamplingExperimentCrudService). + conditions: CONDITIONS.map((c, index) => ({ + id: c.tempId, + name: c.conditionCode, + description: '', + conditionCode: c.conditionCode, + assignmentWeight: 100 / CONDITIONS.length, // ignored for Thompson Sampling, but required by the DTO + order: index + 1, + })), + partitions: [{ id: 'quicktest-adaptive-dp-1', site, target, description: '', order: 1, excludeIfReached: false }], + thompsonSamplingConfig: { + warmupThreshold: WARMUP_THRESHOLD, + batchSize: BATCH_SIZE, + minimumDrawDifference: MINIMUM_DRAW_DIFFERENCE, + priors: Object.fromEntries( + CONDITIONS.map((c) => [c.tempId, { success: c.priorSuccess, failure: c.priorFailure }]) + ), + }, + }; + + try { + const response = await adminClient.post('/experiments', payload); + return response.data; + } catch (error) { + logAxiosError('Create experiment', error); + return null; + } +} + +async function setExperimentState(experimentId: string, state: string): Promise { + try { + await adminClient.post('/experiments/state', { experimentId, state }); + } catch (error) { + logAxiosError('Set experiment state', error); + } +} + +async function printRewardsSummary(experimentId: string): Promise { + try { + const response = await adminClient.get(`/experiments/rewards/${experimentId}`); + console.log('\n[Rewards summary]:', JSON.stringify(response.data, null, 2)); + } catch (error) { + logAxiosError('Rewards summary', error); + } +} + +async function deleteExperiment(experimentId: string): Promise { + try { + await adminClient.delete(`/experiments/${experimentId}`); + } catch (error) { + logAxiosError('Delete experiment', error); + } +} + +/** simulated user flow (client SDK, same as a real client would call) **********************/ + +async function simulateUser(index: number, experimentId: string): Promise { + const userId = `quicktest_adaptive_user_${Date.now()}_${index}`; + const client = new UpgradeClient(userId, hostUrl, context); + + try { + await client.init(); + + const assignment = await client.getDecisionPointAssignment(site, target); + const condition = assignment.getCondition(); + await assignment.markDecisionPoint(UpgradeClient.MARKED_DECISION_POINT_STATUS.CONDITION_APPLIED); + + const rewardValue = rewardForUser(index); + await client.sendReward({ rewardValue, experimentId }); + + console.log(`[User ${index}]: condition=${condition} reward=${rewardValue}`); + } catch (error) { + logAxiosError(`User ${index}`, error); + } +} + +/** utility functions *************************************************************************/ + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function logAxiosError(functionContext: string, error: unknown): void { + const axiosError = error as AxiosError; + const data = axiosError?.response?.data; + console.error(`\n[${functionContext} error]:`, data ?? axiosError?.message ?? error); +} diff --git a/packages/backend/src/api/services/ThompsonSamplingService.ts b/packages/backend/src/api/services/ThompsonSamplingService.ts index 38abe2276f..304a5b6330 100644 --- a/packages/backend/src/api/services/ThompsonSamplingService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingService.ts @@ -118,7 +118,14 @@ export class ThompsonSamplingService { // Warmup phase: use uniform random until sufficient reward evidence has been collected. // Gated on reward observations (not assignments) — the posteriors only move when rewards // arrive, so that's the right measure of "how much evidence do we actually have." - if (config.warmupThreshold !== undefined && totalRewardCount <= config.warmupThreshold) { + // warmupThreshold === 0 means warmup is disabled (matches the frontend's isWarmupConfigured + // check) — without the > 0 guard, totalRewardCount <= 0 would still force one uniform draw + // before any reward ever arrives, contradicting a UI that already reports TS as active at 0/0. + if ( + config.warmupThreshold !== undefined && + config.warmupThreshold > 0 && + totalRewardCount <= config.warmupThreshold + ) { return this.uniformRandom(conditionIds); } diff --git a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts index a215a1964f..78479c2c1b 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts @@ -78,6 +78,27 @@ describe('ThompsonSamplingService', () => { // After warmup, A's dominant posterior should win nearly every draw expect(aCount / runs).toBeGreaterThan(0.95); }); + + it('treats warmupThreshold of 0 as disabled, not a one-draw warmup, even with no rewards yet', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionId: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, + ]; + const config: ThompsonSamplingConfig = { warmupThreshold: 0 }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, 0, config) === 'A') { + aCount++; + } + } + // Matches the frontend's isWarmupConfigured (warmupThreshold > 0): at totalRewardCount 0 + // with warmupThreshold 0, real Thompson Sampling should already be in effect, not one + // forced uniform draw. + expect(aCount / runs).toBeGreaterThan(0.95); + }); }); describe('Thompson Sampling selection', () => { From 70bb630b42048899c91a773dff0c64569ce1a5d5 Mon Sep 17 00:00:00 2001 From: doswalt Date: Mon, 14 Sep 2026 14:35:19 -0400 Subject: [PATCH 20/28] chore: trigger CI From fe7e703a9956c7c325186b99f7b3a31520805afd Mon Sep 17 00:00:00 2001 From: doswalt Date: Wed, 16 Sep 2026 09:40:53 -0400 Subject: [PATCH 21/28] fix priors data missing after update, guard against priors changes in background in disallowed states --- .../api/controllers/ExperimentController.ts | 88 +++++++++++- ...ExperimentControllerAdaptiveConfig.test.ts | 129 +++++++++++++++++- 2 files changed, 214 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index 28ac1e3201..19efd57086 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -32,7 +32,14 @@ import { AdaptiveExperimentConfigDispatcherService } from '../services/AdaptiveE import { Response } from 'express'; import { NotFoundException } from '@nestjs/common/exceptions'; import { ExperimentIdValidator } from '../DTO/ExperimentDTO'; -import { CACHE_PREFIX, IImportError, LIST_FILTER_MODE, SERVER_ERROR, ExperimentRewardsSummary } from 'upgrade_types'; +import { + CACHE_PREFIX, + EXPERIMENT_STATE, + IImportError, + LIST_FILTER_MODE, + SERVER_ERROR, + ExperimentRewardsSummary, +} from 'upgrade_types'; import { ImportExportService } from '../services/ImportExportService'; import { getInstanceId } from '../../lib/instanceIdentity'; import { ExperimentSegmentInclusion } from '../models/ExperimentSegmentInclusion'; @@ -1197,13 +1204,14 @@ export class ExperimentController { @CurrentUser() currentUser: UserDTO, @Req() request: AppRequest ): Promise { - return this.experimentService.updateState( + const updatedExperiment = await this.experimentService.updateState( experiment.experimentId, experiment.state, currentUser, request.logger, experiment.scheduleDate ); + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(updatedExperiment); } /** @@ -1258,6 +1266,7 @@ export class ExperimentController { const previousExperiment = await this.experimentService.getSingleExperiment(id, request.logger); if (previousExperiment) { await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(previousExperiment); + this.assertConditionsNotModifiedAfterStart(previousExperiment, experiment); } const updatedExperiment = await this.experimentService.update({ ...experiment, id }, currentUser, request.logger); @@ -1293,6 +1302,81 @@ export class ExperimentController { return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(updatedExperiment); } + /** + * Mirrors the frontend's own restriction (selectSectionCardRestriction / + * selectDisabledExperimentFields in experiments.selectors.ts): conditions -- including Thompson + * Sampling priors -- are only editable while an experiment is still in one of these states. The + * frontend disables the relevant fields/section card once an experiment has left this set, but + * nothing previously stopped the same change from being sent directly to this endpoint. + */ + private static readonly CONDITIONS_EDITABLE_STATES = new Set([ + EXPERIMENT_STATE.INACTIVE, + EXPERIMENT_STATE.SCHEDULED, + EXPERIMENT_STATE.PREVIEW, + EXPERIMENT_STATE.DRAFT, + ]); + + private assertConditionsNotModifiedAfterStart( + previousExperiment: ExperimentDTO, + incomingExperiment: ExperimentDTO + ): void { + // Mirrors selectDisabledExperimentFields/selectSectionCardRestriction's own `!state || ...` + // guard: a missing state means there's nothing to compare against yet, not that editing should + // be blocked. + if (!previousExperiment.state || ExperimentController.CONDITIONS_EDITABLE_STATES.has(previousExperiment.state)) { + return; + } + + const previousConditions = previousExperiment.conditions ?? []; + const incomingConditions = incomingExperiment.conditions ?? []; + + const previousIds = new Set(previousConditions.map((condition) => condition.id)); + const incomingIds = new Set(incomingConditions.map((condition) => condition.id)); + const conditionSetChanged = + previousIds.size !== incomingIds.size || [...previousIds].some((id) => !incomingIds.has(id)); + + const conditionFieldsChanged = previousConditions.some((previousCondition) => { + const incomingCondition = incomingConditions.find((condition) => condition.id === previousCondition.id); + return ( + incomingCondition && + (previousCondition.conditionCode !== incomingCondition.conditionCode || + previousCondition.name !== incomingCondition.name || + previousCondition.description !== incomingCondition.description || + previousCondition.assignmentWeight !== incomingCondition.assignmentWeight) + ); + }); + + const priorsChanged = this.havePriorsChanged( + previousExperiment.thompsonSamplingConfig?.priors, + incomingExperiment.thompsonSamplingConfig?.priors + ); + + if (conditionSetChanged || conditionFieldsChanged || priorsChanged) { + throw new BadRequestError( + `Conditions (including Thompson Sampling priors) cannot be modified once an experiment has started. Current state: ${previousExperiment.state}.` + ); + } + } + + private havePriorsChanged( + previousPriors: Record | undefined, + incomingPriors: Record | undefined + ): boolean { + // No priors submitted at all (e.g. a non-Thompson-Sampling experiment, or a caller that only + // sends the fields it means to change) -- nothing to compare or block. + if (!incomingPriors) { + return false; + } + + const previous = previousPriors ?? {}; + const keys = new Set([...Object.keys(previous), ...Object.keys(incomingPriors)]); + return [...keys].some((conditionId) => { + const previousPrior = previous[conditionId]; + const incomingPrior = incomingPriors[conditionId]; + return previousPrior?.success !== incomingPrior?.success || previousPrior?.failure !== incomingPrior?.failure; + }); + } + /** * @swagger * /experiments/{validation}: diff --git a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts index 3f88182ca2..baa51bd30d 100644 --- a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts +++ b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts @@ -1,5 +1,5 @@ import { ExperimentController } from '../../../src/api/controllers/ExperimentController'; -import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; +import { ASSIGNMENT_ALGORITHM, EXPERIMENT_STATE } from 'upgrade_types'; describe('ExperimentController adaptive config wiring', () => { let experimentService: any; @@ -12,6 +12,7 @@ describe('ExperimentController adaptive config wiring', () => { validateExperimentContext: jest.fn().mockReturnValue(undefined), create: jest.fn(), update: jest.fn(), + updateState: jest.fn(), delete: jest.fn().mockResolvedValue(undefined), getSingleExperiment: jest.fn(), }; @@ -192,5 +193,131 @@ describe('ExperimentController adaptive config wiring', () => { expect(experimentService.update).toHaveBeenCalledTimes(1); }); + + it('rejects a condition field change once the experiment has started', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.RUNNING, + conditions: [{ id: 'condition-1', conditionCode: 'A', name: 'A', description: '', assignmentWeight: 50 }], + }; + const experiment = { + id: 'experiment-1', + conditions: [ + { id: 'condition-1', conditionCode: 'A-renamed', name: 'A', description: '', assignmentWeight: 50 }, + ], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be modified/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('rejects an added/removed condition once the experiment has started', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.PAUSED, + conditions: [{ id: 'condition-1', conditionCode: 'A', assignmentWeight: 100 }], + }; + const experiment = { + id: 'experiment-1', + conditions: [ + { id: 'condition-1', conditionCode: 'A', assignmentWeight: 50 }, + { id: 'condition-2', conditionCode: 'B', assignmentWeight: 50 }, + ], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be modified/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('rejects a Thompson Sampling prior change once the experiment has started', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.RUNNING, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 5, failure: 1 } } }, + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be modified/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('allows an unrelated field change once the experiment has started, when conditions/priors are unchanged', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.RUNNING, + name: 'old name', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A', assignmentWeight: 100 }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + }; + const experiment = { + id: 'experiment-1', + name: 'new name', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A', assignmentWeight: 100 }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: experiment.conditions }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + + it('allows a condition/prior change while the experiment has not started yet', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A-renamed' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 5, failure: 1 } } }, + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: experiment.conditions }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + }); + + describe('updateState()', () => { + it('attaches the Thompson Sampling config onto the state-change response, like the other endpoints', async () => { + const updatedExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.PAUSED, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + }; + experimentService.updateState.mockResolvedValue(updatedExperiment); + + const stateUpdate = { experimentId: 'experiment-1', state: EXPERIMENT_STATE.PAUSED } as any; + const result = await controller.updateState(stateUpdate, {} as any, request); + + expect(adaptiveExperimentConfigDispatcher.attachConfigToExperiment).toHaveBeenCalledWith(updatedExperiment); + expect(result).toBe(updatedExperiment); + }); }); }); From f05df20bc8def3d90e4b5da55b46cdfd1e6ba791 Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Wed, 16 Sep 2026 11:19:00 -0400 Subject: [PATCH 22/28] Change LOCAL port from 3032 to 3030 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- clientlibs/js/quickTestAdaptive.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clientlibs/js/quickTestAdaptive.ts b/clientlibs/js/quickTestAdaptive.ts index a7dae9078b..bf2d54e3a4 100644 --- a/clientlibs/js/quickTestAdaptive.ts +++ b/clientlibs/js/quickTestAdaptive.ts @@ -19,7 +19,7 @@ const URL = { // 3030 is the standard docker-compose port (see root CLAUDE.md); a git worktree set up via // /new-worktree auto-assigns its own port instead (check packages/backend/.env's APP_PORT) -- // update this if you're running in a worktree. - LOCAL: 'http://localhost:3032', + LOCAL: 'http://localhost:3030', ECS_QA: 'https://apps.qa-cli.net/upgrade-service', ECS_STAGING: 'https://apps.qa-cli.com/upgrade-service', }; From cb4ee389b54fcf442bf6725abf8ae4cba54cc2eb Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Wed, 16 Sep 2026 11:19:51 -0400 Subject: [PATCH 23/28] Optimize reward flushing with Promise.all Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/api/services/ThompsonSamplingRewardService.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index aea04452b1..72de6e6b48 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -174,7 +174,11 @@ export class ThompsonSamplingRewardService { } if (effectiveBatchSize <= 1) { - await this.flushPendingRewards(manager, current); + await Promise.all( + experimentStates + .filter((s) => s.pendingTotalCount > 0) + .map((s) => this.flushPendingRewards(manager, s)) + ); return; } From 2c54cae97a247183cca6c1ddc15ae3a540a24f8e Mon Sep 17 00:00:00 2001 From: danoswaltCL <97542869+danoswaltCL@users.noreply.github.com> Date: Wed, 16 Sep 2026 11:21:02 -0400 Subject: [PATCH 24/28] Fix condition to check updated experiment's algorithm Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/api/services/ThompsonSamplingExperimentCrudService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index dc03ce1cda..953d8aa323 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -103,7 +103,7 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment * experiment as Thompson Sampling. */ public async syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise { - if (experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + if (updatedExperiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { await this.deleteConfigIfExists(updatedExperiment.id); return; } From 6c78f346e405a02656c9e7d4c0fa6d4fe3433d3c Mon Sep 17 00:00:00 2001 From: doswalt Date: Wed, 16 Sep 2026 12:56:25 -0400 Subject: [PATCH 25/28] fix test fail --- .../backend/src/api/services/ThompsonSamplingRewardService.ts | 4 +--- .../services/ThompsonSamplingExperimentCrudService.test.ts | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts index 72de6e6b48..cf271e231b 100644 --- a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -175,9 +175,7 @@ export class ThompsonSamplingRewardService { if (effectiveBatchSize <= 1) { await Promise.all( - experimentStates - .filter((s) => s.pendingTotalCount > 0) - .map((s) => this.flushPendingRewards(manager, s)) + experimentStates.filter((s) => s.pendingTotalCount > 0).map((s) => this.flushPendingRewards(manager, s)) ); return; } diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts index 82702de599..39d32dd08f 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -229,6 +229,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { await service.syncConfigIfApplicable(experiment, { id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [{ id: 'condition-1' }], } as any); @@ -258,6 +259,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { await service.syncConfigIfApplicable(experiment, { id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [{ id: 'condition-1' }], } as any); From 5cf555a770e0a34133ac047d81a44534f6b94fa7 Mon Sep 17 00:00:00 2001 From: doswalt Date: Thu, 17 Sep 2026 14:14:44 -0400 Subject: [PATCH 26/28] do not allow switching to or from adaptive experiments in edit mode, keep it simple, no need to do so and too much to code up to handle it --- .../api/controllers/ExperimentController.ts | 23 +++++++ ...ExperimentControllerAdaptiveConfig.test.ts | 68 +++++++++++++++++-- .../upsert-experiment-modal.component.ts | 61 ++++++++++------- 3 files changed, 123 insertions(+), 29 deletions(-) diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index 19efd57086..a26c937458 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -33,6 +33,7 @@ import { Response } from 'express'; import { NotFoundException } from '@nestjs/common/exceptions'; import { ExperimentIdValidator } from '../DTO/ExperimentDTO'; import { + ASSIGNMENT_ALGORITHM, CACHE_PREFIX, EXPERIMENT_STATE, IImportError, @@ -1266,6 +1267,7 @@ export class ExperimentController { const previousExperiment = await this.experimentService.getSingleExperiment(id, request.logger); if (previousExperiment) { await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(previousExperiment); + this.assertAssignmentAlgorithmNotChangedToOrFromThompsonSampling(previousExperiment, experiment); this.assertConditionsNotModifiedAfterStart(previousExperiment, experiment); } @@ -1302,6 +1304,27 @@ export class ExperimentController { return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(updatedExperiment); } + /** + * Switching an experiment to or from Thompson Sampling on an update is not supported at all -- + * regardless of experiment state, even INACTIVE/DRAFT -- because there is no sane way to backfill + * or discard the config/posterior rows a switch implies. The intended workflow is to delete the + * experiment and create a new one with the desired algorithm instead. Unlike + * assertConditionsNotModifiedAfterStart below, this has no "not started yet" exception. + */ + private assertAssignmentAlgorithmNotChangedToOrFromThompsonSampling( + previousExperiment: ExperimentDTO, + incomingExperiment: ExperimentDTO + ): void { + const wasThompsonSampling = previousExperiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + const isThompsonSampling = incomingExperiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + + if (wasThompsonSampling !== isThompsonSampling) { + throw new BadRequestError( + 'The assignment algorithm cannot be changed to or from Thompson Sampling. Create a new experiment instead.' + ); + } + } + /** * Mirrors the frontend's own restriction (selectSectionCardRestriction / * selectDisabledExperimentFields in experiments.selectors.ts): conditions -- including Thompson diff --git a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts index baa51bd30d..b13f1acc8e 100644 --- a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts +++ b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts @@ -72,7 +72,7 @@ describe('ExperimentController adaptive config wiring', () => { describe('update()', () => { it('does not touch previous state on the happy path', async () => { const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; - const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: [] }); @@ -84,9 +84,9 @@ describe('ExperimentController adaptive config wiring', () => { it('reverts the experiment and re-syncs the config when syncConfigIfApplicable fails', async () => { const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; - const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; const updatedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; - const revertedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + const revertedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; const syncError = new Error('config sync failed'); experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); @@ -162,7 +162,7 @@ describe('ExperimentController adaptive config wiring', () => { it('still throws the original error, logged rather than masked, when the revert attempt itself fails', async () => { const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; - const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM }; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; const syncError = new Error('config sync failed'); const revertError = new Error('revert update failed'); @@ -281,6 +281,66 @@ describe('ExperimentController adaptive config wiring', () => { expect(experimentService.update).toHaveBeenCalledTimes(1); }); + it('rejects switching the assignment algorithm from Thompson Sampling to something else, even on an inactive experiment', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [], + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + conditions: [], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be changed to or from Thompson Sampling/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('rejects switching the assignment algorithm to Thompson Sampling from something else, even on an inactive experiment', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + conditions: [], + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be changed to or from Thompson Sampling/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('allows switching between two non-Thompson-Sampling algorithms', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + conditions: [], + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING, + conditions: [], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: [] }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + it('allows a condition/prior change while the experiment has not started yet', async () => { const previousExperiment = { id: 'experiment-1', diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts index f9069c65e5..d0b18d77ab 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts @@ -487,10 +487,7 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { // Thompson Sampling algorithm back to Random rather than leave an invalid combination // sitting in the form. this.updateAssignmentAlgorithms(); - if ( - assignmentUnit === ASSIGNMENT_UNIT.WITHIN_SUBJECTS && - this.assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING - ) { + if (assignmentUnit === ASSIGNMENT_UNIT.WITHIN_SUBJECTS && this.isCurrentAlgorithmThompsonSampling) { this.experimentForm.get('assignmentAlgorithm')?.setValue(ASSIGNMENT_ALGORITHM.RANDOM); } }) @@ -498,9 +495,7 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { } checkForAlgorithmChange(): void { - const algorithm = this.assignmentAlgorithmValue; - - if (algorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + if (this.isCurrentAlgorithmThompsonSampling) { if (!this.thompsonSamplingConfigFormValue) { this.thompsonSamplingConfigFormValue = this.thompsonSamplingHelperService.buildConfig( this.thompsonSamplingHelperService.getDefaults() @@ -508,35 +503,47 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { } } else { this.thompsonSamplingConfigFormValue = undefined; - // The TS sub-form is about to be removed from the DOM (its @if goes false) without emitting - // a final validity event. Without resetting these, a form that was invalid at the moment of - // switching away would leave isTSFormValid$ stuck at false, permanently disabling Save for - // an otherwise-valid non-TS experiment until the modal is reopened. this.isTSFormValid$.next(true); this.isTSFormChanged$.next(false); } } + setAlgorithmDisabled(algorithm: ASSIGNMENT_ALGORITHM, disabled: boolean): void { + const option = this.assignmentAlgorithms.find((alg) => alg.value === algorithm); + if (option) { + (option as any).disabled = disabled; + } + } + updateAssignmentAlgorithms(): void { - // Disable stratified random sampling if no stratification factors are available - const stratifiedAlgorithm = this.assignmentAlgorithms.find( - (alg) => alg.value === ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING + // Switching an experiment to or from Thompson Sampling on edit isn't supported + const isEditingExistingExperiment = this.config.params.action === UPSERT_EXPERIMENT_ACTION.EDIT; + const wasThompsonSampling = + isEditingExistingExperiment && + this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm( + this.config.params.sourceExperiment?.assignmentAlgorithm + ); + + // Disable stratified random sampling if no stratification factors are available, or if editing + // an experiment that's already Thompson Sampling (switching away isn't allowed). + this.setAlgorithmDisabled( + ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING, + this.allStratificationFactors.length === 0 || wasThompsonSampling ); - if (stratifiedAlgorithm) { - (stratifiedAlgorithm as any).disabled = this.allStratificationFactors.length === 0; - } + // Disable Random for the same "already Thompson Sampling" reason as above. + this.setAlgorithmDisabled(ASSIGNMENT_ALGORITHM.RANDOM, wasThompsonSampling); // Thompson Sampling can't be used with Within-Subjects assignment: that assignment unit never // stores a condition on the individual enrollment (it's tracked per-repeat instead), which is - // what Thompson Sampling's reward path reads to attribute a reward to a condition. - const thompsonSamplingAlgorithm = this.assignmentAlgorithms.find( - (alg) => alg.value === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING + // what Thompson Sampling's reward path reads to attribute a reward to a condition. It's also + // disabled when editing an experiment that *wasn't* created as Thompson Sampling, for the + // switch-lock reason above. + this.setAlgorithmDisabled( + ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + this.unitOfAssignmentValue === ASSIGNMENT_UNIT.WITHIN_SUBJECTS || + (isEditingExistingExperiment && !wasThompsonSampling) ); - - if (thompsonSamplingAlgorithm) { - (thompsonSamplingAlgorithm as any).disabled = this.unitOfAssignmentValue === ASSIGNMENT_UNIT.WITHIN_SUBJECTS; - } } validateStratificationFactorSelection(algorithm: ASSIGNMENT_ALGORITHM): void { @@ -562,6 +569,10 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { return this.experimentForm.get('unitOfAssignment')?.value; } + get isCurrentAlgorithmThompsonSampling(): boolean { + return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(this.assignmentAlgorithmValue); + } + /** * Event handlers for Thompson Sampling config child form * @@ -783,7 +794,7 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { revertTo: sourceExperiment.revertTo, }; - if (this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(assignmentAlgorithm)) { + if (this.isCurrentAlgorithmThompsonSampling) { experimentRequest.thompsonSamplingConfig = this.thompsonSamplingConfigFormValue; } else { experimentRequest.thompsonSamplingConfig = undefined; From cdcb06e58219170ac883b12a2fa4cb16d3010295 Mon Sep 17 00:00:00 2001 From: doswalt Date: Tue, 22 Sep 2026 14:26:23 -0400 Subject: [PATCH 27/28] experimentId property for conditionposteriorstate --- .../src/api/models/ConditionPosteriorState.ts | 8 ++++ .../ThompsonSamplingExperimentCrudService.ts | 2 + .../1788362726319-nativeThompsonSampling.ts | 45 ++++++++++++++++--- ...mpsonSamplingExperimentCrudService.test.ts | 5 +++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/api/models/ConditionPosteriorState.ts b/packages/backend/src/api/models/ConditionPosteriorState.ts index 491e9d480a..a24bd79424 100644 --- a/packages/backend/src/api/models/ConditionPosteriorState.ts +++ b/packages/backend/src/api/models/ConditionPosteriorState.ts @@ -1,6 +1,7 @@ import { Entity, Column, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Unique } from 'typeorm'; import { ThompsonSamplingExperimentConfig } from './ThompsonSamplingExperimentConfig'; import { ExperimentCondition } from './ExperimentCondition'; +import { Experiment } from './Experiment'; import { BaseModel } from './base/BaseModel'; @Entity() @@ -9,6 +10,13 @@ export class ConditionPosteriorState extends BaseModel { @PrimaryGeneratedColumn('uuid') public id: string; + @ManyToOne(() => Experiment, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'experimentId' }) + experiment: Experiment; + + @Column() + experimentId: string; + @ManyToOne(() => ThompsonSamplingExperimentConfig, (config) => config.conditionPosteriorStates, { onDelete: 'CASCADE', }) diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts index 953d8aa323..2ae0c0b466 100644 --- a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -162,6 +162,7 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment await Promise.all( conditions.map((condition) => this.posteriorStateRepository.save({ + experimentId, configId: config.id, conditionId: condition.id, priorSuccess: params.priors?.[condition.id]?.success ?? 1, @@ -306,6 +307,7 @@ export class ThompsonSamplingExperimentCrudService implements AdaptiveExperiment await Promise.all( toAdd.map((condition) => this.posteriorStateRepository.save({ + experimentId, configId: config.id, conditionId: condition.id, priorSuccess: 1, diff --git a/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts index f48d2818cc..0eebc91664 100644 --- a/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts +++ b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts @@ -39,6 +39,16 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_version_condition_map"`); await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_experiment_ref"`); + // experiment_condition pre-exists this migration (base schema), so its composite unique + // constraint is added here via ALTER rather than in a CREATE TABLE. It's the target of the + // composite FKs below: pairing the already-unique "id" with "experimentId" lets a child row + // pin down *which experiment's* condition it's referencing, not just that the id exists + // somewhere, without changing the real uniqueness semantics (a condition's id is still + // globally unique on its own). + await queryRunner.query( + `ALTER TABLE "experiment_condition" ADD CONSTRAINT "UQ_experiment_condition_experimentId_id" UNIQUE ("experimentId", "id")` + ); + // thompson_sampling_experiment_config: one-to-one with experiment await queryRunner.query( `CREATE TABLE "thompson_sampling_experiment_config" ( @@ -51,16 +61,21 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), "versionNumber" integer NOT NULL, CONSTRAINT "UQ_ts_config_experimentId" UNIQUE ("experimentId"), + CONSTRAINT "UQ_ts_config_experimentId_id" UNIQUE ("experimentId", "id"), CONSTRAINT "PK_ts_config" PRIMARY KEY ("id") )` ); // condition_posterior_state: per-condition Beta distribution state. pendingSuccessCount/ // pendingTotalCount buffer rewards between batch flushes (see ThompsonSamplingRewardService) — - // included from the start since nothing has been applied anywhere yet. + // included from the start since nothing has been applied anywhere yet. "experimentId" is + // denormalized down from the config so configId/conditionId can be tied together with composite + // FKs below, rather than two independent simple FKs that could each point at a *different* + // experiment's config/condition with nothing to catch it. await queryRunner.query( `CREATE TABLE "condition_posterior_state" ( "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "experimentId" uuid NOT NULL, "configId" uuid NOT NULL, "conditionId" uuid NOT NULL, "priorSuccess" double precision NOT NULL DEFAULT 1, @@ -100,17 +115,27 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { await queryRunner.query( `ALTER TABLE "thompson_sampling_experiment_config" ADD CONSTRAINT "FK_ts_config_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` ); + // condition_posterior_state gets a direct FK on experimentId (for a clean cascade if the + // experiment itself is deleted) plus two *composite* FKs -- each requiring experimentId to + // agree with the config's/condition's own experimentId, so a row can no longer pair a config + // and a condition that belong to different experiments. await queryRunner.query( - `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_config" FOREIGN KEY ("configId") REFERENCES "thompson_sampling_experiment_config"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` ); await queryRunner.query( - `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_condition" FOREIGN KEY ("conditionId") REFERENCES "experiment_condition"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_config" FOREIGN KEY ("experimentId", "configId") REFERENCES "thompson_sampling_experiment_config"("experimentId", "id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_condition" FOREIGN KEY ("experimentId", "conditionId") REFERENCES "experiment_condition"("experimentId", "id") ON DELETE CASCADE ON UPDATE NO ACTION` ); await queryRunner.query( `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` ); + // Same fix as condition_posterior_state above: thompson_sampling_reward already denormalizes + // experimentId (originally just for the composite index below), so pairing it with conditionId + // in this FK is a small extension that closes the identical gap here. await queryRunner.query( - `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_condition" FOREIGN KEY ("conditionId") REFERENCES "experiment_condition"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_condition" FOREIGN KEY ("experimentId", "conditionId") REFERENCES "experiment_condition"("experimentId", "id") ON DELETE CASCADE ON UPDATE NO ACTION` ); // Bootstrap config + posterior state rows for any experiment already flagged thompson_sampling @@ -125,8 +150,8 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { await queryRunner.query(` INSERT INTO "condition_posterior_state" - ("configId", "conditionId", "priorSuccess", "priorFailure", "successCount", "failureCount", "totalCount", "pendingSuccessCount", "pendingFailureCount", "pendingTotalCount", "versionNumber") - SELECT c.id, ec.id, 1, 1, 0, 0, 0, 0, 0, 0, 1 + ("experimentId", "configId", "conditionId", "priorSuccess", "priorFailure", "successCount", "failureCount", "totalCount", "pendingSuccessCount", "pendingFailureCount", "pendingTotalCount", "versionNumber") + SELECT c."experimentId", c.id, ec.id, 1, 1, 0, 0, 0, 0, 0, 0, 1 FROM "thompson_sampling_experiment_config" c JOIN "experiment_condition" ec ON ec."experimentId" = c."experimentId" `); @@ -137,6 +162,7 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { await queryRunner.query(`ALTER TABLE "thompson_sampling_reward" DROP CONSTRAINT "FK_ts_reward_experiment"`); await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_condition"`); await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_config"`); + await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_experiment"`); await queryRunner.query( `ALTER TABLE "thompson_sampling_experiment_config" DROP CONSTRAINT "FK_ts_config_experiment"` ); @@ -146,6 +172,13 @@ export class NativeThompsonSampling1788362726319 implements MigrationInterface { await queryRunner.query(`DROP TABLE "condition_posterior_state"`); await queryRunner.query(`DROP TABLE "thompson_sampling_experiment_config"`); + // experiment_condition pre-exists this migration and isn't dropped above, so its composite + // unique constraint (added in up(), once the referencing composite FKs above are gone) needs + // an explicit drop rather than going away with the table. + await queryRunner.query( + `ALTER TABLE "experiment_condition" DROP CONSTRAINT "UQ_experiment_condition_experimentId_id"` + ); + // Best-effort: the pre-migration enum has no 'thompson_sampling' value, so any experiment left in // that state would fail the column cast below. Fall back to 'random' — this is a rollback of a // feature that was never live, not a data-preserving downgrade. diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts index 39d32dd08f..6d37a5b434 100644 --- a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -116,6 +116,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { expect.objectContaining({ experimentId: 'experiment-1', warmupThreshold: 10, batchSize: 5 }) ); expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', configId: 'config-1', conditionId: 'condition-1', priorSuccess: 7, @@ -150,6 +151,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { ); expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', configId: 'config-1', conditionId: 'server-id-1', priorSuccess: 7, @@ -158,6 +160,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { totalCount: 0, }); expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', configId: 'config-1', conditionId: 'server-id-2', priorSuccess: 3, @@ -179,6 +182,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { } as any); expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', configId: 'config-1', conditionId: 'server-id-1', priorSuccess: 1, @@ -237,6 +241,7 @@ describe('ThompsonSamplingExperimentCrudService', () => { expect.objectContaining({ experimentId: 'experiment-1', batchSize: 5 }) ); expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', configId: 'config-1', conditionId: 'condition-1', priorSuccess: 2, From f48f28f11abfbed8bca71f5a1d88908a30aa4fa5 Mon Sep 17 00:00:00 2001 From: doswalt Date: Tue, 22 Sep 2026 15:19:37 -0400 Subject: [PATCH 28/28] kick cicd